From 85ae113d5e7c97f2879181e37c0259ae57a828fc Mon Sep 17 00:00:00 2001 From: iGN5117 Date: Tue, 6 Jun 2023 13:29:39 -0700 Subject: [PATCH 01/41] Remove prebuild_index config from search plugin --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 996dc6ae09..a9ec3645d5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -164,7 +164,7 @@ markdown_extensions: - pymdownx.tilde plugins: - search: - prebuild_index: true + #prebuild_index: true - macros - git-revision-date-localized: type: datetime From 8e829feab2f64542b86d609d7216d6218b2d4248 Mon Sep 17 00:00:00 2001 From: iGN5117 Date: Tue, 6 Jun 2023 13:30:32 -0700 Subject: [PATCH 02/41] Update compile the code documentation for sedona --- docs/setup/compile.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/setup/compile.md b/docs/setup/compile.md index b3ae72b7bd..3958e1cc22 100644 --- a/docs/setup/compile.md +++ b/docs/setup/compile.md @@ -6,7 +6,7 @@ ## Compile Scala / Java source code Sedona Scala/Java code is a project with multiple modules. Each module is a Scala/Java mixed project which is managed by Apache Maven 3. -* Make sure your Linux/Mac machine has Java 1.8, Apache Maven 3.3.1+, and Python3. The compilation of Sedona is not tested on Windows machine. +* Make sure your Linux/Mac machine has Java 1.8, Apache Maven 3.3.1+, and Python3.7+. The compilation of Sedona is not tested on Windows machine. To compile all modules, please make sure you are in the root folder of all modules. Then enter the following command in the terminal: @@ -76,7 +76,7 @@ export PYTHONPATH=$SPARK_HOME/python ``` 2. Compile the Sedona Scala and Java code with `-Dgeotools` and then copy the ==sedona-spark-shaded-{{ sedona.current_version }}.jar== to ==SPARK_HOME/jars/== folder. ``` -cp spark-shaded/target/sedona-spark-shaded-xxx.jar SPARK_HOME/jars/ +cp spark-shaded/target/sedona-spark-shaded-xxx.jar $SPARK_HOME/jars/ ``` 3. Install the following libraries ``` @@ -86,6 +86,9 @@ sudo pip3 install -U wheel sudo pip3 install -U virtualenvwrapper sudo pip3 install -U pipenv ``` +!!!tip + Homebrew can be used to install libgeos-dev in macOS: ```brew install geos``` + 4. Set up pipenv to the desired Python version: 3.7, 3.8, or 3.9 ``` cd python @@ -94,7 +97,8 @@ pipenv --python 3.7 5. Install the PySpark version and other dependency ``` cd python -pipenv install pyspark==3.0.1 +pipenv install pyspark +pipenv install shapely~=1.7 pipenv install --dev ``` 6. Run the Python tests From 4bcf99caa555e9cfc14f60e281459d80a4d668c0 Mon Sep 17 00:00:00 2001 From: iGN5117 Date: Tue, 6 Jun 2023 13:32:44 -0700 Subject: [PATCH 03/41] approximate float comparisons in python test cases --- python/sedona/core/geom/envelope.py | 11 ++++++++++- .../streaming/spark/test_constructor_functions.py | 13 ++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/python/sedona/core/geom/envelope.py b/python/sedona/core/geom/envelope.py index c0a90eb35a..d449f7c600 100644 --- a/python/sedona/core/geom/envelope.py +++ b/python/sedona/core/geom/envelope.py @@ -19,7 +19,7 @@ from shapely.geometry.base import BaseGeometry from sedona.utils.decorators import require - +import math class Envelope(Polygon): @@ -35,6 +35,15 @@ def __init__(self, minx=0, maxx=1, miny=0, maxy=1): [self.maxx, self.miny] ]) + def isClose(self, a, b) -> bool: + return math.isclose(a, b, rel_tol=1e-9) + + def __eq__(self, other) -> bool: + return self.isClose(self.minx, other.minx) and\ + self.isClose(self.miny, other.miny) and\ + self.isClose(self.maxx, other.maxx) and\ + self.isClose(self.maxy, other.maxy) + @require(["Envelope"]) def create_jvm_instance(self, jvm): return jvm.Envelope( diff --git a/python/tests/streaming/spark/test_constructor_functions.py b/python/tests/streaming/spark/test_constructor_functions.py index 6fc55b9956..2136cd0616 100644 --- a/python/tests/streaming/spark/test_constructor_functions.py +++ b/python/tests/streaming/spark/test_constructor_functions.py @@ -27,6 +27,7 @@ from tests import tests_resource from tests.streaming.spark.cases_builder import SuiteContainer from tests.test_base import TestBase +import math SCHEMA = StructType( [ @@ -80,8 +81,10 @@ (SuiteContainer.empty() .with_function_name("ST_Transform") .with_arguments(["ST_GeomFromText('POINT(21.5 52.5)')", "'epsg:4326'", "'epsg:2180'"]) - .with_expected_result("POINT (-2501415.806893427 4119952.52325666)") - .with_transform("ST_ASText")), + .with_expected_result(-2501415.806893427) + #.with_expected_result("POINT (-2501415.806893427 4119952.52325666)") + .with_transform("ST_X")), + #.with_transform("ST_ASText")), (SuiteContainer.empty() .with_function_name("ST_Intersection") .with_arguments(["ST_GeomFromText('POINT(21.5 52.5)')", "ST_GeomFromText('POINT(21.5 52.5)')"]) @@ -338,4 +341,8 @@ def test_geospatial_function_on_stream(self, function_name: str, arguments: List # then result should be as expected transform_query = "result" if not transform else f"{transform}(result)" - assert self.spark.sql(f"select {transform_query} from {random_table_name}").collect()[0][0] == expected_result + queryResult = self.spark.sql(f"select {transform_query} from {random_table_name}").collect()[0][0] + if (type(queryResult) is float and type(expected_result) is float): + assert math.isclose(queryResult, expected_result, rel_tol=1e-9) + else: + assert queryResult == expected_result From c7f6236d02035292f4b59aa1625982a34acfa003 Mon Sep 17 00:00:00 2001 From: iGN5117 Date: Tue, 6 Jun 2023 13:30:32 -0700 Subject: [PATCH 04/41] Update compile the code documentation for sedona --- docs/setup/compile.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/setup/compile.md b/docs/setup/compile.md index b3ae72b7bd..3958e1cc22 100644 --- a/docs/setup/compile.md +++ b/docs/setup/compile.md @@ -6,7 +6,7 @@ ## Compile Scala / Java source code Sedona Scala/Java code is a project with multiple modules. Each module is a Scala/Java mixed project which is managed by Apache Maven 3. -* Make sure your Linux/Mac machine has Java 1.8, Apache Maven 3.3.1+, and Python3. The compilation of Sedona is not tested on Windows machine. +* Make sure your Linux/Mac machine has Java 1.8, Apache Maven 3.3.1+, and Python3.7+. The compilation of Sedona is not tested on Windows machine. To compile all modules, please make sure you are in the root folder of all modules. Then enter the following command in the terminal: @@ -76,7 +76,7 @@ export PYTHONPATH=$SPARK_HOME/python ``` 2. Compile the Sedona Scala and Java code with `-Dgeotools` and then copy the ==sedona-spark-shaded-{{ sedona.current_version }}.jar== to ==SPARK_HOME/jars/== folder. ``` -cp spark-shaded/target/sedona-spark-shaded-xxx.jar SPARK_HOME/jars/ +cp spark-shaded/target/sedona-spark-shaded-xxx.jar $SPARK_HOME/jars/ ``` 3. Install the following libraries ``` @@ -86,6 +86,9 @@ sudo pip3 install -U wheel sudo pip3 install -U virtualenvwrapper sudo pip3 install -U pipenv ``` +!!!tip + Homebrew can be used to install libgeos-dev in macOS: ```brew install geos``` + 4. Set up pipenv to the desired Python version: 3.7, 3.8, or 3.9 ``` cd python @@ -94,7 +97,8 @@ pipenv --python 3.7 5. Install the PySpark version and other dependency ``` cd python -pipenv install pyspark==3.0.1 +pipenv install pyspark +pipenv install shapely~=1.7 pipenv install --dev ``` 6. Run the Python tests From ceed7a46cfa6376859aa0b18749e19a79d21ebe9 Mon Sep 17 00:00:00 2001 From: iGN5117 Date: Tue, 6 Jun 2023 13:29:39 -0700 Subject: [PATCH 05/41] Remove prebuild_index config from search plugin --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 996dc6ae09..a9ec3645d5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -164,7 +164,7 @@ markdown_extensions: - pymdownx.tilde plugins: - search: - prebuild_index: true + #prebuild_index: true - macros - git-revision-date-localized: type: datetime From 15c62557202263a683baae0d973c9c706aba05f3 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Tue, 6 Jun 2023 18:34:46 -0700 Subject: [PATCH 06/41] Revert "Update compile the code documentation for sedona" This reverts commit c7f6236d02035292f4b59aa1625982a34acfa003. --- docs/setup/compile.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/setup/compile.md b/docs/setup/compile.md index 3958e1cc22..b3ae72b7bd 100644 --- a/docs/setup/compile.md +++ b/docs/setup/compile.md @@ -6,7 +6,7 @@ ## Compile Scala / Java source code Sedona Scala/Java code is a project with multiple modules. Each module is a Scala/Java mixed project which is managed by Apache Maven 3. -* Make sure your Linux/Mac machine has Java 1.8, Apache Maven 3.3.1+, and Python3.7+. The compilation of Sedona is not tested on Windows machine. +* Make sure your Linux/Mac machine has Java 1.8, Apache Maven 3.3.1+, and Python3. The compilation of Sedona is not tested on Windows machine. To compile all modules, please make sure you are in the root folder of all modules. Then enter the following command in the terminal: @@ -76,7 +76,7 @@ export PYTHONPATH=$SPARK_HOME/python ``` 2. Compile the Sedona Scala and Java code with `-Dgeotools` and then copy the ==sedona-spark-shaded-{{ sedona.current_version }}.jar== to ==SPARK_HOME/jars/== folder. ``` -cp spark-shaded/target/sedona-spark-shaded-xxx.jar $SPARK_HOME/jars/ +cp spark-shaded/target/sedona-spark-shaded-xxx.jar SPARK_HOME/jars/ ``` 3. Install the following libraries ``` @@ -86,9 +86,6 @@ sudo pip3 install -U wheel sudo pip3 install -U virtualenvwrapper sudo pip3 install -U pipenv ``` -!!!tip - Homebrew can be used to install libgeos-dev in macOS: ```brew install geos``` - 4. Set up pipenv to the desired Python version: 3.7, 3.8, or 3.9 ``` cd python @@ -97,8 +94,7 @@ pipenv --python 3.7 5. Install the PySpark version and other dependency ``` cd python -pipenv install pyspark -pipenv install shapely~=1.7 +pipenv install pyspark==3.0.1 pipenv install --dev ``` 6. Run the Python tests From 471e281fda3a32498f92e021b87127c2e9c990be Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Tue, 6 Jun 2023 18:34:56 -0700 Subject: [PATCH 07/41] Revert "Remove prebuild_index config from search plugin" This reverts commit 85ae113d5e7c97f2879181e37c0259ae57a828fc. --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index a9ec3645d5..996dc6ae09 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -164,7 +164,7 @@ markdown_extensions: - pymdownx.tilde plugins: - search: - #prebuild_index: true + prebuild_index: true - macros - git-revision-date-localized: type: datetime From 2a5a6d8d03766793a60c6b382548d4738ee2b5ac Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Tue, 6 Jun 2023 18:42:03 -0700 Subject: [PATCH 08/41] FIxed incorrect indentation --- .../spark/test_constructor_functions.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/python/tests/streaming/spark/test_constructor_functions.py b/python/tests/streaming/spark/test_constructor_functions.py index 2136cd0616..3ab4258d63 100644 --- a/python/tests/streaming/spark/test_constructor_functions.py +++ b/python/tests/streaming/spark/test_constructor_functions.py @@ -322,27 +322,27 @@ class TestConstructorFunctions(TestBase): @pytest.mark.sparkstreaming def test_geospatial_function_on_stream(self, function_name: str, arguments: List[str], expected_result: Any, transform: Optional[str]): - # given input stream + # given input stream - input_stream = self.spark.readStream.schema(SCHEMA).parquet(os.path.join( - tests_resource, - "streaming/geometry_example") - ).selectExpr(f"{function_name}({', '.join(arguments)}) AS result") + input_stream = self.spark.readStream.schema(SCHEMA).parquet(os.path.join( + tests_resource, + "streaming/geometry_example") + ).selectExpr(f"{function_name}({', '.join(arguments)}) AS result") - # and target table - random_table_name = f"view_{uuid.uuid4().hex}" + # and target table + random_table_name = f"view_{uuid.uuid4().hex}" - # when saving stream to memory - streaming_query = input_stream.writeStream.format("memory") \ - .queryName(random_table_name) \ - .outputMode("append").start() + # when saving stream to memory + streaming_query = input_stream.writeStream.format("memory") \ + .queryName(random_table_name) \ + .outputMode("append").start() - streaming_query.processAllAvailable() + streaming_query.processAllAvailable() - # then result should be as expected - transform_query = "result" if not transform else f"{transform}(result)" - queryResult = self.spark.sql(f"select {transform_query} from {random_table_name}").collect()[0][0] - if (type(queryResult) is float and type(expected_result) is float): + # then result should be as expected + transform_query = "result" if not transform else f"{transform}(result)" + queryResult = self.spark.sql(f"select {transform_query} from {random_table_name}").collect()[0][0] + if (type(queryResult) is float and type(expected_result) is float): assert math.isclose(queryResult, expected_result, rel_tol=1e-9) - else: + else: assert queryResult == expected_result From 6e9883d3b6c57cecaddcf3400dda4d1fe490b331 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Wed, 7 Jun 2023 09:46:13 -0700 Subject: [PATCH 09/41] Addressed PR comments for documentation changes Removed note/tip block elements that caused numbering to be reset --- docs/setup/compile.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/setup/compile.md b/docs/setup/compile.md index 3958e1cc22..bb9fc57d88 100644 --- a/docs/setup/compile.md +++ b/docs/setup/compile.md @@ -66,9 +66,7 @@ User can specify `-Dspark` and `-Dscala` command line options to compile with di Sedona uses GitHub action to automatically generate jars per commit. You can go [here](https://github.com/apache/sedona/actions/workflows/java.yml) and download the jars by clicking the commit's ==Artifacts== tag. ## Run Python test - 1. Set up the environment variable SPARK_HOME and PYTHONPATH - For example, ``` export SPARK_HOME=$PWD/spark-3.0.1-bin-hadoop2.7 @@ -86,9 +84,7 @@ sudo pip3 install -U wheel sudo pip3 install -U virtualenvwrapper sudo pip3 install -U pipenv ``` -!!!tip - Homebrew can be used to install libgeos-dev in macOS: ```brew install geos``` - +Homebrew can be used to install libgeos-dev in macOS: `brew install geos` 4. Set up pipenv to the desired Python version: 3.7, 3.8, or 3.9 ``` cd python @@ -98,9 +94,10 @@ pipenv --python 3.7 ``` cd python pipenv install pyspark -pipenv install shapely~=1.7 pipenv install --dev ``` +`pipenv install pyspark` install the latest version of pyspark. +In order to remain consistent with installed spark version, use `pipenv install pyspark==` 6. Run the Python tests ``` cd python From cf7e78d35783a6f6f9119d4e1dce3103bceaca2a Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Wed, 7 Jun 2023 23:23:14 -0700 Subject: [PATCH 10/41] Add ST_NumPoints --- .../java/org/apache/sedona/common/Functions.java | 8 ++++++++ .../org/apache/sedona/common/FunctionsTest.java | 16 ++++++++++++++++ docs/api/flink/Function.md | 15 +++++++++++++++ docs/api/sql/Function.md | 14 ++++++++++++++ .../java/org/apache/sedona/flink/Catalog.java | 3 ++- .../sedona/flink/expressions/Functions.java | 8 ++++++++ .../org/apache/sedona/flink/FunctionTest.java | 8 ++++++++ python/sedona/sql/st_functions.py | 10 ++++++++++ python/tests/sql/test_dataframe_api.py | 1 + python/tests/sql/test_function.py | 5 +++++ .../org/apache/sedona/sql/UDF/Catalog.scala | 1 + .../sql/sedona_sql/expressions/Functions.scala | 7 +++++++ .../sedona_sql/expressions/st_functions.scala | 4 ++++ .../sedona/sql/dataFrameAPITestScala.scala | 8 ++++++++ .../apache/sedona/sql/functionTestScala.scala | 12 ++++++++++++ 15 files changed, 119 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index ad7af11986..fa9d184300 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -845,6 +845,14 @@ private static Coordinate[] extractCoordinates(Geometry geometry) { return coordinates; } + public static int numPoints(Geometry geometry) throws Exception { + String geometryType = geometry.getGeometryType(); + if (!(Geometry.TYPENAME_LINESTRING.equalsIgnoreCase(geometryType))) { + throw new Exception("Unsupported geometry type: " + geometryType); + } + return geometry.getNumPoints(); + } + public static Geometry geometricMedian(Geometry geometry, double tolerance, int maxIter, boolean failIfNotConverged) throws Exception { String geometryType = geometry.getGeometryType(); if(!(Geometry.TYPENAME_POINT.equals(geometryType) || Geometry.TYPENAME_MULTIPOINT.equals(geometryType))) { diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index 029c888a6f..bca6e61462 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -565,4 +565,20 @@ public void spheroidLength() { GeometryCollection geometryCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {point, line, multiLineString}); assertEquals(3.0056262514183864E7, Spheroid.length(geometryCollection), 0.1); } + + @Test + public void numPoints() throws Exception{ + LineString line = GEOMETRY_FACTORY.createLineString(coordArray(0, 1, 1, 0, 2, 0)); + int expected = 3; + int actual = Functions.numPoints(line); + assertEquals(expected, actual); + } + + @Test + public void numPointsUnsupported() throws Exception { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(0, 0, 0, 90, 0, 0)); + String expected = "Unsupported geometry type: " + "Polygon"; + Exception e = assertThrows(Exception.class, () -> Functions.numPoints(polygon)); + assertEquals(expected, e.getMessage()); + } } diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index e2f1d14241..ffdeaac8c2 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -1035,3 +1035,18 @@ SELECT ST_ZMin(ST_GeomFromText('LINESTRING(1 3 4, 5 6 7)')) ``` Output: `4.0` + +## ST_NumPoints + +Introduction: Returns number of points in a LineString + +Format: `ST_NumPoints(geom: geometry)` + +Since: `v1.4.2` + +Spark SQL example: +```sql +SELECT ST_NumPoints(ST_GeomFromText('LINESTRING(1 2, 1 3)')) +``` + +Output: `2` diff --git a/docs/api/sql/Function.md b/docs/api/sql/Function.md index 3eba9a036f..4a6fb9c623 100644 --- a/docs/api/sql/Function.md +++ b/docs/api/sql/Function.md @@ -1668,3 +1668,17 @@ SELECT ST_ZMin(ST_GeomFromText('LINESTRING(1 3 4, 5 6 7)')) Output: `4.0` +## ST_NumPoints +Introduction: Returns number of points in a LineString + +Format: `ST_NumPoints(geom: geometry)` + +Since: `v1.4.2` + +Spark SQL example: +```sql +SELECT ST_NumPoints(ST_GeomFromText('LINESTRING(0 1, 1 0, 2 0)')) +``` + +Output: `3` + diff --git a/flink/src/main/java/org/apache/sedona/flink/Catalog.java b/flink/src/main/java/org/apache/sedona/flink/Catalog.java index 66e4bffa2b..884126c00e 100644 --- a/flink/src/main/java/org/apache/sedona/flink/Catalog.java +++ b/flink/src/main/java/org/apache/sedona/flink/Catalog.java @@ -94,7 +94,8 @@ public static UserDefinedFunction[] getFuncs() { new Functions.ST_LineFromMultiPoint(), new Functions.ST_Split(), new Functions.ST_S2CellIDs(), - new Functions.ST_GeometricMedian() + new Functions.ST_GeometricMedian(), + new Functions.ST_NumPoints() }; } diff --git a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java index 608a461bdc..7001345a4c 100644 --- a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java +++ b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java @@ -574,4 +574,12 @@ public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.j } + public static class ST_NumPoints extends ScalarFunction { + @DataTypeHint(value = "Integer") + public int eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o) throws Exception { + Geometry geometry = (Geometry) o; + return org.apache.sedona.common.Functions.numPoints(geometry); + } + } + } diff --git a/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java b/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java index 933e216fe1..eac04d2fde 100644 --- a/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java +++ b/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java @@ -691,4 +691,12 @@ public void testGeometricMedianParamsFull() throws ParseException { 0, expected.compareTo(actual, COORDINATE_SEQUENCE_COMPARATOR)); } + @Test + public void testNumPoints() { + Integer expected = 3; + Table pointTable = tableEnv.sqlQuery("SELECT ST_NumPoints(ST_GeomFromWKT('LINESTRING(0 1, 1 0, 2 0)'))"); + Integer actual = (Integer) first(pointTable).getField(0); + assertEquals(expected, actual); + } + } diff --git a/python/sedona/sql/st_functions.py b/python/sedona/sql/st_functions.py index f3cea50b4b..d5c7602b1d 100644 --- a/python/sedona/sql/st_functions.py +++ b/python/sedona/sql/st_functions.py @@ -108,6 +108,7 @@ "ST_Z", "ST_ZMax", "ST_ZMin", + "ST_NumPoints" ] @@ -1231,3 +1232,12 @@ def ST_ZMin(geometry: ColumnOrName) -> Column: :rtype: Column """ return _call_st_function("ST_ZMin", geometry) + +def ST_NumPoints(geometry: ColumnOrName) -> Column: + """Return the number of points in a LineString + :param geometry: Geometry column to get number of points from. + :type geometry: ColumnOrName + :return: Number of points in a LineString as an integer column + :rtype: Column + """ + return _call_st_function("ST_NumPoints", geometry) diff --git a/python/tests/sql/test_dataframe_api.py b/python/tests/sql/test_dataframe_api.py index 6551d3bda9..1ece9f699f 100644 --- a/python/tests/sql/test_dataframe_api.py +++ b/python/tests/sql/test_dataframe_api.py @@ -135,6 +135,7 @@ (stf.ST_YMax, ("geom",), "triangle_geom", "", 1.0), (stf.ST_YMin, ("geom",), "triangle_geom", "", 0.0), (stf.ST_Z, ("b",), "two_points", "", 4.0), + (stf.ST_NumPoints, ("line",), "linestring_geom", "", 6), # predicates (stp.ST_Contains, ("geom", lambda: f.expr("ST_Point(0.5, 0.25)")), "triangle_geom", "", True), diff --git a/python/tests/sql/test_function.py b/python/tests/sql/test_function.py index 059fd30b6c..ba6657741f 100644 --- a/python/tests/sql/test_function.py +++ b/python/tests/sql/test_function.py @@ -1074,3 +1074,8 @@ def test_st_s2_cell_ids(self): # test null case cell_ids = self.spark.sql("select ST_S2CellIDs(null, 6)").take(1)[0][0] assert cell_ids is None + + def test_st_numPoints(self): + actual = self.spark.sql("SELECT ST_NumPoints(ST_GeomFromText('LINESTRING(0 1, 1 0, 2 0)'))").take(1)[0][0] + expected = 3 + assert expected == actual diff --git a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala index d50af2ab4f..eede0a130b 100644 --- a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala +++ b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala @@ -147,6 +147,7 @@ object Catalog { function[ST_DistanceSpheroid](), function[ST_AreaSpheroid](), function[ST_LengthSpheroid](), + function[ST_NumPoints](), // Expression for rasters function[RS_NormalizedDifference](), function[RS_Mean](), diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala index 0a4c0dcb4e..2a6dfde3c6 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala @@ -981,3 +981,10 @@ case class ST_LengthSpheroid(inputExpressions: Seq[Expression]) } } +case class ST_NumPoints(inputExpressions: Seq[Expression]) + extends InferredUnaryExpression(Functions.numPoints) with FoldableExpression { + protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { + copy(inputExpressions = newChildren) + } +} + diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala index ad29b854ed..94e0874d0d 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala @@ -301,4 +301,8 @@ object st_functions extends DataFrameAPI { def ST_LengthSpheroid(a: Column): Column = wrapExpression[ST_LengthSpheroid](a) def ST_LengthSpheroid(a: String): Column = wrapExpression[ST_LengthSpheroid](a) + + def ST_NumPoints(geometry: Column): Column = wrapExpression[ST_NumPoints](geometry) + + def ST_NumPoints(geometry: String): Column = wrapExpression[ST_NumPoints](geometry) } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala index 787a7300cd..e3eaf8ff41 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala @@ -949,5 +949,13 @@ class dataFrameAPITestScala extends TestBaseScala { val expectedResult = 10018754.171394622 assertEquals(expectedResult, actualResult, 0.1) } + + it("Passed ST_NumPoints") { + val lineDf = sparkSession.sql("SELECT ST_GeomFromWKT('LINESTRING (0 1, 1 0, 2 0)') AS geom") + val df = lineDf.select(ST_NumPoints("geom")) + val actualResult = df.take(1)(0).getInt(0) + val expectedResult = 3 + assert(actualResult == expectedResult) + } } } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index 0e598e6b5f..1dca8364f1 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -1909,4 +1909,16 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample assertEquals(expected, actual, 0.1) } } + + it("Should pass ST_NumPoints") { + val geomTestCases = Map( + ("'LINESTRING (0 1, 1 0, 2 0)'") -> "3" + ) + for (((geom), expectedResult) <- geomTestCases) { + val df = sparkSession.sql(s"SELECT ST_NumPoints(ST_GeomFromWKT($geom)), " + s"$expectedResult") + val actual = df.take(1)(0).get(0).asInstanceOf[Int] + val expected = df.take(1)(0).get(1).asInstanceOf[java.math.BigDecimal].intValue() + assertEquals(expected, actual) + } + } } From fc23fc9c95052043f27c07347b36480dbfdc2edd Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Wed, 7 Jun 2023 23:27:53 -0700 Subject: [PATCH 11/41] Update available version to 1.4.1 in documentation --- docs/api/flink/Function.md | 2 +- docs/api/sql/Function.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index ffdeaac8c2..c942a14e48 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -1042,7 +1042,7 @@ Introduction: Returns number of points in a LineString Format: `ST_NumPoints(geom: geometry)` -Since: `v1.4.2` +Since: `v1.4.1` Spark SQL example: ```sql diff --git a/docs/api/sql/Function.md b/docs/api/sql/Function.md index 4a6fb9c623..fe394372d0 100644 --- a/docs/api/sql/Function.md +++ b/docs/api/sql/Function.md @@ -1673,7 +1673,7 @@ Introduction: Returns number of points in a LineString Format: `ST_NumPoints(geom: geometry)` -Since: `v1.4.2` +Since: `v1.4.1` Spark SQL example: ```sql From 816e4ed2218f9a54b64654d23189bf8810ab659d Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Thu, 8 Jun 2023 00:05:46 -0700 Subject: [PATCH 12/41] Fix failing scala test case --- .../test/scala/org/apache/sedona/sql/functionTestScala.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index 1dca8364f1..21e897a5ee 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -1917,7 +1917,7 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample for (((geom), expectedResult) <- geomTestCases) { val df = sparkSession.sql(s"SELECT ST_NumPoints(ST_GeomFromWKT($geom)), " + s"$expectedResult") val actual = df.take(1)(0).get(0).asInstanceOf[Int] - val expected = df.take(1)(0).get(1).asInstanceOf[java.math.BigDecimal].intValue() + val expected = df.take(1)(0).get(1).asInstanceOf[java.lang.Integer].intValue() assertEquals(expected, actual) } } From 471a87e5780d616aa0d38c245b11ccf5f2e86e72 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Thu, 8 Jun 2023 13:33:50 -0700 Subject: [PATCH 13/41] Updated documentation to include negative flow. Changed generic Exception to IllegalArgumentException in ST_NumPoints implementation and its corresponding test --- .../org/apache/sedona/common/Functions.java | 2 +- .../apache/sedona/common/FunctionsTest.java | 4 +- docs/api/flink/Function.md | 37 +++++++++++-------- docs/api/sql/Function.md | 34 ++++++++++------- 4 files changed, 45 insertions(+), 32 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index fa9d184300..7fa3d802ba 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -848,7 +848,7 @@ private static Coordinate[] extractCoordinates(Geometry geometry) { public static int numPoints(Geometry geometry) throws Exception { String geometryType = geometry.getGeometryType(); if (!(Geometry.TYPENAME_LINESTRING.equalsIgnoreCase(geometryType))) { - throw new Exception("Unsupported geometry type: " + geometryType); + throw new IllegalArgumentException("Unsupported geometry type: " + geometryType + ", only LineString geometry is supported."); } return geometry.getNumPoints(); } diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index bca6e61462..93d20b0702 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -577,8 +577,8 @@ public void numPoints() throws Exception{ @Test public void numPointsUnsupported() throws Exception { Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(0, 0, 0, 90, 0, 0)); - String expected = "Unsupported geometry type: " + "Polygon"; - Exception e = assertThrows(Exception.class, () -> Functions.numPoints(polygon)); + String expected = "Unsupported geometry type: " + "Polygon" + ", only LineString geometry is supported."; + Exception e = assertThrows(IllegalArgumentException.class, () -> Functions.numPoints(polygon)); assertEquals(expected, e.getMessage()); } } diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index c942a14e48..270b7e7094 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -695,6 +695,28 @@ SELECT ST_NumInteriorRings(ST_GeomFromText('POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), Output: `1` +## ST_NumPoints + +Introduction: Returns number of points in a LineString. + +!!!note + If any other geometry is provided as an argument, an IllegalArgumentException is thrown. + Example: + `SELECT ST_NumPoints(ST_GeomFromWKT('MULTIPOINT ((0 0), (1 1), (0 1), (2 2))'))` + + Output: `IllegalArgumentException: Unsupported geometry type: MultiPoint, only LineString geometry is supported.` + +Format: `ST_NumPoints(geom: geometry)` + +Since: `v1.4.1` + +Spark SQL example: +```sql +SELECT ST_NumPoints(ST_GeomFromText('LINESTRING(1 2, 1 3)')) +``` + +Output: `2` + ## ST_PointN Introduction: Return the Nth point in a single linestring or circular linestring in the geometry. Negative values are counted backwards from the end of the LineString, so that -1 is the last point. Returns NULL if there is no linestring in the geometry. @@ -1035,18 +1057,3 @@ SELECT ST_ZMin(ST_GeomFromText('LINESTRING(1 3 4, 5 6 7)')) ``` Output: `4.0` - -## ST_NumPoints - -Introduction: Returns number of points in a LineString - -Format: `ST_NumPoints(geom: geometry)` - -Since: `v1.4.1` - -Spark SQL example: -```sql -SELECT ST_NumPoints(ST_GeomFromText('LINESTRING(1 2, 1 3)')) -``` - -Output: `2` diff --git a/docs/api/sql/Function.md b/docs/api/sql/Function.md index fe394372d0..9935f0f069 100644 --- a/docs/api/sql/Function.md +++ b/docs/api/sql/Function.md @@ -1093,6 +1093,26 @@ SELECT ST_NumInteriorRings(ST_GeomFromText('POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), Output: `1` +## ST_NumPoints +Introduction: Returns number of points in a LineString + +!!!note + If any other geometry is provided as an argument, an IllegalArgumentException is thrown. + Example: + `SELECT ST_NumPoints(ST_GeomFromWKT('MULTIPOINT ((0 0), (1 1), (0 1), (2 2))'))` + + Output: `IllegalArgumentException: Unsupported geometry type: MultiPoint, only LineString geometry is supported.` +Format: `ST_NumPoints(geom: geometry)` + +Since: `v1.4.1` + +Spark SQL example: +```sql +SELECT ST_NumPoints(ST_GeomFromText('LINESTRING(0 1, 1 0, 2 0)')) +``` + +Output: `3` + ## ST_PointN Introduction: Return the Nth point in a single linestring or circular linestring in the geometry. Negative values are counted backwards from the end of the LineString, so that -1 is the last point. Returns NULL if there is no linestring in the geometry. @@ -1668,17 +1688,3 @@ SELECT ST_ZMin(ST_GeomFromText('LINESTRING(1 3 4, 5 6 7)')) Output: `4.0` -## ST_NumPoints -Introduction: Returns number of points in a LineString - -Format: `ST_NumPoints(geom: geometry)` - -Since: `v1.4.1` - -Spark SQL example: -```sql -SELECT ST_NumPoints(ST_GeomFromText('LINESTRING(0 1, 1 0, 2 0)')) -``` - -Output: `3` - From b33499ddbca96354d0b2aec6f92f2ef233c41fe7 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Thu, 8 Jun 2023 13:54:23 -0700 Subject: [PATCH 14/41] Removed Spark SQL from flink documentation --- docs/api/flink/Function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index 270b7e7094..0b79c9b9c1 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -710,7 +710,7 @@ Format: `ST_NumPoints(geom: geometry)` Since: `v1.4.1` -Spark SQL example: +Example: ```sql SELECT ST_NumPoints(ST_GeomFromText('LINESTRING(1 2, 1 3)')) ``` From 0d68337c7444bfa3173802b36d0cc299a491040c Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Fri, 9 Jun 2023 11:12:44 -0700 Subject: [PATCH 15/41] Add ST_Force3D to sedona --- .../org/apache/sedona/common/Functions.java | 8 ++++ .../apache/sedona/common/utils/GeomUtils.java | 19 +++++++++ .../apache/sedona/common/FunctionsTest.java | 41 +++++++++++++++++++ docs/api/flink/Function.md | 33 +++++++++++++++ docs/api/sql/Function.md | 31 ++++++++++++++ .../java/org/apache/sedona/flink/Catalog.java | 3 +- .../sedona/flink/expressions/Functions.java | 17 ++++++++ .../org/apache/sedona/flink/FunctionTest.java | 10 +++++ python/sedona/sql/st_functions.py | 14 ++++++- python/tests/sql/test_dataframe_api.py | 3 +- python/tests/sql/test_function.py | 7 ++++ .../org/apache/sedona/sql/UDF/Catalog.scala | 1 + .../sedona_sql/expressions/Functions.scala | 8 ++++ .../sedona_sql/expressions/st_functions.scala | 8 ++++ .../sedona/sql/dataFrameAPITestScala.scala | 8 ++++ .../apache/sedona/sql/functionTestScala.scala | 13 ++++++ 16 files changed, 221 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index 7fa3d802ba..2f0dacd287 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -853,6 +853,14 @@ public static int numPoints(Geometry geometry) throws Exception { return geometry.getNumPoints(); } + public static Geometry force3D(Geometry geometry, double zValue) { + return GeomUtils.getGeom3d(geometry, zValue, false); + } + + public static Geometry force3D(Geometry geometry) { + return GeomUtils.getGeom3d(geometry, -1, true); + } + public static Geometry geometricMedian(Geometry geometry, double tolerance, int maxIter, boolean failIfNotConverged) throws Exception { String geometryType = geometry.getGeometryType(); if(!(Geometry.TYPENAME_POINT.equals(geometryType) || Geometry.TYPENAME_MULTIPOINT.equals(geometryType))) { diff --git a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java index 635c8cd4b2..239f0de109 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java @@ -419,4 +419,23 @@ public static Geometry[] getSubGeometries(Geometry geom) { } return geometries; } + + + public static Geometry getGeom3d(Geometry geometry, double zValue, boolean addDefaultZValue) { + Coordinate[] coordinates = geometry.getCoordinates(); +// if(points.length == 0) +// return points; + boolean is3d = !Double.isNaN(coordinates[0].z); + for(int i = 0; i < coordinates.length; i++) { + if(!is3d) { + if (addDefaultZValue) { + coordinates[i].z = 0.0; + }else { + coordinates[i].z = zValue; + } + } + } + geometry.geometryChanged(); + return geometry; + } } diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index 93d20b0702..a9cca6c3b7 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -57,6 +57,14 @@ private Coordinate[] coordArray(double... coordValues) { return coords; } + private Coordinate[] coordArray3d(double... coordValues) { + Coordinate[] coords = new Coordinate[(int)(coordValues.length / 3)]; + for (int i = 0; i < coordValues.length; i += 3) { + coords[(int)(i / 3)] = new Coordinate(coordValues[i], coordValues[i+1], coordValues[i+2]); + } + return coords; + } + @Test public void splitLineStringByMultipoint() { LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray(0.0, 0.0, 1.5, 1.5, 2.0, 2.0)); @@ -581,4 +589,37 @@ public void numPointsUnsupported() throws Exception { Exception e = assertThrows(IllegalArgumentException.class, () -> Functions.numPoints(polygon)); assertEquals(expected, e.getMessage()); } + + @Test + public void force3DObject2D() { + int expectedDims = 3; + LineString line = GEOMETRY_FACTORY.createLineString(coordArray(0, 1, 1, 0, 2, 0)); + Geometry forcedLine = Functions.force3D(line, 1.0); + assertEquals(expectedDims, Functions.nDims(forcedLine)); + } + + @Test + public void force3DObject2DDefaultValue() { + int expectedDims = 3; + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(0, 0, 0, 90, 0, 0)); + Geometry forcedPolygon = Functions.force3D(polygon); + assertEquals(expectedDims, Functions.nDims(forcedPolygon)); + } + + @Test + public void force3DObject3D() { + int expectedDims = 3; + LineString line3D = GEOMETRY_FACTORY.createLineString(coordArray3d(0, 1, 1, 1, 2, 1, 1, 2, 2)); + Geometry forcedLine3D = Functions.force3D(line3D, 1.0); + assertEquals(expectedDims, Functions.nDims(forcedLine3D)); + } + + @Test + public void force3DObject3DDefaultValue() { + int expectedDims = 3; + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray3d(0, 0, 0, 90, 0, 0, 0, 0, 0)); + Geometry forcedPolygon = Functions.force3D(polygon); + assertEquals(expectedDims, Functions.nDims(forcedPolygon)); + } + } diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index 0b79c9b9c1..e7f964e644 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -390,6 +390,39 @@ Input: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` Output: `POLYGON((0 0,0 5,5 0,0 0),(1 1,3 1,1 3,1 1))` +## ST_Force3D +Introduction: Forces the geometry into a 3-dimensional model so that all output representations will have X, Y and Z coordinates. +An optionally given zValue is tacked onto the geometry if the geometry is 2-dimensional. Default value of zValue is 0.0 +If the given geometry is 3-dimensional, no change is done to it. + +Example: + +```sql +SELECT ST_Force3D(df.geometry) AS geom +from df +``` + +Input: `LINESTRING(0 1, 1 2, 2 1)` + +Output: `LINESTRING(0 1 0, 1 2 0, 2 1 0)` + +Input: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` + +Output: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` + +```sql +SELECT ST_Force3D(df.geometry, 2.3) AS geom +from df +``` + +Input: `LINESTRING(0 1, 1 2, 2 1)` + +Output: `LINESTRING(0 1 2.3, 1 2 2.3, 2 1 2.3)` + +Input: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` + +Output: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` + ## ST_GeoHash Introduction: Returns GeoHash of the geometry with given precision diff --git a/docs/api/sql/Function.md b/docs/api/sql/Function.md index 9935f0f069..050051eeef 100644 --- a/docs/api/sql/Function.md +++ b/docs/api/sql/Function.md @@ -576,6 +576,37 @@ Result: +---------------------------------------------------------------+ ``` +## ST_Force3D +Introduction: Forces the geometry into a 3-dimensional model so that all output representations will have X, Y and Z coordinates. +An optionally given zValue is tacked onto the geometry if the geometry is 2-dimensional. Default value of zValue is 0.0 +If the given geometry is 3-dimensional, no change is done to it. + +Spark SQL Example: + +```sql +SELECT ST_Force3D(geometry) AS geom +``` + +Input: `LINESTRING(0 1, 1 2, 2 1)` + +Output: `LINESTRING(0 1 0, 1 2 0, 2 1 0)` + +Input: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` + +Output: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` + +```sql +SELECT ST_Force3D(geometry, 2.3) AS geom +``` + +Input: `LINESTRING(0 1, 1 2, 2 1)` + +Output: `LINESTRING(0 1 2.3, 1 2 2.3, 2 1 2.3)` + +Input: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` + +Output: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` + ## ST_GeoHash Introduction: Returns GeoHash of the geometry with given precision diff --git a/flink/src/main/java/org/apache/sedona/flink/Catalog.java b/flink/src/main/java/org/apache/sedona/flink/Catalog.java index 884126c00e..f72f0793ec 100644 --- a/flink/src/main/java/org/apache/sedona/flink/Catalog.java +++ b/flink/src/main/java/org/apache/sedona/flink/Catalog.java @@ -95,7 +95,8 @@ public static UserDefinedFunction[] getFuncs() { new Functions.ST_Split(), new Functions.ST_S2CellIDs(), new Functions.ST_GeometricMedian(), - new Functions.ST_NumPoints() + new Functions.ST_NumPoints(), + new Functions.ST_Force3D() }; } diff --git a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java index 7001345a4c..33f1b6c0f4 100644 --- a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java +++ b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java @@ -13,6 +13,7 @@ */ package org.apache.sedona.flink.expressions; +import org.apache.calcite.runtime.Geometries; import org.apache.flink.table.annotation.DataTypeHint; import org.apache.flink.table.functions.ScalarFunction; import org.locationtech.jts.geom.Geometry; @@ -582,4 +583,20 @@ public int eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.ge } } + public static class ST_Force3D extends ScalarFunction { + + @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) + public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o, + @DataTypeHint("Double") Double zValue) { + Geometry geometry = (Geometry) o; + return org.apache.sedona.common.Functions.force3D(geometry, zValue); + } + + @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) + public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o) { + Geometry geometry = (Geometry) o; + return org.apache.sedona.common.Functions.force3D(geometry); + } + } + } diff --git a/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java b/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java index eac04d2fde..190a678550 100644 --- a/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java +++ b/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java @@ -699,4 +699,14 @@ public void testNumPoints() { assertEquals(expected, actual); } + @Test + public void testForce3D() { + Integer expectedDims = 3; + Table pointTable = tableEnv.sqlQuery("SELECT ST_Force3D(ST_GeomFromWKT('LINESTRING(0 1, 1 0, 2 0)')) " + + "AS " + polygonColNames[0]); + pointTable = pointTable.select(call(Functions.ST_NDims.class.getSimpleName(), $(polygonColNames[0]))); + Integer actual = (Integer) first(pointTable).getField(0); + assertEquals(expectedDims, actual); + } + } diff --git a/python/sedona/sql/st_functions.py b/python/sedona/sql/st_functions.py index d5c7602b1d..66cf149ad2 100644 --- a/python/sedona/sql/st_functions.py +++ b/python/sedona/sql/st_functions.py @@ -108,7 +108,8 @@ "ST_Z", "ST_ZMax", "ST_ZMin", - "ST_NumPoints" + "ST_NumPoints", + "ST_Force3D" ] @@ -1241,3 +1242,14 @@ def ST_NumPoints(geometry: ColumnOrName) -> Column: :rtype: Column """ return _call_st_function("ST_NumPoints", geometry) + + +def ST_Force3D(geometry: ColumnOrName, zValue: Optional[Union[ColumnOrName, float]] = 0.0) -> Column: + """ + Return a geometry with a 3D coordinate of value 'zValue' forced upon it. No change happens if the geometry is already 3D + :param zValue: Optional value of z coordinate to be potentially added, default value is 0.0 + :param geometry: Geometry column to make 3D + :return: 3D geometry with either already present z coordinate if any, or zcoordinate with given zValue + """ + args = (geometry, zValue) + return _call_st_function("ST_Force3D", args) diff --git a/python/tests/sql/test_dataframe_api.py b/python/tests/sql/test_dataframe_api.py index 1ece9f699f..c3f2bdf414 100644 --- a/python/tests/sql/test_dataframe_api.py +++ b/python/tests/sql/test_dataframe_api.py @@ -85,6 +85,7 @@ (stf.ST_ExteriorRing, ("geom",), "triangle_geom", "", "LINESTRING (0 0, 1 0, 1 1, 0 0)"), (stf.ST_FlipCoordinates, ("point",), "point_geom", "", "POINT (1 0)"), (stf.ST_Force_2D, ("point",), "point_geom", "", "POINT (0 1)"), + (stf.ST_Force3D, ("point", 1), "point_geom", "", "POINT (0 1 1)"), (stf.ST_GeometricMedian, ("multipoint",), "multipoint_geom", "", "POINT (22.500002656424286 21.250001168173426)"), (stf.ST_GeometryN, ("geom", 0), "multipoint", "", "POINT (0 0)"), (stf.ST_GeometryType, ("point",), "point_geom", "", "ST_Point"), @@ -111,6 +112,7 @@ (stf.ST_NPoints, ("line",), "linestring_geom", "", 6), (stf.ST_NumGeometries, ("geom",), "multipoint", "", 2), (stf.ST_NumInteriorRings, ("geom",), "geom_with_hole", "", 1), + (stf.ST_NumPoints, ("line",), "linestring_geom", "", 6), (stf.ST_PointN, ("line", 2), "linestring_geom", "", "POINT (1 0)"), (stf.ST_PointOnSurface, ("line",), "linestring_geom", "", "POINT (2 0)"), (stf.ST_PrecisionReduce, ("geom", 1), "precision_reduce_point", "", "POINT (0.1 0.2)"), @@ -135,7 +137,6 @@ (stf.ST_YMax, ("geom",), "triangle_geom", "", 1.0), (stf.ST_YMin, ("geom",), "triangle_geom", "", 0.0), (stf.ST_Z, ("b",), "two_points", "", 4.0), - (stf.ST_NumPoints, ("line",), "linestring_geom", "", 6), # predicates (stp.ST_Contains, ("geom", lambda: f.expr("ST_Point(0.5, 0.25)")), "triangle_geom", "", True), diff --git a/python/tests/sql/test_function.py b/python/tests/sql/test_function.py index ba6657741f..18b38c1182 100644 --- a/python/tests/sql/test_function.py +++ b/python/tests/sql/test_function.py @@ -1079,3 +1079,10 @@ def test_st_numPoints(self): actual = self.spark.sql("SELECT ST_NumPoints(ST_GeomFromText('LINESTRING(0 1, 1 0, 2 0)'))").take(1)[0][0] expected = 3 assert expected == actual + + def test_force3D(self): + expected = 3 + actualDf = self.spark.sql("SELECT ST_Force3D(ST_GeomFromText('LINESTRING(0 1, 1 0, 2 0)'), 1.1) AS geom") + actual = actualDf.selectExpr("ST_NDims(geom)").take(1)[0][0] + assert expected == actual + diff --git a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala index eede0a130b..495aace49e 100644 --- a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala +++ b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala @@ -148,6 +148,7 @@ object Catalog { function[ST_AreaSpheroid](), function[ST_LengthSpheroid](), function[ST_NumPoints](), + function[ST_Force3D](), // Expression for rasters function[RS_NormalizedDifference](), function[RS_Mean](), diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala index 2a6dfde3c6..89f42be77f 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala @@ -988,3 +988,11 @@ case class ST_NumPoints(inputExpressions: Seq[Expression]) } } +case class ST_Force3D(inputExpressions: Seq[Expression]) + extends InferredBinaryExpression(Functions.force3D) with FoldableExpression { + + override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { + copy(inputExpressions = newChildren) + } +} + diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala index 94e0874d0d..6101222b53 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala @@ -305,4 +305,12 @@ object st_functions extends DataFrameAPI { def ST_NumPoints(geometry: Column): Column = wrapExpression[ST_NumPoints](geometry) def ST_NumPoints(geometry: String): Column = wrapExpression[ST_NumPoints](geometry) + + def ST_Force3D(geometry: Column): Column = wrapExpression[ST_Force3D](geometry, 0.0) + + def ST_Force3D(geometry: String): Column = wrapExpression[ST_Force3D](geometry, 0.0) + + def ST_Force3D(geometry: Column, zValue: Column): Column = wrapExpression[ST_Force3D](geometry, zValue) + + def ST_Force3D(geometry: String, zValue: Double): Column = wrapExpression[ST_Force3D](geometry, zValue) } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala index e3eaf8ff41..1171ce6a61 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala @@ -957,5 +957,13 @@ class dataFrameAPITestScala extends TestBaseScala { val expectedResult = 3 assert(actualResult == expectedResult) } + + it("Passed ST_Force3D") { + val lineDf = sparkSession.sql("SELECT ST_Force3D(ST_GeomFromWKT('LINESTRING (0 1, 1 0, 2 0)'), 2.3) AS geom") + val df = lineDf.select(ST_NDims("geom")) + val actual = df.take(1)(0).getInt(0) + val expected = 3 + assert(expected == actual) + } } } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index 21e897a5ee..a3e733d6d1 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -1921,4 +1921,17 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample assertEquals(expected, actual) } } + + it("should pass ST_Force3D") { + val geomTestCases = Map( + ("'LINESTRING (0 1, 1 0, 2 0)'") -> "'LINESTRING (0 1 1, 1 0 1, 2 0 1)'", + ("'LINESTRING (0 1 3, 1 0 3, 2 0 3)'") -> "'LINESTRING (0 1 3, 1 0 3, 2 0 3)'" + ) + for (((geom), expectedResult) <- geomTestCases) { + val df = sparkSession.sql(s"SELECT ST_AsText(ST_Force3D(ST_GeomFromWKT($geom), 1)), " + s"$expectedResult") + val actual = df.take(1)(0).get(1).asInstanceOf[String] + val expected = df.take(1)(0).get(1).asInstanceOf[java.lang.String].toString() + assertEquals(expected, actual) + } + } } From da0314a1d3d4db889b1deaad46a196fadd9faf32 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Fri, 9 Jun 2023 15:37:45 -0700 Subject: [PATCH 16/41] Updated force3D logic to handle empty geometries Refactored function name Made java tests more comprehensive by checking both nDims and WKT of returned geometry Added more test cases in scala test cases Updated documentation with empty geometry case and more examples --- .../org/apache/sedona/common/Functions.java | 4 +-- .../apache/sedona/common/utils/GeomUtils.java | 10 +++---- .../apache/sedona/common/FunctionsTest.java | 30 +++++++++++++++++-- docs/api/flink/Function.md | 11 ++++++- docs/api/sql/Function.md | 11 ++++++- .../apache/sedona/sql/functionTestScala.scala | 11 +++---- 6 files changed, 61 insertions(+), 16 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index 2f0dacd287..072fa40b84 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -854,11 +854,11 @@ public static int numPoints(Geometry geometry) throws Exception { } public static Geometry force3D(Geometry geometry, double zValue) { - return GeomUtils.getGeom3d(geometry, zValue, false); + return GeomUtils.get3DGeom(geometry, zValue, false); } public static Geometry force3D(Geometry geometry) { - return GeomUtils.getGeom3d(geometry, -1, true); + return GeomUtils.get3DGeom(geometry, -1, true); } public static Geometry geometricMedian(Geometry geometry, double tolerance, int maxIter, boolean failIfNotConverged) throws Exception { diff --git a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java index 239f0de109..3f19783e45 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java @@ -421,17 +421,17 @@ public static Geometry[] getSubGeometries(Geometry geom) { } - public static Geometry getGeom3d(Geometry geometry, double zValue, boolean addDefaultZValue) { + public static Geometry get3DGeom(Geometry geometry, double zValue, boolean addDefaultZValue) { Coordinate[] coordinates = geometry.getCoordinates(); -// if(points.length == 0) -// return points; + if (coordinates.length == 0) return geometry; + boolean is3d = !Double.isNaN(coordinates[0].z); for(int i = 0; i < coordinates.length; i++) { if(!is3d) { if (addDefaultZValue) { - coordinates[i].z = 0.0; + coordinates[i].setZ(0.0); }else { - coordinates[i].z = zValue; + coordinates[i].setZ(zValue); } } } diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index a9cca6c3b7..f54f2711cd 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -17,10 +17,12 @@ import com.google.common.math.DoubleMath; import org.apache.sedona.common.sphere.Haversine; import org.apache.sedona.common.sphere.Spheroid; +import org.apache.sedona.common.utils.GeomUtils; import org.apache.sedona.common.utils.S2Utils; import org.junit.Test; import org.locationtech.jts.geom.*; import org.locationtech.jts.io.WKTReader; +import org.locationtech.jts.io.WKTWriter; import java.util.Arrays; import java.util.HashSet; @@ -594,7 +596,10 @@ public void numPointsUnsupported() throws Exception { public void force3DObject2D() { int expectedDims = 3; LineString line = GEOMETRY_FACTORY.createLineString(coordArray(0, 1, 1, 0, 2, 0)); - Geometry forcedLine = Functions.force3D(line, 1.0); + LineString expectedLine = GEOMETRY_FACTORY.createLineString(coordArray3d(0, 1, 1.1, 1, 0, 1.1, 2, 0, 1.1)); + Geometry forcedLine = Functions.force3D(line, 1.1); + WKTWriter wktWriter = new WKTWriter(GeomUtils.getDimension(expectedLine)); + assertEquals(wktWriter.write(expectedLine), wktWriter.write(forcedLine)); assertEquals(expectedDims, Functions.nDims(forcedLine)); } @@ -602,7 +607,10 @@ public void force3DObject2D() { public void force3DObject2DDefaultValue() { int expectedDims = 3; Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(0, 0, 0, 90, 0, 0)); + Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray3d(0, 0, 0, 0, 90, 0, 0, 0, 0)); Geometry forcedPolygon = Functions.force3D(polygon); + WKTWriter wktWriter = new WKTWriter(GeomUtils.getDimension(expectedPolygon)); + assertEquals(wktWriter.write(expectedPolygon), wktWriter.write(forcedPolygon)); assertEquals(expectedDims, Functions.nDims(forcedPolygon)); } @@ -610,7 +618,9 @@ public void force3DObject2DDefaultValue() { public void force3DObject3D() { int expectedDims = 3; LineString line3D = GEOMETRY_FACTORY.createLineString(coordArray3d(0, 1, 1, 1, 2, 1, 1, 2, 2)); - Geometry forcedLine3D = Functions.force3D(line3D, 1.0); + Geometry forcedLine3D = Functions.force3D(line3D, 2.0); + WKTWriter wktWriter = new WKTWriter(GeomUtils.getDimension(line3D)); + assertEquals(wktWriter.write(line3D), wktWriter.write(forcedLine3D)); assertEquals(expectedDims, Functions.nDims(forcedLine3D)); } @@ -619,7 +629,23 @@ public void force3DObject3DDefaultValue() { int expectedDims = 3; Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray3d(0, 0, 0, 90, 0, 0, 0, 0, 0)); Geometry forcedPolygon = Functions.force3D(polygon); + WKTWriter wktWriter = new WKTWriter(GeomUtils.getDimension(polygon)); + assertEquals(wktWriter.write(polygon), wktWriter.write(forcedPolygon)); assertEquals(expectedDims, Functions.nDims(forcedPolygon)); } + @Test + public void force3DEmptyObject() { + LineString emptyLine = GEOMETRY_FACTORY.createLineString(); + Geometry forcedEmptyLine = Functions.force3D(emptyLine, 1.2); + assertEquals(emptyLine.isEmpty(), forcedEmptyLine.isEmpty()); + } + + @Test + public void force3DEmptyObjectDefaultValue() { + LineString emptyLine = GEOMETRY_FACTORY.createLineString(); + Geometry forcedEmptyLine = Functions.force3D(emptyLine); + assertEquals(emptyLine.isEmpty(), forcedEmptyLine.isEmpty()); + } + } diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index e7f964e644..28335bc309 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -393,7 +393,12 @@ Output: `POLYGON((0 0,0 5,5 0,0 0),(1 1,3 1,1 3,1 1))` ## ST_Force3D Introduction: Forces the geometry into a 3-dimensional model so that all output representations will have X, Y and Z coordinates. An optionally given zValue is tacked onto the geometry if the geometry is 2-dimensional. Default value of zValue is 0.0 -If the given geometry is 3-dimensional, no change is done to it. +If the given geometry is 3-dimensional, no change is performed on it. +If the given geometry is empty, no change is performed on it. + +Format: `ST_Force3D(geometry, zValue)` + +Since: `1.4.1` Example: @@ -423,6 +428,10 @@ Input: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` Output: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` +Input: `LINESTRING EMPTY` + +Output: `LINESTRING EMPTY` + ## ST_GeoHash Introduction: Returns GeoHash of the geometry with given precision diff --git a/docs/api/sql/Function.md b/docs/api/sql/Function.md index 050051eeef..9b3e390a34 100644 --- a/docs/api/sql/Function.md +++ b/docs/api/sql/Function.md @@ -579,7 +579,12 @@ Result: ## ST_Force3D Introduction: Forces the geometry into a 3-dimensional model so that all output representations will have X, Y and Z coordinates. An optionally given zValue is tacked onto the geometry if the geometry is 2-dimensional. Default value of zValue is 0.0 -If the given geometry is 3-dimensional, no change is done to it. +If the given geometry is 3-dimensional, no change is performed on it. +If the given geometry is empty, no change is performed on it. + +Format: `ST_Force3D(geometry, zValue)` + +Since: `1.4.1` Spark SQL Example: @@ -607,6 +612,10 @@ Input: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` Output: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` +Input: `LINESTRING EMPTY` + +Output: `LINESTRING EMPTY` + ## ST_GeoHash Introduction: Returns GeoHash of the geometry with given precision diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index a3e733d6d1..735ec65e24 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -1924,13 +1924,14 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample it("should pass ST_Force3D") { val geomTestCases = Map( - ("'LINESTRING (0 1, 1 0, 2 0)'") -> "'LINESTRING (0 1 1, 1 0 1, 2 0 1)'", - ("'LINESTRING (0 1 3, 1 0 3, 2 0 3)'") -> "'LINESTRING (0 1 3, 1 0 3, 2 0 3)'" + ("'LINESTRING (0 1, 1 0, 2 0)'") -> "'LINESTRING Z(0 1 1, 1 0 1, 2 0 1)'", + ("'LINESTRING Z(0 1 3, 1 0 3, 2 0 3)'") -> "'LINESTRING Z(0 1 3, 1 0 3, 2 0 3)'", + ("'LINESTRING EMPTY'") -> "'LINESTRING EMPTY'" ) for (((geom), expectedResult) <- geomTestCases) { - val df = sparkSession.sql(s"SELECT ST_AsText(ST_Force3D(ST_GeomFromWKT($geom), 1)), " + s"$expectedResult") - val actual = df.take(1)(0).get(1).asInstanceOf[String] - val expected = df.take(1)(0).get(1).asInstanceOf[java.lang.String].toString() + val df = sparkSession.sql(s"SELECT ST_AsText(ST_Force3D(ST_GeomFromWKT($geom), 1)) AS geom, " + s"$expectedResult") + val actual = df.take(1)(0).get(0).asInstanceOf[String] + val expected = df.take(1)(0).get(1).asInstanceOf[java.lang.String] assertEquals(expected, actual) } } From 020e9b985e9b7154b3116273895ce302465fdfd4 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Sat, 10 Jun 2023 09:37:23 -0700 Subject: [PATCH 17/41] Updated force3D dataframe test --- python/tests/sql/test_dataframe_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/sql/test_dataframe_api.py b/python/tests/sql/test_dataframe_api.py index 316a98a76b..f9fbfc82fc 100644 --- a/python/tests/sql/test_dataframe_api.py +++ b/python/tests/sql/test_dataframe_api.py @@ -85,7 +85,7 @@ (stf.ST_ExteriorRing, ("geom",), "triangle_geom", "", "LINESTRING (0 0, 1 0, 1 1, 0 0)"), (stf.ST_FlipCoordinates, ("point",), "point_geom", "", "POINT (1 0)"), (stf.ST_Force_2D, ("point",), "point_geom", "", "POINT (0 1)"), - (stf.ST_Force3D, ("point", 1), "point_geom", "", "POINT (0 1 1)"), + (stf.ST_Force3D, ("point", 1), "point_geom", "", "POINT Z (0 1 1)"), (stf.ST_GeometricMedian, ("multipoint",), "multipoint_geom", "", "POINT (22.500002656424286 21.250001168173426)"), (stf.ST_GeometryN, ("geom", 0), "multipoint", "", "POINT (0 0)"), (stf.ST_GeometryType, ("point",), "point_geom", "", "ST_Point"), From 86de0547ff36333f36b406848dae3ab34e37a913 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Sat, 10 Jun 2023 13:49:29 -0700 Subject: [PATCH 18/41] fix error in test case --- .../org/apache/spark/sql/sedona_sql/expressions/Functions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala index 89f42be77f..e70f14735a 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala @@ -991,7 +991,7 @@ case class ST_NumPoints(inputExpressions: Seq[Expression]) case class ST_Force3D(inputExpressions: Seq[Expression]) extends InferredBinaryExpression(Functions.force3D) with FoldableExpression { - override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { + protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { copy(inputExpressions = newChildren) } } From 307c060ea93752e85860cdd797d755c7792a6f51 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Sun, 11 Jun 2023 11:11:57 -0700 Subject: [PATCH 19/41] Updated documentation for Force3D to include Z format WKT in the output. Added a note explaining the same --- docs/api/flink/Function.md | 11 +++++++---- docs/api/sql/Function.md | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index 28335bc309..11d86ff6e1 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -396,6 +396,9 @@ An optionally given zValue is tacked onto the geometry if the geometry is 2-dime If the given geometry is 3-dimensional, no change is performed on it. If the given geometry is empty, no change is performed on it. +!!!Note + Example output is after calling ST_AsText() on returned geometry, which adds Z for in the WKT for 3D geometries + Format: `ST_Force3D(geometry, zValue)` Since: `1.4.1` @@ -409,11 +412,11 @@ from df Input: `LINESTRING(0 1, 1 2, 2 1)` -Output: `LINESTRING(0 1 0, 1 2 0, 2 1 0)` +Output: `LINESTRING Z(0 1 0, 1 2 0, 2 1 0)` Input: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` -Output: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` +Output: `POLYGON Z((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` ```sql SELECT ST_Force3D(df.geometry, 2.3) AS geom @@ -422,11 +425,11 @@ from df Input: `LINESTRING(0 1, 1 2, 2 1)` -Output: `LINESTRING(0 1 2.3, 1 2 2.3, 2 1 2.3)` +Output: `LINESTRING Z(0 1 2.3, 1 2 2.3, 2 1 2.3)` Input: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` -Output: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` +Output: `POLYGON Z((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` Input: `LINESTRING EMPTY` diff --git a/docs/api/sql/Function.md b/docs/api/sql/Function.md index 9b3e390a34..05265b5381 100644 --- a/docs/api/sql/Function.md +++ b/docs/api/sql/Function.md @@ -582,6 +582,9 @@ An optionally given zValue is tacked onto the geometry if the geometry is 2-dime If the given geometry is 3-dimensional, no change is performed on it. If the given geometry is empty, no change is performed on it. +!!!Note + Example output is after calling ST_AsText() on returned geometry, which adds Z for in the WKT for 3D geometries + Format: `ST_Force3D(geometry, zValue)` Since: `1.4.1` @@ -594,11 +597,11 @@ SELECT ST_Force3D(geometry) AS geom Input: `LINESTRING(0 1, 1 2, 2 1)` -Output: `LINESTRING(0 1 0, 1 2 0, 2 1 0)` +Output: `LINESTRING Z(0 1 0, 1 2 0, 2 1 0)` Input: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` -Output: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` +Output: `POLYGON Z((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` ```sql SELECT ST_Force3D(geometry, 2.3) AS geom @@ -606,11 +609,11 @@ SELECT ST_Force3D(geometry, 2.3) AS geom Input: `LINESTRING(0 1, 1 2, 2 1)` -Output: `LINESTRING(0 1 2.3, 1 2 2.3, 2 1 2.3)` +Output: `LINESTRING Z(0 1 2.3, 1 2 2.3, 2 1 2.3)` Input: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` -Output: `POLYGON((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` +Output: `POLYGON Z((0 0 2,0 5 2,5 0 2,0 0 2),(1 1 2,3 1 2,1 3 2,1 1 2))` Input: `LINESTRING EMPTY` From dbab04c3a7387086132afd50a7d9f19aec5b740c Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Sun, 11 Jun 2023 11:12:37 -0700 Subject: [PATCH 20/41] Added default zValue test case in sedona flink --- .../java/org/apache/sedona/flink/FunctionTest.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java b/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java index 190a678550..219970c543 100644 --- a/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java +++ b/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java @@ -702,11 +702,21 @@ public void testNumPoints() { @Test public void testForce3D() { Integer expectedDims = 3; - Table pointTable = tableEnv.sqlQuery("SELECT ST_Force3D(ST_GeomFromWKT('LINESTRING(0 1, 1 0, 2 0)')) " + + Table pointTable = tableEnv.sqlQuery("SELECT ST_Force3D(ST_GeomFromWKT('LINESTRING(0 1, 1 0, 2 0)'), 1.2) " + "AS " + polygonColNames[0]); pointTable = pointTable.select(call(Functions.ST_NDims.class.getSimpleName(), $(polygonColNames[0]))); Integer actual = (Integer) first(pointTable).getField(0); assertEquals(expectedDims, actual); } + @Test + public void testForce3DDefaultValue() { + Integer expectedDims = 3; + Table pointTable = tableEnv.sqlQuery("SELECT ST_Force3D(ST_GeomFromWKT('LINESTRING(0 1, 1 0, 2 0)')) " + + "AS " + polygonColNames[0]); + pointTable = pointTable.select(call(Functions.ST_NDims.class.getSimpleName(), $(polygonColNames[0]))); + Integer actual = (Integer) first(pointTable).getField(0); + assertEquals(expectedDims, actual); + } + } From c0eb8bcc03bd6222e0aba54fdb483486f57528df Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Sun, 11 Jun 2023 11:12:56 -0700 Subject: [PATCH 21/41] Added default zValue dataframe test case --- .../scala/org/apache/sedona/sql/dataFrameAPITestScala.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala index 1171ce6a61..3e8401e774 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala @@ -964,6 +964,11 @@ class dataFrameAPITestScala extends TestBaseScala { val actual = df.take(1)(0).getInt(0) val expected = 3 assert(expected == actual) + + val lineDfDefaultValue = sparkSession.sql("SELECT ST_Force3D(ST_GeomFromWKT('LINESTRING (0 1, 1 0, 2 0)')) AS geom") + val dfDefaultValue = lineDfDefaultValue.select(ST_NDims("geom")) + val actualDefaultValue = dfDefaultValue.take(1)(0).getInt(0) + assert(expected == actualDefaultValue) } } } From 479b264f399dfffbb600ebf84f56f360d24ded69 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Sun, 11 Jun 2023 11:13:08 -0700 Subject: [PATCH 22/41] Added default zValue scala test case --- .../org/apache/sedona/sql/functionTestScala.scala | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index 735ec65e24..b7b1abd884 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -21,6 +21,7 @@ package org.apache.sedona.sql import org.apache.commons.codec.binary.Hex import org.apache.sedona.sql.implicits._ +import org.apache.spark.sql.catalyst.expressions.{GenericRow, GenericRowWithSchema} import org.apache.spark.sql.functions._ import org.apache.spark.sql.{DataFrame, Row} import org.geotools.referencing.CRS @@ -1924,15 +1925,19 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample it("should pass ST_Force3D") { val geomTestCases = Map( - ("'LINESTRING (0 1, 1 0, 2 0)'") -> "'LINESTRING Z(0 1 1, 1 0 1, 2 0 1)'", - ("'LINESTRING Z(0 1 3, 1 0 3, 2 0 3)'") -> "'LINESTRING Z(0 1 3, 1 0 3, 2 0 3)'", - ("'LINESTRING EMPTY'") -> "'LINESTRING EMPTY'" + ("'LINESTRING (0 1, 1 0, 2 0)'") -> ("'LINESTRING Z(0 1 1, 1 0 1, 2 0 1)'", "'LINESTRING Z(0 1 0, 1 0 0, 2 0 0)'"), + ("'LINESTRING Z(0 1 3, 1 0 3, 2 0 3)'") -> ("'LINESTRING Z(0 1 3, 1 0 3, 2 0 3)'", "'LINESTRING Z(0 1 3, 1 0 3, 2 0 3)'"), + ("'LINESTRING EMPTY'") -> ("'LINESTRING EMPTY'", "'LINESTRING EMPTY'") ) for (((geom), expectedResult) <- geomTestCases) { val df = sparkSession.sql(s"SELECT ST_AsText(ST_Force3D(ST_GeomFromWKT($geom), 1)) AS geom, " + s"$expectedResult") + val dfDefaultValue = sparkSession.sql(s"SELECT ST_AsText(ST_Force3D(ST_GeomFromWKT($geom), 0.0)) AS geom, " + s"$expectedResult") val actual = df.take(1)(0).get(0).asInstanceOf[String] - val expected = df.take(1)(0).get(1).asInstanceOf[java.lang.String] + val expected = df.take(1)(0).get(1).asInstanceOf[GenericRowWithSchema].get(0).asInstanceOf[String] + val actualDefaultValue = dfDefaultValue.take(1)(0).get(0).asInstanceOf[String] + val expectedDefaultValue = dfDefaultValue.take(1)(0).get(1).asInstanceOf[GenericRowWithSchema].get(1).asInstanceOf[String] assertEquals(expected, actual) + assertEquals(expectedDefaultValue, actualDefaultValue); } } } From 641d2d92136a4238b36c01e995dbe3339768e0eb Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Sun, 11 Jun 2023 12:31:03 -0700 Subject: [PATCH 23/41] fix dataframe testcase --- .../scala/org/apache/sedona/sql/dataFrameAPITestScala.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala index 3e8401e774..c4311dadd9 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala @@ -963,12 +963,12 @@ class dataFrameAPITestScala extends TestBaseScala { val df = lineDf.select(ST_NDims("geom")) val actual = df.take(1)(0).getInt(0) val expected = 3 - assert(expected == actual) - val lineDfDefaultValue = sparkSession.sql("SELECT ST_Force3D(ST_GeomFromWKT('LINESTRING (0 1, 1 0, 2 0)')) AS geom") + val lineDfDefaultValue = sparkSession.sql("SELECT ST_Force3D(ST_GeomFromWKT('LINESTRING (0 1, 1 0, 2 0)'), 0.0) AS geom") val dfDefaultValue = lineDfDefaultValue.select(ST_NDims("geom")) val actualDefaultValue = dfDefaultValue.take(1)(0).getInt(0) assert(expected == actualDefaultValue) + assert(expected == actual) } } } From c2e27d856a7af9c714300151dd7735917069ef40 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Sun, 11 Jun 2023 21:23:54 -0700 Subject: [PATCH 24/41] Addressed PR comments --- .../org/apache/sedona/common/Functions.java | 4 ++-- .../apache/sedona/common/utils/GeomUtils.java | 9 ++------- .../org/apache/sedona/sql/UDF/Catalog.scala | 2 +- .../sedona/sql/dataFrameAPITestScala.scala | 20 +++++++++---------- .../apache/sedona/sql/functionTestScala.scala | 2 +- 5 files changed, 16 insertions(+), 21 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index 072fa40b84..7a0e469fff 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -854,11 +854,11 @@ public static int numPoints(Geometry geometry) throws Exception { } public static Geometry force3D(Geometry geometry, double zValue) { - return GeomUtils.get3DGeom(geometry, zValue, false); + return GeomUtils.get3DGeom(geometry, zValue); } public static Geometry force3D(Geometry geometry) { - return GeomUtils.get3DGeom(geometry, -1, true); + return GeomUtils.get3DGeom(geometry, 0.0); } public static Geometry geometricMedian(Geometry geometry, double tolerance, int maxIter, boolean failIfNotConverged) throws Exception { diff --git a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java index 3f19783e45..29ddf5725b 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java @@ -421,18 +421,13 @@ public static Geometry[] getSubGeometries(Geometry geom) { } - public static Geometry get3DGeom(Geometry geometry, double zValue, boolean addDefaultZValue) { + public static Geometry get3DGeom(Geometry geometry, double zValue) { Coordinate[] coordinates = geometry.getCoordinates(); if (coordinates.length == 0) return geometry; - boolean is3d = !Double.isNaN(coordinates[0].z); for(int i = 0; i < coordinates.length; i++) { if(!is3d) { - if (addDefaultZValue) { - coordinates[i].setZ(0.0); - }else { - coordinates[i].setZ(zValue); - } + coordinates[i].setZ(zValue); } } geometry.geometryChanged(); diff --git a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala index 495aace49e..9edc563f26 100644 --- a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala +++ b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala @@ -148,7 +148,7 @@ object Catalog { function[ST_AreaSpheroid](), function[ST_LengthSpheroid](), function[ST_NumPoints](), - function[ST_Force3D](), + function[ST_Force3D](0.0), // Expression for rasters function[RS_NormalizedDifference](), function[RS_Mean](), diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala index c4311dadd9..70b5aa693a 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.sedona_sql.expressions.st_functions._ import org.apache.spark.sql.sedona_sql.expressions.st_predicates._ import org.apache.spark.sql.sedona_sql.expressions.st_aggregates._ import org.junit.Assert.assertEquals +import org.locationtech.jts.io.WKTWriter import scala.collection.mutable @@ -959,16 +960,15 @@ class dataFrameAPITestScala extends TestBaseScala { } it("Passed ST_Force3D") { - val lineDf = sparkSession.sql("SELECT ST_Force3D(ST_GeomFromWKT('LINESTRING (0 1, 1 0, 2 0)'), 2.3) AS geom") - val df = lineDf.select(ST_NDims("geom")) - val actual = df.take(1)(0).getInt(0) - val expected = 3 - - val lineDfDefaultValue = sparkSession.sql("SELECT ST_Force3D(ST_GeomFromWKT('LINESTRING (0 1, 1 0, 2 0)'), 0.0) AS geom") - val dfDefaultValue = lineDfDefaultValue.select(ST_NDims("geom")) - val actualDefaultValue = dfDefaultValue.take(1)(0).getInt(0) - assert(expected == actualDefaultValue) - assert(expected == actual) + val lineDf = sparkSession.sql("SELECT ST_GeomFromWKT('LINESTRING (0 1, 1 0, 2 0)') AS geom") + val expectedGeom = "LINESTRING Z(0 1 2.3, 1 0 2.3, 2 0 2.3)" + val expectedGeomDefaultValue = "LINESTRING Z(0 1 0, 1 0 0, 2 0 0)" + val wktWriter = new WKTWriter(3) + val forcedGeom = lineDf.select(ST_Force3D("geom", 2.3)).take(1)(0).get(0).asInstanceOf[Geometry] + assertEquals(expectedGeom, wktWriter.write(forcedGeom)) + val lineDfDefaultValue = sparkSession.sql("SELECT ST_GeomFromWKT('LINESTRING (0 1, 1 0, 2 0)') AS geom") + val actualGeomDefaultValue = lineDfDefaultValue.select(ST_Force3D("geom")).take(1)(0).get(0).asInstanceOf[Geometry] + assertEquals(expectedGeomDefaultValue, wktWriter.write(actualGeomDefaultValue)) } } } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index b7b1abd884..b2ad34e0b1 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -1931,7 +1931,7 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample ) for (((geom), expectedResult) <- geomTestCases) { val df = sparkSession.sql(s"SELECT ST_AsText(ST_Force3D(ST_GeomFromWKT($geom), 1)) AS geom, " + s"$expectedResult") - val dfDefaultValue = sparkSession.sql(s"SELECT ST_AsText(ST_Force3D(ST_GeomFromWKT($geom), 0.0)) AS geom, " + s"$expectedResult") + val dfDefaultValue = sparkSession.sql(s"SELECT ST_AsText(ST_Force3D(ST_GeomFromWKT($geom))) AS geom, " + s"$expectedResult") val actual = df.take(1)(0).get(0).asInstanceOf[String] val expected = df.take(1)(0).get(1).asInstanceOf[GenericRowWithSchema].get(0).asInstanceOf[String] val actualDefaultValue = dfDefaultValue.take(1)(0).get(0).asInstanceOf[String] From 98151613400996387b2d4278cc943d60ec01ec19 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Mon, 12 Jun 2023 13:20:19 -0700 Subject: [PATCH 25/41] Update community/develop to include steps to run python test cases. Update java/scala test execution --- docs/community/develop.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/community/develop.md b/docs/community/develop.md index dd8c47d2e9..9483322395 100644 --- a/docs/community/develop.md +++ b/docs/community/develop.md @@ -4,7 +4,7 @@ ### IDE -We recommend Intellij IDEA with Scala plugin installed. +We recommend Intellij IDEA with Scala plugin installed. Please make sure that the IDE has JDK 1.8 set as project default. ### Import the project @@ -51,6 +51,10 @@ Make sure you reload the POM.xml or reload the maven project. The IDE will ask y #### Run all unit tests In a terminal, go to the Sedona root folder. Run `mvn clean install`. All tests will take more than 15 minutes. To only build the project jars, run `mvn clean install -DskipTests`. +!!!Note + `mvn clean install` will compile Sedona with Spark 3.0 and Scala 2.12. If you have a different version of Spark in $SPARK_HOME, make sure to specify that using -Dspark command line arg. + For example, to compile sedona with Spark 3.4 and Scala 2.12, use: `mvn clean install -Dspark=3.4 -Dscala=2.12` + More details can be found on [Compile Sedona](../../setup/compile/) @@ -78,7 +82,19 @@ Re-run the test case. Do NOT right click the test case to re-run. Instead, click ## Python developers -More details to come. +#### Run all python tests + +To run all Python test cases, follow steps mentioned [here](../../setup/compile/#run-python-test). + +#### Run all python tests in a single test file +To run a particular python test file, specify the path of the .py file to pipenv. + +For example, to run all tests in `test_function.py` located in `python/tests/sql/`, use: `pipenv run pytest tests/sql/test_function.py`. + +#### Run a single test +To run a particular test in a particular .py test file, specify `file_name::class_name::test_name` to the pytest command. + +For example, to run the test on ST_Contains function located in sql/test_predicate.py, use: `pipenv run pytest tests/sql/test_predicate.py::TestPredicate::test_st_contains` ### IDE From b5cf30cb9f5f475d685473465c237134a65be4e0 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Tue, 13 Jun 2023 10:25:22 -0700 Subject: [PATCH 26/41] Add ST_NRings --- .../org/apache/sedona/common/Functions.java | 20 +++++ .../apache/sedona/common/utils/GeomUtils.java | 9 ++ .../apache/sedona/common/FunctionsTest.java | 83 +++++++++++++++++++ docs/api/flink/Function.md | 39 ++++++++- docs/api/sql/Function.md | 31 +++++++ .../java/org/apache/sedona/flink/Catalog.java | 3 +- .../sedona/flink/expressions/Functions.java | 8 ++ .../org/apache/sedona/flink/FunctionTest.java | 8 ++ python/sedona/sql/st_functions.py | 11 ++- python/tests/sql/test_dataframe_api.py | 4 +- python/tests/sql/test_function.py | 6 ++ .../org/apache/sedona/sql/UDF/Catalog.scala | 1 + .../sedona_sql/expressions/Functions.scala | 7 ++ .../sedona_sql/expressions/st_functions.scala | 4 + .../sedona/sql/dataFrameAPITestScala.scala | 8 ++ .../apache/sedona/sql/functionTestScala.scala | 14 ++++ 16 files changed, 250 insertions(+), 6 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index 7a0e469fff..fdd6cc9a55 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -861,6 +861,26 @@ public static Geometry force3D(Geometry geometry) { return GeomUtils.get3DGeom(geometry, 0.0); } + public static Integer nRings(Geometry geometry) throws Exception { + String geometryType = geometry.getGeometryType(); + if (!(geometry instanceof Polygon || geometry instanceof MultiPolygon)) { + throw new IllegalArgumentException("Unsupported geometry type: " + geometryType + ", only Polygon or MultiPolygon geometries are supported."); + } + int numRings = 0; + if (geometry instanceof Polygon) { + Polygon polygon = (Polygon) geometry; + numRings = GeomUtils.getPolygonNumRings(polygon); + }else { + MultiPolygon multiPolygon = (MultiPolygon) geometry; + int numPolygons = multiPolygon.getNumGeometries(); + for (int i = 0; i < numPolygons; i++) { + Polygon polygon = (Polygon) multiPolygon.getGeometryN(i); + numRings += GeomUtils.getPolygonNumRings(polygon); + } + } + return numRings; + } + public static Geometry geometricMedian(Geometry geometry, double tolerance, int maxIter, boolean failIfNotConverged) throws Exception { String geometryType = geometry.getGeometryType(); if(!(Geometry.TYPENAME_POINT.equals(geometryType) || Geometry.TYPENAME_MULTIPOINT.equals(geometryType))) { diff --git a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java index 29ddf5725b..13585df8a0 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java @@ -433,4 +433,13 @@ public static Geometry get3DGeom(Geometry geometry, double zValue) { geometry.geometryChanged(); return geometry; } + + public static int getPolygonNumRings(Polygon polygon) { + LinearRing shell = polygon.getExteriorRing(); + if (shell == null || shell.isEmpty()) { + return 0; + }else { + return 1 + polygon.getNumInteriorRing(); + } + } } diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index f54f2711cd..71525ec174 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -24,6 +24,7 @@ import org.locationtech.jts.io.WKTReader; import org.locationtech.jts.io.WKTWriter; +import javax.sound.sampled.Line; import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -648,4 +649,86 @@ public void force3DEmptyObjectDefaultValue() { assertEquals(emptyLine.isEmpty(), forcedEmptyLine.isEmpty()); } + @Test + public void nRingsPolygonOnlyExternal() throws Exception { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); + Integer expected = 1; + Integer actual = Functions.nRings(polygon); + assertEquals(expected, actual); + } + + @Test + public void nRingsPolygonWithHoles() throws Exception { + LinearRing shell = GEOMETRY_FACTORY.createLinearRing(coordArray(1, 0, 1, 6, 6, 6, 6, 0, 1, 0)); + LinearRing[] holes = new LinearRing[] {GEOMETRY_FACTORY.createLinearRing(coordArray(2, 1, 2, 2, 3, 2, 3, 1, 2, 1)), + GEOMETRY_FACTORY.createLinearRing(coordArray(4, 1, 4, 2, 5, 2, 5, 1, 4, 1))}; + Polygon polygonWithHoles = GEOMETRY_FACTORY.createPolygon(shell, holes); + Integer expected = 3; + Integer actual = Functions.nRings(polygonWithHoles); + assertEquals(expected, actual); + } + + @Test public void nRingsPolygonEmpty() throws Exception { + Polygon emptyPolygon = GEOMETRY_FACTORY.createPolygon(); + Integer expected = 0; + Integer actual = Functions.nRings(emptyPolygon); + assertEquals(expected, actual); + } + + @Test + public void nRingsMultiPolygonOnlyExternal() throws Exception { + MultiPolygon multiPolygon = GEOMETRY_FACTORY.createMultiPolygon(new Polygon[] {GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)), + GEOMETRY_FACTORY.createPolygon(coordArray(5, 0, 5, 1, 7, 1, 7, 0, 5, 0))}); + Integer expected = 2; + Integer actual = Functions.nRings(multiPolygon); + assertEquals(expected, actual); + } + + @Test + public void nRingsMultiPolygonOnlyWithHoles() throws Exception { + LinearRing shell1 = GEOMETRY_FACTORY.createLinearRing(coordArray(1, 0, 1, 6, 6, 6, 6, 0, 1, 0)); + LinearRing[] holes1 = new LinearRing[] {GEOMETRY_FACTORY.createLinearRing(coordArray(2, 1, 2, 2, 3, 2, 3, 1, 2, 1)), + GEOMETRY_FACTORY.createLinearRing(coordArray(4, 1, 4, 2, 5, 2, 5, 1, 4, 1))}; + Polygon polygonWithHoles1 = GEOMETRY_FACTORY.createPolygon(shell1, holes1); + LinearRing shell2 = GEOMETRY_FACTORY.createLinearRing(coordArray(10, 0, 10, 6, 16, 6, 16, 0, 10, 0)); + LinearRing[] holes2 = new LinearRing[] {GEOMETRY_FACTORY.createLinearRing(coordArray(12, 1, 12, 2, 13, 2, 13, 1, 12, 1)), + GEOMETRY_FACTORY.createLinearRing(coordArray(14, 1, 14, 2, 15, 2, 15, 1, 14, 1))}; + Polygon polygonWithHoles2 = GEOMETRY_FACTORY.createPolygon(shell2, holes2); + MultiPolygon multiPolygon = GEOMETRY_FACTORY.createMultiPolygon(new Polygon[]{polygonWithHoles1, polygonWithHoles2}); + Integer expected = 6; + Integer actual = Functions.nRings(multiPolygon); + assertEquals(expected, actual); + } + + @Test + public void nRingsMultiPolygonEmpty() throws Exception { + MultiPolygon multiPolygon = GEOMETRY_FACTORY.createMultiPolygon(new Polygon[] {GEOMETRY_FACTORY.createPolygon(), + GEOMETRY_FACTORY.createPolygon()}); + Integer expected = 0; + Integer actual = Functions.nRings(multiPolygon); + assertEquals(expected, actual); + } + + @Test + public void nRingsMultiPolygonMixed() throws Exception { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); + LinearRing shell = GEOMETRY_FACTORY.createLinearRing(coordArray(1, 0, 1, 6, 6, 6, 6, 0, 1, 0)); + LinearRing[] holes = new LinearRing[] {GEOMETRY_FACTORY.createLinearRing(coordArray(2, 1, 2, 2, 3, 2, 3, 1, 2, 1)), + GEOMETRY_FACTORY.createLinearRing(coordArray(4, 1, 4, 2, 5, 2, 5, 1, 4, 1))}; + Polygon polygonWithHoles = GEOMETRY_FACTORY.createPolygon(shell, holes); + Polygon emptyPolygon = GEOMETRY_FACTORY.createPolygon(); + MultiPolygon multiPolygon = GEOMETRY_FACTORY.createMultiPolygon(new Polygon[] {polygon, polygonWithHoles, emptyPolygon}); + Integer expected = 4; + Integer actual = Functions.nRings(multiPolygon); + assertEquals(expected, actual); + } + + @Test + public void nRingsUnsupported() { + LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray3d(0, 1, 1, 1, 2, 1, 1, 2, 2)); + String expected = "Unsupported geometry type: " + "LineString" + ", only Polygon or MultiPolygon geometries are supported."; + Exception e = assertThrows(IllegalArgumentException.class, () -> Functions.nRings(lineString)); + assertEquals(expected, e.getMessage()); + } + } diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index 11d86ff6e1..24793c4518 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -703,7 +703,7 @@ SELECT ST_NDims(ST_GeomFromEWKT('POINT(1 1 2)')) Output: `3` -Spark SQL example with x,y coordinate: +Example with x,y coordinate: ```sql SELECT ST_NDims(ST_GeomFromText('POINT(1 1)')) @@ -711,6 +711,39 @@ SELECT ST_NDims(ST_GeomFromText('POINT(1 1)')) Output: `2` +## ST_NRings + +Introduction: Returns the number of rings in a Polygon or MultiPolygon. Contrary to ST_NumInteriorRings, +this function also takes into account the number of exterior rings. + +This function returns 0 for an empty Polygon or MultiPolygon. +If the geometry is not a Polygon or MultiPolygon, an IllegalArgument Exception is thrown. + +Format: `ST_NRings(geom: geometry)` + +Since: `1.4.1` + + +Examples: + +Input: `POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))` + +Output: `1` + +Input: `'MULTIPOLYGON (((1 0, 1 6, 6 6, 6 0, 1 0), (2 1, 2 2, 3 2, 3 1, 2 1)), ((10 0, 10 6, 16 6, 16 0, 10 0), (12 1, 12 2, 13 2, 13 1, 12 1)))'` + +Output: `4` + +Input: `'POLYGON EMPTY'` + +Output: `0` + +Input: `'LINESTRING (1 0, 1 1, 2 1)'` + +Output: `Unsupported geometry type: LineString, only Polygon or MultiPolygon geometries are supported.` + + + ## ST_NumGeometries Introduction: Returns the number of Geometries. If geometry is a GEOMETRYCOLLECTION (or MULTI*) return the number of geometries, for single geometries will return 1. @@ -945,13 +978,13 @@ Format: `ST_Transform (A:geometry, SourceCRS:string, TargetCRS:string ,[Optional Since: `v1.2.0` -Spark SQL example (simple): +Example (simple): ```sql SELECT ST_Transform(polygondf.countyshape, 'epsg:4326','epsg:3857') FROM polygondf ``` -Spark SQL example (with optional parameters): +Example (with optional parameters): ```sql SELECT ST_Transform(polygondf.countyshape, 'epsg:4326','epsg:3857', false) FROM polygondf diff --git a/docs/api/sql/Function.md b/docs/api/sql/Function.md index 05265b5381..c3665ecd98 100644 --- a/docs/api/sql/Function.md +++ b/docs/api/sql/Function.md @@ -1108,6 +1108,37 @@ SELECT ST_NPoints(polygondf.countyshape) FROM polygondf ``` +## ST_NRings + +Introduction: Returns the number of rings in a Polygon or MultiPolygon. Contrary to ST_NumInteriorRings, +this function also takes into account the number of exterior rings. + +This function returns 0 for an empty Polygon or MultiPolygon. +If the geometry is not a Polygon or MultiPolygon, an IllegalArgument Exception is thrown. + +Format: `ST_NRings(geom: geometry)` + +Since: `1.4.1` + + +Examples: + +Input: `POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))` + +Output: `1` + +Input: `'MULTIPOLYGON (((1 0, 1 6, 6 6, 6 0, 1 0), (2 1, 2 2, 3 2, 3 1, 2 1)), ((10 0, 10 6, 16 6, 16 0, 10 0), (12 1, 12 2, 13 2, 13 1, 12 1)))'` + +Output: `4` + +Input: `'POLYGON EMPTY'` + +Output: `0` + +Input: `'LINESTRING (1 0, 1 1, 2 1)'` + +Output: `Unsupported geometry type: LineString, only Polygon or MultiPolygon geometries are supported.` + ## ST_NumGeometries Introduction: Returns the number of Geometries. If geometry is a GEOMETRYCOLLECTION (or MULTI*) return the number of geometries, for single geometries will return 1. diff --git a/flink/src/main/java/org/apache/sedona/flink/Catalog.java b/flink/src/main/java/org/apache/sedona/flink/Catalog.java index f72f0793ec..1000e37022 100644 --- a/flink/src/main/java/org/apache/sedona/flink/Catalog.java +++ b/flink/src/main/java/org/apache/sedona/flink/Catalog.java @@ -96,7 +96,8 @@ public static UserDefinedFunction[] getFuncs() { new Functions.ST_S2CellIDs(), new Functions.ST_GeometricMedian(), new Functions.ST_NumPoints(), - new Functions.ST_Force3D() + new Functions.ST_Force3D(), + new Functions.ST_NRings() }; } diff --git a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java index 33f1b6c0f4..dce65f6ef0 100644 --- a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java +++ b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java @@ -599,4 +599,12 @@ public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.j } } + public static class ST_NRings extends ScalarFunction { + @DataTypeHint(value = "Integer") + public int eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o) throws Exception { + Geometry geom = (Geometry) o; + return org.apache.sedona.common.Functions.nRings(geom); + } + } + } diff --git a/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java b/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java index 219970c543..2fd80acd10 100644 --- a/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java +++ b/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java @@ -719,4 +719,12 @@ public void testForce3DDefaultValue() { assertEquals(expectedDims, actual); } + @Test + public void testNRings() { + Integer expected = 1; + Table pointTable = tableEnv.sqlQuery("SELECT ST_NRings(ST_GeomFromWKT('POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))'))"); + Integer actual = (Integer) first(pointTable).getField(0); + assertEquals(expected, actual); + } + } diff --git a/python/sedona/sql/st_functions.py b/python/sedona/sql/st_functions.py index 66cf149ad2..fbfc73f4fa 100644 --- a/python/sedona/sql/st_functions.py +++ b/python/sedona/sql/st_functions.py @@ -109,7 +109,8 @@ "ST_ZMax", "ST_ZMin", "ST_NumPoints", - "ST_Force3D" + "ST_Force3D", + "ST_NRings" ] @@ -1253,3 +1254,11 @@ def ST_Force3D(geometry: ColumnOrName, zValue: Optional[Union[ColumnOrName, floa """ args = (geometry, zValue) return _call_st_function("ST_Force3D", args) + +def ST_NRings(geometry: ColumnOrName) -> Column: + """ + Returns the total number of rings in a Polygon or MultiPolygon. Compared to ST_NumInteriorRings, ST_NRings takes exterior rings into account as well. + :param geometry: Geometry column to calculate rings for + :return: Number of exterior rings + interior rings (if any) for the given Polygon or MultiPolygon + """ + return _call_st_function("ST_NRings", geometry) diff --git a/python/tests/sql/test_dataframe_api.py b/python/tests/sql/test_dataframe_api.py index f9fbfc82fc..c4cf18474e 100644 --- a/python/tests/sql/test_dataframe_api.py +++ b/python/tests/sql/test_dataframe_api.py @@ -110,6 +110,7 @@ (stf.ST_Multi, ("point",), "point_geom", "", "MULTIPOINT (0 1)"), (stf.ST_Normalize, ("geom",), "triangle_geom", "", "POLYGON ((0 0, 1 1, 1 0, 0 0))"), (stf.ST_NPoints, ("line",), "linestring_geom", "", 6), + (stf.ST_NRings, ("geom",), "square_geom", "", 1), (stf.ST_NumGeometries, ("geom",), "multipoint", "", 2), (stf.ST_NumInteriorRings, ("geom",), "geom_with_hole", "", 1), (stf.ST_NumPoints, ("line",), "linestring_geom", "", 6), @@ -137,7 +138,6 @@ (stf.ST_YMax, ("geom",), "triangle_geom", "", 1.0), (stf.ST_YMin, ("geom",), "triangle_geom", "", 0.0), (stf.ST_Z, ("b",), "two_points", "", 4.0), - (stf.ST_NumPoints, ("line",), "linestring_geom", "", 6), # predicates (stp.ST_Contains, ("geom", lambda: f.expr("ST_Point(0.5, 0.25)")), "triangle_geom", "", True), @@ -387,6 +387,8 @@ def base_df(self, request): return TestDataFrameAPI.spark.sql("SELECT ST_GeomFromWKT('POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))') AS a, ST_GeomFromWKT('POLYGON ((1 0, 2 0, 2 1, 1 1, 1 0))') AS b") elif request.param == "line_crossing_poly": return TestDataFrameAPI.spark.sql("SELECT ST_GeomFromWKT('LINESTRING (0 0, 2 1)') AS line, ST_GeomFromWKT('POLYGON ((1 0, 2 0, 2 2, 1 2, 1 0))') AS poly") + elif request.param == "square_geom": + return TestDataFrameAPI.spark.sql("SELECT ST_GeomFromWKT('POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))') AS geom") raise ValueError(f"Invalid base_df name passed: {request.param}") def _id_test_configuration(val): diff --git a/python/tests/sql/test_function.py b/python/tests/sql/test_function.py index 18b38c1182..4f8e670d00 100644 --- a/python/tests/sql/test_function.py +++ b/python/tests/sql/test_function.py @@ -1086,3 +1086,9 @@ def test_force3D(self): actual = actualDf.selectExpr("ST_NDims(geom)").take(1)[0][0] assert expected == actual + def test_nRings(self): + expected = 1 + actualDf = self.spark.sql("SELECT ST_GeomFromText('POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))') AS geom") + actual = actualDf.selectExpr("ST_NRings(geom)").take(1)[0][0] + assert expected == actual + diff --git a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala index 9edc563f26..dc01ea388d 100644 --- a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala +++ b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala @@ -149,6 +149,7 @@ object Catalog { function[ST_LengthSpheroid](), function[ST_NumPoints](), function[ST_Force3D](0.0), + function[ST_NRings](), // Expression for rasters function[RS_NormalizedDifference](), function[RS_Mean](), diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala index e70f14735a..c3b6e6c6de 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala @@ -996,3 +996,10 @@ case class ST_Force3D(inputExpressions: Seq[Expression]) } } +case class ST_NRings(inputExpressions: Seq[Expression]) + extends InferredUnaryExpression(Functions.nRings) with FoldableExpression { + protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { + copy(inputExpressions = newChildren) + } +} + diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala index 6101222b53..42662421d4 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala @@ -313,4 +313,8 @@ object st_functions extends DataFrameAPI { def ST_Force3D(geometry: Column, zValue: Column): Column = wrapExpression[ST_Force3D](geometry, zValue) def ST_Force3D(geometry: String, zValue: Double): Column = wrapExpression[ST_Force3D](geometry, zValue) + + def ST_NRings(geometry: Column): Column = wrapExpression[ST_NRings](geometry) + + def ST_NRings(geometry: String): Column = wrapExpression[ST_NRings](geometry) } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala index 70b5aa693a..5e183e7a8f 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala @@ -970,5 +970,13 @@ class dataFrameAPITestScala extends TestBaseScala { val actualGeomDefaultValue = lineDfDefaultValue.select(ST_Force3D("geom")).take(1)(0).get(0).asInstanceOf[Geometry] assertEquals(expectedGeomDefaultValue, wktWriter.write(actualGeomDefaultValue)) } + + it("Passed ST_NRings") { + val polyDf = sparkSession.sql("SELECT ST_GeomFromWKT('POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))') AS geom") + val expected = 1 + val df = polyDf.select(ST_NRings("geom")) + val actual = df.take(1)(0).getInt(0) + assert(expected == actual) + } } } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index b2ad34e0b1..5dd127820f 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -1940,4 +1940,18 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample assertEquals(expectedDefaultValue, actualDefaultValue); } } + + it("should pass ST_NRings") { + val geomTestCases = Map( + ("'POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))'") -> 1, + ("'MULTIPOLYGON (((1 0, 1 6, 6 6, 6 0, 1 0), (2 1, 2 2, 3 2, 3 1, 2 1)), ((10 0, 10 6, 16 6, 16 0, 10 0), (12 1, 12 2, 13 2, 13 1, 12 1)))'") -> 4, + ("'POLYGON EMPTY'") -> 0 + ) + for (((geom), expectedResult) <- geomTestCases) { + val df = sparkSession.sql(s"SELECT ST_NRings(ST_GeomFromWKT($geom)), " + s"$expectedResult") + val actual = df.take(1)(0).get(0).asInstanceOf[Int] + val expected = df.take(1)(0).get(1).asInstanceOf[java.lang.Integer].intValue() + assertEquals(expected, actual) + } + } } From b89cec165385a2bc0a5709b2a791a6f42a7f91a2 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Tue, 13 Jun 2023 15:31:28 -0700 Subject: [PATCH 27/41] Add ST_Translate --- .../org/apache/sedona/common/Functions.java | 32 +++++++ .../apache/sedona/common/utils/GeomUtils.java | 20 +++++ .../apache/sedona/common/FunctionsTest.java | 83 +++++++++++++++++++ docs/api/flink/Function.md | 25 ++++++ docs/api/sql/Function.md | 24 ++++++ .../java/org/apache/sedona/flink/Catalog.java | 3 +- .../sedona/flink/expressions/Functions.java | 16 ++++ .../org/apache/sedona/flink/FunctionTest.java | 9 ++ python/sedona/sql/st_functions.py | 21 ++++- python/tests/sql/test_dataframe_api.py | 3 +- python/tests/sql/test_function.py | 6 ++ .../org/apache/sedona/sql/UDF/Catalog.scala | 1 + .../sedona_sql/expressions/Functions.scala | 7 ++ .../sedona_sql/expressions/st_functions.scala | 10 +++ .../sedona/sql/dataFrameAPITestScala.scala | 9 ++ .../apache/sedona/sql/functionTestScala.scala | 18 ++++ 16 files changed, 282 insertions(+), 5 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index fdd6cc9a55..e766b72cfd 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -881,6 +881,38 @@ public static Integer nRings(Geometry geometry) throws Exception { return numRings; } + public static Geometry translate(Geometry geometry, double deltaX, double deltaY, double deltaZ) { + if (!geometry.isEmpty()) { + if (geometry instanceof GeometryCollection) { + GeometryCollection geometryCollection = (GeometryCollection) geometry; + int geometriesSize = geometryCollection.getNumGeometries(); + for (int i = 0; i < geometriesSize; i++) { + Geometry currGeometry = geometryCollection.getGeometryN(i); + GeomUtils.translateGeom(currGeometry, deltaX, deltaY, deltaZ); + } + }else { + GeomUtils.translateGeom(geometry, deltaX, deltaY, deltaZ); + } + } + return geometry; + } + + public static Geometry translate(Geometry geometry, double deltaX, double deltaY) { + if (!geometry.isEmpty()) { + if (geometry instanceof GeometryCollection) { + GeometryCollection geometryCollection = (GeometryCollection) geometry; + int geometriesSize = geometryCollection.getNumGeometries(); + for (int i = 0; i < geometriesSize; i++) { + Geometry currGeometry = geometryCollection.getGeometryN(i); + GeomUtils.translateGeom(currGeometry, deltaX, deltaY, 0.0); + } + }else { + GeomUtils.translateGeom(geometry, deltaX, deltaY, 0.0); + } + } + return geometry; + } + public static Geometry geometricMedian(Geometry geometry, double tolerance, int maxIter, boolean failIfNotConverged) throws Exception { String geometryType = geometry.getGeometryType(); if(!(Geometry.TYPENAME_POINT.equals(geometryType) || Geometry.TYPENAME_MULTIPOINT.equals(geometryType))) { diff --git a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java index 13585df8a0..88f8e540cd 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java @@ -14,6 +14,8 @@ package org.apache.sedona.common.utils; import org.locationtech.jts.geom.*; +import org.locationtech.jts.geom.Point; +import org.locationtech.jts.geom.Polygon; import org.locationtech.jts.geom.impl.CoordinateArraySequence; import org.locationtech.jts.geom.CoordinateSequence; @@ -25,8 +27,10 @@ import org.locationtech.jts.operation.polygonize.Polygonizer; import org.locationtech.jts.operation.union.UnaryUnionOp; +import java.awt.*; import java.nio.ByteOrder; import java.util.*; +import java.util.List; import static org.locationtech.jts.geom.Coordinate.NULL_ORDINATE; @@ -442,4 +446,20 @@ public static int getPolygonNumRings(Polygon polygon) { return 1 + polygon.getNumInteriorRing(); } } + + public static Geometry translateGeom(Geometry geometry, double deltaX, double deltaY, double deltaZ) { + Coordinate[] coordinates = geometry.getCoordinates(); + for (int i = 0; i < coordinates.length; i++) { + Coordinate currCoordinate = coordinates[i]; + currCoordinate.setX(currCoordinate.getX() + deltaX); + currCoordinate.setY(currCoordinate.getY() + deltaY); + if (!Double.isNaN(currCoordinate.z)) { + currCoordinate.setZ(currCoordinate.getZ() + deltaZ); + } + } + if (deltaX != 0 || deltaY != 0 || deltaZ != 0) { + geometry.geometryChanged(); + } + return geometry; + } } diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index 71525ec174..568a923fec 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -731,4 +731,87 @@ public void nRingsUnsupported() { assertEquals(expected, e.getMessage()); } + @Test + public void translateEmptyObjectNoDeltaZ() { + LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); + String expected = emptyLineString.toText(); + String actual = Functions.translate(emptyLineString, 1, 1).toText(); + assertEquals(expected, actual); + } + + @Test + public void translateEmptyObjectDeltaZ() { + LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); + String expected = emptyLineString.toText(); + String actual = Functions.translate(emptyLineString, 1, 1, 2).toText(); + assertEquals(expected, actual); + } + + @Test + public void translate2DGeomNoDeltaZ() { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); + String expected = GEOMETRY_FACTORY.createPolygon(coordArray(2, 1, 2, 2, 3, 2, 3, 1, 2, 1)).toText(); + String actual = Functions.translate(polygon, 1, 1).toText(); + assertEquals(expected, actual); + } + + @Test + public void translate2DGeomDeltaZ() { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); + String expected = GEOMETRY_FACTORY.createPolygon(coordArray(2, 1, 2, 2, 3, 2, 3, 1, 2, 1)).toText(); + String actual = Functions.translate(polygon, 1, 1, 1).toText(); + assertEquals(expected, actual); + } + + @Test + public void translate3DGeomNoDeltaZ() { + WKTWriter wktWriter = new WKTWriter(3); + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 0, 1, 1, 1, 1, 2, 1, 1, 2, 0, 1, 1, 0, 1)); + Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray3d(2, 1, 1, 2, 2, 1, 3, 2, 1, 3, 1, 1, 2, 1, 1)); + assertEquals(wktWriter.write(expectedPolygon), wktWriter.write(Functions.translate(polygon, 1, 1))); + } + + @Test + public void translate3DGeomDeltaZ() { + WKTWriter wktWriter = new WKTWriter(3); + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 0, 1, 1, 1, 1, 2, 1, 1, 2, 0, 1, 1, 0, 1)); + Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray3d(2, 1, 2, 2, 2, 2, 3, 2, 2, 3, 1, 2, 2, 1, 2)); + assertEquals(wktWriter.write(expectedPolygon), wktWriter.write(Functions.translate(polygon, 1, 1, 1))); + } + + @Test + public void translateHybridGeomCollectionNoDeltaZ() { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); + Point point3D = GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1, 1)); + LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); + Geometry geomCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {polygon, point3D, emptyLineString}); + Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray(2, 1, 2, 2, 3, 2, 3, 1, 2, 1)); + Point expectedPoint3D = GEOMETRY_FACTORY.createPoint(new Coordinate(2, 2, 1)); + WKTWriter wktWriter3D = new WKTWriter(3); + GeometryCollection actualGeometry = (GeometryCollection) Functions.translate(geomCollection, 1, 1); + + assertEquals(expectedPolygon.toText(), actualGeometry.getGeometryN(0).toText()); + assertEquals(wktWriter3D.write(expectedPoint3D), wktWriter3D.write(actualGeometry.getGeometryN(1))); + assertEquals(emptyLineString.toText(), actualGeometry.getGeometryN(2).toText()); + } + + @Test + public void translateHybridGeomCollectionDeltaZ() { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); + Point point3D = GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1, 1)); + LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); + Geometry geomCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {polygon, point3D, emptyLineString}); + Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray(2, 1, 2, 2, 3, 2, 3, 1, 2, 1)); + Point expectedPoint3D = GEOMETRY_FACTORY.createPoint(new Coordinate(2, 2, 2)); + WKTWriter wktWriter3D = new WKTWriter(3); + GeometryCollection actualGeometry = (GeometryCollection) Functions.translate(geomCollection, 1, 1, 1); + + assertEquals(expectedPolygon.toText(), actualGeometry.getGeometryN(0).toText()); + assertEquals(wktWriter3D.write(expectedPoint3D), wktWriter3D.write(actualGeometry.getGeometryN(1))); + assertEquals(emptyLineString.toText(), actualGeometry.getGeometryN(2).toText()); + } + + + + } diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index 24793c4518..d226e8c367 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -993,6 +993,31 @@ FROM polygondf !!!note The detailed EPSG information can be searched on [EPSG.io](https://epsg.io/). +## ST_Translate +Introduction: Returns the input geometry with its X, Y and Z coordinates (if present in the geometry) translated by deltaX, deltaY and deltaZ (if specified) + +If the geometry is 2D, and a deltaZ parameter is specified, no change is done to the Z coordinate of the geometry and the resultant geometry is also 2D. + +If the geometry is empty, no change is done to it. + +If a geometry collection is given, all geometries of the collection are individually translated. + +Format: `ST_Translate(geometry: geometry, deltaX: deltaX, deltaY: deltaY, deltaZ: deltaZ)` + +Since: `1.4.1` + +Example: + +Input: `ST_Translate(GEOMETRYCOLLECTION(POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0)), POINT(1, 1, 1), LINESTRING EMPTY), 2, 2, 3)` + +Output: `GEOMETRYCOLLECTION(POLYGON ((3 2, 3 3, 4 3, 4 2, 3 2)), POINT(3, 3, 4), LINESTRING EMPTY)` + +Input: `ST_Translate(POINT(1, 3, 2), 1, 2)` + +Output: `POINT(2, 5, 2)` + + + ## ST_X Introduction: Returns X Coordinate of given Point, null otherwise. diff --git a/docs/api/sql/Function.md b/docs/api/sql/Function.md index c3665ecd98..65b77e612f 100644 --- a/docs/api/sql/Function.md +++ b/docs/api/sql/Function.md @@ -1598,6 +1598,30 @@ FROM polygondf !!!note The detailed EPSG information can be searched on [EPSG.io](https://epsg.io/). + +## ST_Translate +Introduction: Returns the input geometry with its X, Y and Z coordinates (if present in the geometry) translated by deltaX, deltaY and deltaZ (if specified) + +If the geometry is 2D, and a deltaZ parameter is specified, no change is done to the Z coordinate of the geometry and the resultant geometry is also 2D. + +If the geometry is empty, no change is done to it. + +If a geometry collection is given, all geometries of the collection are individually translated. + +Format: `ST_Translate(geometry: geometry, deltaX: deltaX, deltaY: deltaY, deltaZ: deltaZ)` + +Since: `1.4.1` + +Example: + +Input: `ST_Translate(GEOMETRYCOLLECTION(POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0)), POINT(1, 1, 1), LINESTRING EMPTY), 2, 2, 3)` + +Output: `GEOMETRYCOLLECTION(POLYGON ((3 2, 3 3, 4 3, 4 2, 3 2)), POINT(3, 3, 4), LINESTRING EMPTY)` + +Input: `ST_Translate(POINT(1, 3, 2), 1, 2)` + +Output: `POINT(2, 5, 2)` + ## ST_Union Introduction: Return the union of geometry A and B diff --git a/flink/src/main/java/org/apache/sedona/flink/Catalog.java b/flink/src/main/java/org/apache/sedona/flink/Catalog.java index 1000e37022..8d3559599e 100644 --- a/flink/src/main/java/org/apache/sedona/flink/Catalog.java +++ b/flink/src/main/java/org/apache/sedona/flink/Catalog.java @@ -97,7 +97,8 @@ public static UserDefinedFunction[] getFuncs() { new Functions.ST_GeometricMedian(), new Functions.ST_NumPoints(), new Functions.ST_Force3D(), - new Functions.ST_NRings() + new Functions.ST_NRings(), + new Functions.ST_Translate(), }; } diff --git a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java index dce65f6ef0..79adcc8d23 100644 --- a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java +++ b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java @@ -607,4 +607,20 @@ public int eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.ge } } + public static class ST_Translate extends ScalarFunction { + @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) + public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o, + @DataTypeHint("Double") Double deltaX, @DataTypeHint("Double") Double deltaY) { + Geometry geometry = (Geometry) o; + return org.apache.sedona.common.Functions.translate(geometry, deltaX, deltaY); + } + + @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) + public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o, + @DataTypeHint("Double") Double deltaX, @DataTypeHint("Double") Double deltaY, @DataTypeHint("Double") Double deltaZ) { + Geometry geometry = (Geometry) o; + return org.apache.sedona.common.Functions.translate(geometry, deltaX, deltaY, deltaZ); + } + } + } diff --git a/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java b/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java index 2fd80acd10..0bf2536d7f 100644 --- a/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java +++ b/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java @@ -727,4 +727,13 @@ public void testNRings() { assertEquals(expected, actual); } + @Test + public void testTranslate() { + Table polyTable = tableEnv.sqlQuery("SELECT ST_Translate(ST_GeomFromWKT('POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))'), 2, 5)" + "AS " + polygonColNames[0]); + polyTable = polyTable.select(call(Functions.ST_AsText.class.getSimpleName(), $(polygonColNames[0]))); + String expected = "POLYGON ((3 5, 3 6, 4 6, 4 5, 3 5))"; + String actual = (String) first(polyTable).getField(0); + assertEquals(expected, actual); + } + } diff --git a/python/sedona/sql/st_functions.py b/python/sedona/sql/st_functions.py index fbfc73f4fa..32a2f2728c 100644 --- a/python/sedona/sql/st_functions.py +++ b/python/sedona/sql/st_functions.py @@ -110,7 +110,8 @@ "ST_ZMin", "ST_NumPoints", "ST_Force3D", - "ST_NRings" + "ST_NRings", + "ST_Translate" ] @@ -1234,7 +1235,7 @@ def ST_ZMin(geometry: ColumnOrName) -> Column: :rtype: Column """ return _call_st_function("ST_ZMin", geometry) - +@validate_argument_types def ST_NumPoints(geometry: ColumnOrName) -> Column: """Return the number of points in a LineString :param geometry: Geometry column to get number of points from. @@ -1244,7 +1245,7 @@ def ST_NumPoints(geometry: ColumnOrName) -> Column: """ return _call_st_function("ST_NumPoints", geometry) - +@validate_argument_types def ST_Force3D(geometry: ColumnOrName, zValue: Optional[Union[ColumnOrName, float]] = 0.0) -> Column: """ Return a geometry with a 3D coordinate of value 'zValue' forced upon it. No change happens if the geometry is already 3D @@ -1255,6 +1256,7 @@ def ST_Force3D(geometry: ColumnOrName, zValue: Optional[Union[ColumnOrName, floa args = (geometry, zValue) return _call_st_function("ST_Force3D", args) +@validate_argument_types def ST_NRings(geometry: ColumnOrName) -> Column: """ Returns the total number of rings in a Polygon or MultiPolygon. Compared to ST_NumInteriorRings, ST_NRings takes exterior rings into account as well. @@ -1262,3 +1264,16 @@ def ST_NRings(geometry: ColumnOrName) -> Column: :return: Number of exterior rings + interior rings (if any) for the given Polygon or MultiPolygon """ return _call_st_function("ST_NRings", geometry) +@validate_argument_types +def ST_Translate(geometry: ColumnOrName, deltaX: Union[ColumnOrName, float], deltaY: Union[ColumnOrName, float], deltaZ: Optional[Union[ColumnOrName, float]] = 0.0) -> Column: + """ + Returns the geometry with x, y and z (if present) coordinates offset by given deltaX, deltaY, and deltaZ values. + :param geometry: Geometry column whose coordinates are to be translated. + :param deltaX: value by which to offset X coordinate. + :param deltaY: value by which to offset Y coordinate. + :param deltaZ: value by which to offset Z coordinate (if present). + :return: The input geometry with its coordinates translated. + """ + args = (geometry, deltaX, deltaY, deltaZ) + return _call_st_function("ST_Translate", args) + diff --git a/python/tests/sql/test_dataframe_api.py b/python/tests/sql/test_dataframe_api.py index c4cf18474e..56229b0e02 100644 --- a/python/tests/sql/test_dataframe_api.py +++ b/python/tests/sql/test_dataframe_api.py @@ -85,7 +85,7 @@ (stf.ST_ExteriorRing, ("geom",), "triangle_geom", "", "LINESTRING (0 0, 1 0, 1 1, 0 0)"), (stf.ST_FlipCoordinates, ("point",), "point_geom", "", "POINT (1 0)"), (stf.ST_Force_2D, ("point",), "point_geom", "", "POINT (0 1)"), - (stf.ST_Force3D, ("point", 1), "point_geom", "", "POINT Z (0 1 1)"), + (stf.ST_Force3D, ("point", 1.0), "point_geom", "", "POINT Z (0 1 1)"), (stf.ST_GeometricMedian, ("multipoint",), "multipoint_geom", "", "POINT (22.500002656424286 21.250001168173426)"), (stf.ST_GeometryN, ("geom", 0), "multipoint", "", "POINT (0 0)"), (stf.ST_GeometryType, ("point",), "point_geom", "", "ST_Point"), @@ -130,6 +130,7 @@ (stf.ST_SubDivideExplode, ("line", 5), "linestring_geom", "collect_list(geom)", ["LINESTRING (0 0, 2.5 0)", "LINESTRING (2.5 0, 5 0)"]), (stf.ST_SymDifference, ("a", "b"), "overlapping_polys", "", "MULTIPOLYGON (((1 0, 0 0, 0 1, 1 1, 1 0)), ((2 0, 2 1, 3 1, 3 0, 2 0)))"), (stf.ST_Transform, ("point", lambda: f.lit("EPSG:4326"), lambda: f.lit("EPSG:32649")), "point_geom", "ST_PrecisionReduce(geom, 2)", "POINT (-33788209.77 0)"), + (stf.ST_Translate, ("geom", 1.0, 1.0,), "square_geom", "", "POLYGON ((2 1, 2 2, 3 2, 3 1, 2 1))"), (stf.ST_Union, ("a", "b"), "overlapping_polys", "", "POLYGON ((1 0, 0 0, 0 1, 1 1, 2 1, 3 1, 3 0, 2 0, 1 0))"), (stf.ST_X, ("b",), "two_points", "", 3.0), (stf.ST_XMax, ("line",), "linestring_geom", "", 5.0), diff --git a/python/tests/sql/test_function.py b/python/tests/sql/test_function.py index 4f8e670d00..6897fb8ee5 100644 --- a/python/tests/sql/test_function.py +++ b/python/tests/sql/test_function.py @@ -1092,3 +1092,9 @@ def test_nRings(self): actual = actualDf.selectExpr("ST_NRings(geom)").take(1)[0][0] assert expected == actual + def test_translate(self): + expected = "POLYGON ((3 5, 3 6, 4 6, 4 5, 3 5))" + actualDf = self.spark.sql("SELECT ST_Translate(ST_GeomFromText('POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))'), 2, 5) AS geom") + actual = actualDf.selectExpr("ST_AsText(geom)").take(1)[0][0] + assert expected == actual + diff --git a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala index dc01ea388d..cbb1319307 100644 --- a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala +++ b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala @@ -150,6 +150,7 @@ object Catalog { function[ST_NumPoints](), function[ST_Force3D](0.0), function[ST_NRings](), + function[ST_Translate](0.0), // Expression for rasters function[RS_NormalizedDifference](), function[RS_Mean](), diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala index c3b6e6c6de..41052806dd 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala @@ -1003,3 +1003,10 @@ case class ST_NRings(inputExpressions: Seq[Expression]) } } +case class ST_Translate(inputExpressions: Seq[Expression]) + extends InferredQuarternaryExpression(Functions.translate) with FoldableExpression { + protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { + copy(inputExpressions = newChildren) + } +} + diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala index 42662421d4..6e110e6ad0 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala @@ -317,4 +317,14 @@ object st_functions extends DataFrameAPI { def ST_NRings(geometry: Column): Column = wrapExpression[ST_NRings](geometry) def ST_NRings(geometry: String): Column = wrapExpression[ST_NRings](geometry) + + def ST_Translate(geometry: Column, deltaX: Column, deltaY: Column, deltaZ: Column): Column = wrapExpression[ST_Translate](geometry, deltaX, deltaY, deltaZ) + + def ST_Translate(geometry: String, deltaX: Double, deltaY: Double, deltaZ: Double): Column = wrapExpression[ST_Translate](geometry, deltaX, deltaY, deltaZ) + + def ST_Translate(geometry: Column, deltaX: Column, deltaY: Column): Column = wrapExpression[ST_Translate](geometry, deltaX, deltaY, 0.0) + + def ST_Translate(geometry: String, deltaX: Double, deltaY: Double): Column = wrapExpression[ST_Translate](geometry, deltaX, deltaY, 0.0) + + } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala index 5e183e7a8f..d50ea07314 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala @@ -978,5 +978,14 @@ class dataFrameAPITestScala extends TestBaseScala { val actual = df.take(1)(0).getInt(0) assert(expected == actual) } + + it("Passed ST_Translate") { + val polyDf = sparkSession.sql("SELECT ST_GeomFromWKT('POLYGON ((1 0 1, 1 1 1, 2 1 1, 2 0 1, 1 0 1))') AS geom") + val df = polyDf.select(ST_Translate("geom", 2, 3, 1)) + val actualGeom = df.take(1)(0).get(0).asInstanceOf[Geometry] + val actual = new WKTWriter(3).write(actualGeom) + val expected = "POLYGON Z((3 3 2, 3 4 2, 4 4 2, 4 3 2, 3 3 2))" + assert(expected == actual) + } } } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index 5dd127820f..f35d666903 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -1954,4 +1954,22 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample assertEquals(expected, actual) } } + + it ("should pass ST_Translate") { + val geomTestCases = Map( + ("'POINT (1 1 1)'") -> ("'POINT Z(2 2 2)'", "'POINT Z(2 2 1)'"), + ("'POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))'") -> ("'POLYGON ((2 1, 2 2, 3 2, 3 1, 2 1))'", "'POLYGON ((2 1, 2 2, 3 2, 3 1, 2 1))'"), + ("'LINESTRING EMPTY'") -> ("'LINESTRING EMPTY'", "'LINESTRING EMPTY'") + ) + for (((geom), expectedResult) <- geomTestCases) { + val df = sparkSession.sql(s"SELECT ST_AsText(ST_Translate(ST_GeomFromWKT($geom), 1, 1, 1)) AS geom, " + s"$expectedResult") + val dfDefaultValue = sparkSession.sql(s"SELECT ST_AsText(ST_Translate(ST_GeomFromWKT($geom), 1, 1)) AS geom, " + s"$expectedResult") + val actual = df.take(1)(0).get(0).asInstanceOf[String] + val actualDefaultValue = dfDefaultValue.take(1)(0).get(0).asInstanceOf[String] + val expected = df.take(1)(0).get(1).asInstanceOf[GenericRowWithSchema].get(0).asInstanceOf[String] + val expectedDefaultValue = dfDefaultValue.take(1)(0).get(1).asInstanceOf[GenericRowWithSchema].get(1).asInstanceOf[String] + assertEquals(expected, actual) + assertEquals(expectedDefaultValue, actualDefaultValue) + } + } } From ece4802ca5ebae48d1c2c08aba138ef506def41b Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Tue, 13 Jun 2023 17:32:52 -0700 Subject: [PATCH 28/41] Updated GeomUtils to remove redundant geom return type --- .../main/java/org/apache/sedona/common/utils/GeomUtils.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java index 88f8e540cd..6478e29a52 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java @@ -447,7 +447,7 @@ public static int getPolygonNumRings(Polygon polygon) { } } - public static Geometry translateGeom(Geometry geometry, double deltaX, double deltaY, double deltaZ) { + public static void translateGeom(Geometry geometry, double deltaX, double deltaY, double deltaZ) { Coordinate[] coordinates = geometry.getCoordinates(); for (int i = 0; i < coordinates.length; i++) { Coordinate currCoordinate = coordinates[i]; @@ -460,6 +460,5 @@ public static Geometry translateGeom(Geometry geometry, double deltaX, double de if (deltaX != 0 || deltaY != 0 || deltaZ != 0) { geometry.geometryChanged(); } - return geometry; } } From 78be0f1791c43d089f0a279aaedd8a5e9c2ace81 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Wed, 14 Jun 2023 15:22:28 -0700 Subject: [PATCH 29/41] Simplified ST_Translate implementation, updated test cases and docs Added geom collection test cases for force3D --- .../org/apache/sedona/common/Functions.java | 25 ++----- .../apache/sedona/common/utils/GeomUtils.java | 2 +- .../apache/sedona/common/FunctionsTest.java | 69 +++++++++++++++---- docs/api/flink/Function.md | 6 +- docs/api/sql/Function.md | 9 ++- .../sedona/sql/dataFrameAPITestScala.scala | 9 ++- .../apache/sedona/sql/functionTestScala.scala | 3 +- 7 files changed, 78 insertions(+), 45 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index e766b72cfd..1efd0f474b 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -835,9 +835,10 @@ private static Coordinate[] extractCoordinates(Geometry geometry) { Coordinate[] points = geometry.getCoordinates(); if(points.length == 0) return points; - boolean is3d = !Double.isNaN(points[0].z); + Coordinate[] coordinates = new Coordinate[points.length]; for(int i = 0; i < points.length; i++) { + boolean is3d = !Double.isNaN(points[i].z); coordinates[i] = points[i].copy(); if(!is3d) coordinates[i].z = 0.0; @@ -883,32 +884,14 @@ public static Integer nRings(Geometry geometry) throws Exception { public static Geometry translate(Geometry geometry, double deltaX, double deltaY, double deltaZ) { if (!geometry.isEmpty()) { - if (geometry instanceof GeometryCollection) { - GeometryCollection geometryCollection = (GeometryCollection) geometry; - int geometriesSize = geometryCollection.getNumGeometries(); - for (int i = 0; i < geometriesSize; i++) { - Geometry currGeometry = geometryCollection.getGeometryN(i); - GeomUtils.translateGeom(currGeometry, deltaX, deltaY, deltaZ); - } - }else { - GeomUtils.translateGeom(geometry, deltaX, deltaY, deltaZ); - } + GeomUtils.translateGeom(geometry, deltaX, deltaY, deltaZ); } return geometry; } public static Geometry translate(Geometry geometry, double deltaX, double deltaY) { if (!geometry.isEmpty()) { - if (geometry instanceof GeometryCollection) { - GeometryCollection geometryCollection = (GeometryCollection) geometry; - int geometriesSize = geometryCollection.getNumGeometries(); - for (int i = 0; i < geometriesSize; i++) { - Geometry currGeometry = geometryCollection.getGeometryN(i); - GeomUtils.translateGeom(currGeometry, deltaX, deltaY, 0.0); - } - }else { - GeomUtils.translateGeom(geometry, deltaX, deltaY, 0.0); - } + GeomUtils.translateGeom(geometry, deltaX, deltaY, 0.0); } return geometry; } diff --git a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java index 6478e29a52..8795f830ac 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java @@ -428,8 +428,8 @@ public static Geometry[] getSubGeometries(Geometry geom) { public static Geometry get3DGeom(Geometry geometry, double zValue) { Coordinate[] coordinates = geometry.getCoordinates(); if (coordinates.length == 0) return geometry; - boolean is3d = !Double.isNaN(coordinates[0].z); for(int i = 0; i < coordinates.length; i++) { + boolean is3d = !Double.isNaN(coordinates[i].z); if(!is3d) { coordinates[i].setZ(zValue); } diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index 568a923fec..dd33bcd840 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -649,6 +649,46 @@ public void force3DEmptyObjectDefaultValue() { assertEquals(emptyLine.isEmpty(), forcedEmptyLine.isEmpty()); } + @Test + public void force3DHybridGeomCollection() { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); + Polygon polygon3D = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 1, 1, 2, 2, 2, 3, 3, 3, 1, 1, 1)); + MultiPolygon multiPolygon = GEOMETRY_FACTORY.createMultiPolygon(new Polygon[] {polygon3D, polygon}); + Point point3D = GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1, 1)); + LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray(1, 0, 1, 1, 1, 2)); + LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); + Geometry geomCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {multiPolygon, point3D, emptyLineString, lineString})}); + Polygon expectedPolygon3D = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 0, 2, 1, 1, 2, 2, 1, 2, 2, 0, 2, 1, 0, 2)); + LineString expectedLineString3D = GEOMETRY_FACTORY.createLineString(coordArray3d(1, 0, 2, 1, 1, 2, 1, 2, 2)); + Geometry actualGeometryCollection = Functions.force3D(geomCollection, 2); + WKTWriter wktWriter3D = new WKTWriter(3); + assertEquals(wktWriter3D.write(polygon3D), wktWriter3D.write(actualGeometryCollection.getGeometryN(0).getGeometryN(0).getGeometryN(0))); + assertEquals(wktWriter3D.write(expectedPolygon3D), wktWriter3D.write(actualGeometryCollection.getGeometryN(0).getGeometryN(0).getGeometryN(1))); + assertEquals(wktWriter3D.write(point3D), wktWriter3D.write(actualGeometryCollection.getGeometryN(0).getGeometryN(1))); + assertEquals(emptyLineString.toText(), actualGeometryCollection.getGeometryN(0).getGeometryN(2).toText()); + assertEquals(wktWriter3D.write(expectedLineString3D), wktWriter3D.write(actualGeometryCollection.getGeometryN(0).getGeometryN(3))); + } + + @Test + public void force3DHybridGeomCollectionDefaultValue() { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); + Polygon polygon3D = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 1, 1, 2, 2, 2, 3, 3, 3, 1, 1, 1)); + MultiPolygon multiPolygon = GEOMETRY_FACTORY.createMultiPolygon(new Polygon[] {polygon3D, polygon}); + Point point3D = GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1, 1)); + LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray(1, 0, 1, 1, 1, 2)); + LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); + Geometry geomCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {multiPolygon, point3D, emptyLineString, lineString})}); + Polygon expectedPolygon3D = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 0, 0, 1, 1, 0, 2, 1, 0, 2, 0, 0, 1, 0, 0)); + LineString expectedLineString3D = GEOMETRY_FACTORY.createLineString(coordArray3d(1, 0, 0, 1, 1, 0, 1, 2, 0)); + Geometry actualGeometryCollection = Functions.force3D(geomCollection); + WKTWriter wktWriter3D = new WKTWriter(3); + assertEquals(wktWriter3D.write(polygon3D), wktWriter3D.write(actualGeometryCollection.getGeometryN(0).getGeometryN(0).getGeometryN(0))); + assertEquals(wktWriter3D.write(expectedPolygon3D), wktWriter3D.write(actualGeometryCollection.getGeometryN(0).getGeometryN(0).getGeometryN(1))); + assertEquals(wktWriter3D.write(point3D), wktWriter3D.write(actualGeometryCollection.getGeometryN(0).getGeometryN(1))); + assertEquals(emptyLineString.toText(), actualGeometryCollection.getGeometryN(0).getGeometryN(2).toText()); + assertEquals(wktWriter3D.write(expectedLineString3D), wktWriter3D.write(actualGeometryCollection.getGeometryN(0).getGeometryN(3))); + } + @Test public void nRingsPolygonOnlyExternal() throws Exception { Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); @@ -782,36 +822,39 @@ public void translate3DGeomDeltaZ() { @Test public void translateHybridGeomCollectionNoDeltaZ() { Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); + Polygon polygon3D = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 0, 1, 2, 0, 2, 2, 1, 2, 1, 0, 1)); + MultiPolygon multiPolygon = GEOMETRY_FACTORY.createMultiPolygon(new Polygon[] {polygon3D, polygon}); Point point3D = GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1, 1)); LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); - Geometry geomCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {polygon, point3D, emptyLineString}); + Geometry geomCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {multiPolygon, point3D, emptyLineString})}); Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray(2, 1, 2, 2, 3, 2, 3, 1, 2, 1)); + Polygon expectedPolygon3D = GEOMETRY_FACTORY.createPolygon(coordArray3d(2, 1, 1, 3, 1, 2, 3, 2, 2, 2, 1, 1)); Point expectedPoint3D = GEOMETRY_FACTORY.createPoint(new Coordinate(2, 2, 1)); WKTWriter wktWriter3D = new WKTWriter(3); GeometryCollection actualGeometry = (GeometryCollection) Functions.translate(geomCollection, 1, 1); - - assertEquals(expectedPolygon.toText(), actualGeometry.getGeometryN(0).toText()); - assertEquals(wktWriter3D.write(expectedPoint3D), wktWriter3D.write(actualGeometry.getGeometryN(1))); - assertEquals(emptyLineString.toText(), actualGeometry.getGeometryN(2).toText()); + assertEquals(wktWriter3D.write(expectedPolygon3D), wktWriter3D.write(actualGeometry.getGeometryN(0).getGeometryN(0).getGeometryN(0))); + assertEquals(expectedPolygon.toText(), actualGeometry.getGeometryN(0).getGeometryN(0).getGeometryN(1).toText()); + assertEquals(wktWriter3D.write(expectedPoint3D), wktWriter3D.write(actualGeometry.getGeometryN(0).getGeometryN(1))); + assertEquals(emptyLineString.toText(), actualGeometry.getGeometryN(0).getGeometryN(2).toText()); } @Test public void translateHybridGeomCollectionDeltaZ() { Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); + Polygon polygon3D = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 0, 1, 2, 0, 2, 2, 1, 2, 1, 0, 1)); + MultiPolygon multiPolygon = GEOMETRY_FACTORY.createMultiPolygon(new Polygon[] {polygon3D, polygon}); Point point3D = GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1, 1)); LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); - Geometry geomCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {polygon, point3D, emptyLineString}); + Geometry geomCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {multiPolygon, point3D, emptyLineString})}); Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray(2, 1, 2, 2, 3, 2, 3, 1, 2, 1)); + Polygon expectedPolygon3D = GEOMETRY_FACTORY.createPolygon(coordArray3d(2, 1, 2, 3, 1, 3, 3, 2, 3, 2, 1, 2)); Point expectedPoint3D = GEOMETRY_FACTORY.createPoint(new Coordinate(2, 2, 2)); WKTWriter wktWriter3D = new WKTWriter(3); GeometryCollection actualGeometry = (GeometryCollection) Functions.translate(geomCollection, 1, 1, 1); - assertEquals(expectedPolygon.toText(), actualGeometry.getGeometryN(0).toText()); - assertEquals(wktWriter3D.write(expectedPoint3D), wktWriter3D.write(actualGeometry.getGeometryN(1))); - assertEquals(emptyLineString.toText(), actualGeometry.getGeometryN(2).toText()); + assertEquals(wktWriter3D.write(expectedPolygon3D), wktWriter3D.write(actualGeometry.getGeometryN(0).getGeometryN(0).getGeometryN(0))); + assertEquals(expectedPolygon.toText(), actualGeometry.getGeometryN(0).getGeometryN(0).getGeometryN(1).toText()); + assertEquals(wktWriter3D.write(expectedPoint3D), wktWriter3D.write(actualGeometry.getGeometryN(0).getGeometryN(1))); + assertEquals(emptyLineString.toText(), actualGeometry.getGeometryN(0).getGeometryN(2).toText()); } - - - - } diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index d226e8c367..fb68b55c93 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -1000,7 +1000,7 @@ If the geometry is 2D, and a deltaZ parameter is specified, no change is done to If the geometry is empty, no change is done to it. -If a geometry collection is given, all geometries of the collection are individually translated. +If the given geometry contains sub-geometries (GEOMETRY COLLECTION, MULTI POLYGON/LINE/POINT), all underlying geometries are individually translated. Format: `ST_Translate(geometry: geometry, deltaX: deltaX, deltaY: deltaY, deltaZ: deltaZ)` @@ -1008,9 +1008,9 @@ Since: `1.4.1` Example: -Input: `ST_Translate(GEOMETRYCOLLECTION(POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0)), POINT(1, 1, 1), LINESTRING EMPTY), 2, 2, 3)` +Input: `ST_Translate(GEOMETRYCOLLECTION(MULTIPOLYGON (((1 0, 1 1, 2 1, 2 0, 1 0)), ((1 2, 3 4, 3 5, 1 2))), POINT(1, 1, 1), LINESTRING EMPTY), 2, 2, 3)` -Output: `GEOMETRYCOLLECTION(POLYGON ((3 2, 3 3, 4 3, 4 2, 3 2)), POINT(3, 3, 4), LINESTRING EMPTY)` +Output: `GEOMETRYCOLLECTION(MULTIPOLYGON (((3 2, 3 3, 4 3, 4 2, 3 2)), ((3 4, 5 6, 5 7, 3 4))), POINT(3, 3, 4), LINESTRING EMPTY)` Input: `ST_Translate(POINT(1, 3, 2), 1, 2)` diff --git a/docs/api/sql/Function.md b/docs/api/sql/Function.md index 65b77e612f..4ef78b148e 100644 --- a/docs/api/sql/Function.md +++ b/docs/api/sql/Function.md @@ -1604,9 +1604,8 @@ Introduction: Returns the input geometry with its X, Y and Z coordinates (if pre If the geometry is 2D, and a deltaZ parameter is specified, no change is done to the Z coordinate of the geometry and the resultant geometry is also 2D. -If the geometry is empty, no change is done to it. - -If a geometry collection is given, all geometries of the collection are individually translated. +If the geometry is empty, no change is done to it. +If the given geometry contains sub-geometries (GEOMETRY COLLECTION, MULTI POLYGON/LINE/POINT), all underlying geometries are individually translated. Format: `ST_Translate(geometry: geometry, deltaX: deltaX, deltaY: deltaY, deltaZ: deltaZ)` @@ -1614,9 +1613,9 @@ Since: `1.4.1` Example: -Input: `ST_Translate(GEOMETRYCOLLECTION(POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0)), POINT(1, 1, 1), LINESTRING EMPTY), 2, 2, 3)` +Input: `ST_Translate(GEOMETRYCOLLECTION(MULTIPOLYGON (((1 0, 1 1, 2 1, 2 0, 1 0)), ((1 2, 3 4, 3 5, 1 2))), POINT(1, 1, 1), LINESTRING EMPTY), 2, 2, 3)` -Output: `GEOMETRYCOLLECTION(POLYGON ((3 2, 3 3, 4 3, 4 2, 3 2)), POINT(3, 3, 4), LINESTRING EMPTY)` +Output: `GEOMETRYCOLLECTION(MULTIPOLYGON (((3 2, 3 3, 4 3, 4 2, 3 2)), ((3 4, 5 6, 5 7, 3 4))), POINT(3, 3, 4), LINESTRING EMPTY)` Input: `ST_Translate(POINT(1, 3, 2), 1, 2)` diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala index d50ea07314..a9aab8444e 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala @@ -982,10 +982,17 @@ class dataFrameAPITestScala extends TestBaseScala { it("Passed ST_Translate") { val polyDf = sparkSession.sql("SELECT ST_GeomFromWKT('POLYGON ((1 0 1, 1 1 1, 2 1 1, 2 0 1, 1 0 1))') AS geom") val df = polyDf.select(ST_Translate("geom", 2, 3, 1)) + val wktWriter3D = new WKTWriter(3); val actualGeom = df.take(1)(0).get(0).asInstanceOf[Geometry] - val actual = new WKTWriter(3).write(actualGeom) + val actual = wktWriter3D.write(actualGeom) val expected = "POLYGON Z((3 3 2, 3 4 2, 4 4 2, 4 3 2, 3 3 2))" assert(expected == actual) + + val dfDefaultValue = polyDf.select(ST_Translate("geom", 2, 3)) + val actualGeomDefaultValue = dfDefaultValue.take(1)(0).get(0).asInstanceOf[Geometry] + val actualDefaultValue = wktWriter3D.write(actualGeomDefaultValue) + val expectedDefaultValue = "POLYGON Z((3 3 1, 3 4 1, 4 4 1, 4 3 1, 3 3 1))" + assert(expectedDefaultValue == actualDefaultValue) } } } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index f35d666903..2bab5e5413 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -1959,7 +1959,8 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample val geomTestCases = Map( ("'POINT (1 1 1)'") -> ("'POINT Z(2 2 2)'", "'POINT Z(2 2 1)'"), ("'POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))'") -> ("'POLYGON ((2 1, 2 2, 3 2, 3 1, 2 1))'", "'POLYGON ((2 1, 2 2, 3 2, 3 1, 2 1))'"), - ("'LINESTRING EMPTY'") -> ("'LINESTRING EMPTY'", "'LINESTRING EMPTY'") + ("'LINESTRING EMPTY'") -> ("'LINESTRING EMPTY'", "'LINESTRING EMPTY'"), + ("'GEOMETRYCOLLECTION (MULTIPOLYGON (((1 0, 1 1, 2 1, 2 0, 1 0)), ((1 2, 3 4, 3 5, 1 2))))'") -> ("'GEOMETRYCOLLECTION (MULTIPOLYGON (((2 1, 2 2, 3 2, 3 1, 2 1)), ((2 3, 4 5, 4 6, 2 3))))'", "'GEOMETRYCOLLECTION (MULTIPOLYGON (((2 1, 2 2, 3 2, 3 1, 2 1)), ((2 3, 4 5, 4 6, 2 3))))'") ) for (((geom), expectedResult) <- geomTestCases) { val df = sparkSession.sql(s"SELECT ST_AsText(ST_Translate(ST_GeomFromWKT($geom), 1, 1, 1)) AS geom, " + s"$expectedResult") From cd12ce2e31492b0b055efc7fe847976c0d66d784 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Wed, 14 Jun 2023 17:43:06 -0700 Subject: [PATCH 30/41] Updated tests for Translate --- .../apache/sedona/common/FunctionsTest.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index dd33bcd840..3a1745b6b4 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -783,23 +783,23 @@ public void translateEmptyObjectNoDeltaZ() { public void translateEmptyObjectDeltaZ() { LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); String expected = emptyLineString.toText(); - String actual = Functions.translate(emptyLineString, 1, 1, 2).toText(); + String actual = Functions.translate(emptyLineString, 1, 3, 2).toText(); assertEquals(expected, actual); } @Test public void translate2DGeomNoDeltaZ() { Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); - String expected = GEOMETRY_FACTORY.createPolygon(coordArray(2, 1, 2, 2, 3, 2, 3, 1, 2, 1)).toText(); - String actual = Functions.translate(polygon, 1, 1).toText(); + String expected = GEOMETRY_FACTORY.createPolygon(coordArray(2, 4, 2, 5, 3, 5, 3, 4, 2, 4)).toText(); + String actual = Functions.translate(polygon, 1, 4).toText(); assertEquals(expected, actual); } @Test public void translate2DGeomDeltaZ() { Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); - String expected = GEOMETRY_FACTORY.createPolygon(coordArray(2, 1, 2, 2, 3, 2, 3, 1, 2, 1)).toText(); - String actual = Functions.translate(polygon, 1, 1, 1).toText(); + String expected = GEOMETRY_FACTORY.createPolygon(coordArray(2, 3, 2, 4, 3, 4, 3, 3, 2, 3)).toText(); + String actual = Functions.translate(polygon, 1, 3, 2).toText(); assertEquals(expected, actual); } @@ -807,16 +807,16 @@ public void translate2DGeomDeltaZ() { public void translate3DGeomNoDeltaZ() { WKTWriter wktWriter = new WKTWriter(3); Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 0, 1, 1, 1, 1, 2, 1, 1, 2, 0, 1, 1, 0, 1)); - Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray3d(2, 1, 1, 2, 2, 1, 3, 2, 1, 3, 1, 1, 2, 1, 1)); - assertEquals(wktWriter.write(expectedPolygon), wktWriter.write(Functions.translate(polygon, 1, 1))); + Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray3d(2, 5, 1, 2, 6, 1, 3, 6, 1, 3, 5, 1, 2, 5, 1)); + assertEquals(wktWriter.write(expectedPolygon), wktWriter.write(Functions.translate(polygon, 1, 5))); } @Test public void translate3DGeomDeltaZ() { WKTWriter wktWriter = new WKTWriter(3); Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 0, 1, 1, 1, 1, 2, 1, 1, 2, 0, 1, 1, 0, 1)); - Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray3d(2, 1, 2, 2, 2, 2, 3, 2, 2, 3, 1, 2, 2, 1, 2)); - assertEquals(wktWriter.write(expectedPolygon), wktWriter.write(Functions.translate(polygon, 1, 1, 1))); + Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray3d(2, 2, 4, 2, 3, 4, 3, 3, 4, 3, 2, 4, 2, 2, 4)); + assertEquals(wktWriter.write(expectedPolygon), wktWriter.write(Functions.translate(polygon, 1, 2, 3))); } @Test @@ -827,11 +827,11 @@ public void translateHybridGeomCollectionNoDeltaZ() { Point point3D = GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1, 1)); LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); Geometry geomCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {multiPolygon, point3D, emptyLineString})}); - Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray(2, 1, 2, 2, 3, 2, 3, 1, 2, 1)); - Polygon expectedPolygon3D = GEOMETRY_FACTORY.createPolygon(coordArray3d(2, 1, 1, 3, 1, 2, 3, 2, 2, 2, 1, 1)); - Point expectedPoint3D = GEOMETRY_FACTORY.createPoint(new Coordinate(2, 2, 1)); + Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray(2, 2, 2, 3, 3, 3, 3, 2, 2, 2)); + Polygon expectedPolygon3D = GEOMETRY_FACTORY.createPolygon(coordArray3d(2, 2, 1, 3, 2, 2, 3, 3, 2, 2, 2, 1)); + Point expectedPoint3D = GEOMETRY_FACTORY.createPoint(new Coordinate(2, 3, 1)); WKTWriter wktWriter3D = new WKTWriter(3); - GeometryCollection actualGeometry = (GeometryCollection) Functions.translate(geomCollection, 1, 1); + GeometryCollection actualGeometry = (GeometryCollection) Functions.translate(geomCollection, 1, 2); assertEquals(wktWriter3D.write(expectedPolygon3D), wktWriter3D.write(actualGeometry.getGeometryN(0).getGeometryN(0).getGeometryN(0))); assertEquals(expectedPolygon.toText(), actualGeometry.getGeometryN(0).getGeometryN(0).getGeometryN(1).toText()); assertEquals(wktWriter3D.write(expectedPoint3D), wktWriter3D.write(actualGeometry.getGeometryN(0).getGeometryN(1))); @@ -846,11 +846,11 @@ public void translateHybridGeomCollectionDeltaZ() { Point point3D = GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1, 1)); LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); Geometry geomCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {multiPolygon, point3D, emptyLineString})}); - Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray(2, 1, 2, 2, 3, 2, 3, 1, 2, 1)); - Polygon expectedPolygon3D = GEOMETRY_FACTORY.createPolygon(coordArray3d(2, 1, 2, 3, 1, 3, 3, 2, 3, 2, 1, 2)); - Point expectedPoint3D = GEOMETRY_FACTORY.createPoint(new Coordinate(2, 2, 2)); + Polygon expectedPolygon = GEOMETRY_FACTORY.createPolygon(coordArray(2, 3, 2, 4, 3, 4, 3, 3, 2, 3)); + Polygon expectedPolygon3D = GEOMETRY_FACTORY.createPolygon(coordArray3d(2, 3, 6, 3, 3, 7, 3, 4, 7, 2, 3, 6)); + Point expectedPoint3D = GEOMETRY_FACTORY.createPoint(new Coordinate(2, 4, 6)); WKTWriter wktWriter3D = new WKTWriter(3); - GeometryCollection actualGeometry = (GeometryCollection) Functions.translate(geomCollection, 1, 1, 1); + GeometryCollection actualGeometry = (GeometryCollection) Functions.translate(geomCollection, 1, 3, 5); assertEquals(wktWriter3D.write(expectedPolygon3D), wktWriter3D.write(actualGeometry.getGeometryN(0).getGeometryN(0).getGeometryN(0))); assertEquals(expectedPolygon.toText(), actualGeometry.getGeometryN(0).getGeometryN(0).getGeometryN(1).toText()); From 19016ae966e68251fac7675f7e43945b50484223 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Mon, 19 Jun 2023 09:14:52 -0700 Subject: [PATCH 31/41] temp affine commit --- .../org/apache/sedona/common/Functions.java | 15 ++ .../apache/sedona/common/utils/GeomUtils.java | 18 +- .../apache/sedona/common/FunctionsTest.java | 90 +++++++- docs/api/flink/Function.md | 64 +----- docs/api/sql/Function.md | 54 ----- docs/community/develop.md | 20 +- .../java/org/apache/sedona/flink/Catalog.java | 1 + .../sedona/flink/expressions/Functions.java | 21 +- python/sedona/sql/st_functions.py | 32 ++- python/tests/sql/test_function.py | 207 +++++++++++------- .../org/apache/sedona/sql/UDF/Catalog.scala | 1 + .../sedona_sql/expressions/Functions.scala | 7 + .../expressions/NullSafeExpressions.scala | 94 ++++++++ .../sedona_sql/expressions/st_functions.scala | 11 + .../sedona/sql/dataFrameAPITestScala.scala | 16 ++ .../apache/sedona/sql/functionTestScala.scala | 12 + 16 files changed, 441 insertions(+), 222 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index 1efd0f474b..7108ad84ee 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -896,6 +896,21 @@ public static Geometry translate(Geometry geometry, double deltaX, double deltaY return geometry; } + public static Geometry affine(Geometry geometry, double a, double b, double d, double e, double xOff, double yOff, double c, + double f, double g, double h, double i, double zOff) { + if (!geometry.isEmpty()) { + GeomUtils.affineGeom(geometry, a, b, d, e, xOff, yOff, c, f, g, h, i, zOff, true); + } + return geometry; + } + + public static Geometry affine(Geometry geometry, double a, double b, double d, double e, double xOff, double yOff) { + if (!geometry.isEmpty()) { + GeomUtils.affineGeom(geometry, a, b, d, e, xOff, yOff, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, false); + } + return geometry; + } + public static Geometry geometricMedian(Geometry geometry, double tolerance, int maxIter, boolean failIfNotConverged) throws Exception { String geometryType = geometry.getGeometryType(); if(!(Geometry.TYPENAME_POINT.equals(geometryType) || Geometry.TYPENAME_MULTIPOINT.equals(geometryType))) { diff --git a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java index 8795f830ac..731843e776 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java @@ -27,7 +27,6 @@ import org.locationtech.jts.operation.polygonize.Polygonizer; import org.locationtech.jts.operation.union.UnaryUnionOp; -import java.awt.*; import java.nio.ByteOrder; import java.util.*; import java.util.List; @@ -461,4 +460,21 @@ public static void translateGeom(Geometry geometry, double deltaX, double deltaY geometry.geometryChanged(); } } + + public static void affineGeom(Geometry geometry, double a, double b, double d, double e, double xOff, double yOff, double c, + double f, double g, double h, double i, double zOff, boolean set3d) { + Coordinate[] coordinates = geometry.getCoordinates(); + for (Coordinate currCoordinate: coordinates) { + double x = currCoordinate.getX(), y = currCoordinate.getY(), z = Double.isNaN(currCoordinate.getZ()) ? 0 : currCoordinate.getZ(); + double newX = a * x + b * y + c * z + xOff; + double newY = d * x + e * y + f * z + yOff; + currCoordinate.setX(newX); + currCoordinate.setY(newY); + if (set3d && !Double.isNaN(currCoordinate.getZ())) { + double newZ = g * x + h * y + i * z + zOff; + currCoordinate.setZ(newZ); + } + } + geometry.geometryChanged(); + } } diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index 3a1745b6b4..ab2980d82e 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -24,7 +24,6 @@ import org.locationtech.jts.io.WKTReader; import org.locationtech.jts.io.WKTWriter; -import javax.sound.sampled.Line; import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -857,4 +856,93 @@ public void translateHybridGeomCollectionDeltaZ() { assertEquals(wktWriter3D.write(expectedPoint3D), wktWriter3D.write(actualGeometry.getGeometryN(0).getGeometryN(1))); assertEquals(emptyLineString.toText(), actualGeometry.getGeometryN(0).getGeometryN(2).toText()); } + + @Test + public void affineEmpty3D() { + LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); + String expected = emptyLineString.toText(); + String actual = Functions.affine(emptyLineString, 1, 1, 2, 3, 5, 6, 2, 3, 4, 4, 5, 6).toText(); + assertEquals(expected, actual); + } + + @Test + public void affineEmpty2D() { + LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); + String expected = emptyLineString.toText(); + String actual = Functions.affine(emptyLineString, 1, 2, 3, 4, 1, 2).toText(); + assertEquals(expected, actual); + } + + @Test + public void affine3DGeom2D() { + LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray(1, 0, 1, 1, 1, 2)); + String expected = GEOMETRY_FACTORY.createLineString(coordArray(6, 18, 7, 11, 8, 14)).toText(); + String actual = Functions.affine(lineString, 1, 1, 2, 3, 5, 6, 2, 3, 4, 4, 5, 6).toText(); + assertEquals(expected, actual); + } + + @Test + public void affine3DGeom3D() { + WKTWriter wktWriter = new WKTWriter(3); +// 2 3 1, 4 5 1, 7 8 2 ,2 3 1 + Polygon polygon3D = GEOMETRY_FACTORY.createPolygon(coordArray3d(2, 3, 1, 4, 5, 1, 7, 8, 2, 2, 3, 1)); + + LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray3d(1, 0, 1, 1, 1, 2, 1, 2, 2)); + String expected = wktWriter.write(GEOMETRY_FACTORY.createLineString(coordArray3d(8, 11, 15, 11, 17, 24, 12, 20, 28))); + String actual = wktWriter.write(Functions.affine(lineString, 1, 1, 2, 3, 5, 6, 2, 3, 4, 4, 5, 6)); + assertEquals(expected, actual); + } + + @Test + public void affine3DHybridGeomCollection() { + Point point3D = GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1, 1)); + Polygon polygon1 = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 0, 2, 1, 1, 2, 2, 1, 2, 2, 0, 2, 1, 0, 2)); + Polygon polygon2 = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 0, 1, 1, 1, 1, 2, 2, 2, 1, 0, 1)); + MultiPolygon multiPolygon = GEOMETRY_FACTORY.createMultiPolygon(new Polygon[] {polygon1, polygon2}); + Geometry geomCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {point3D, multiPolygon})}); + Geometry actualGeomCollection = Functions.affine(geomCollection, 1, 2, 1, 2, 1, 2, 3, 3, 1, 2, 3, 3); + WKTWriter wktWriter3D = new WKTWriter(3); + Point expectedPoint3D = GEOMETRY_FACTORY.createPoint(new Coordinate(7, 8, 9)); + Polygon expectedPolygon1 = GEOMETRY_FACTORY.createPolygon(coordArray3d(8, 9, 10, 10, 11, 12, 11, 12, 13, 9, 10, 11, 8, 9, 10)); + Polygon expectedPolygon2 = GEOMETRY_FACTORY.createPolygon(coordArray3d(5, 6, 7, 7, 8, 9, 13, 14, 15, 5, 6, 7)); + assertEquals(wktWriter3D.write(expectedPoint3D), wktWriter3D.write(actualGeomCollection.getGeometryN(0).getGeometryN(0))); + assertEquals(wktWriter3D.write(expectedPolygon1), wktWriter3D.write(actualGeomCollection.getGeometryN(0).getGeometryN(1).getGeometryN(0))); + assertEquals(wktWriter3D.write(expectedPolygon2), wktWriter3D.write(actualGeomCollection.getGeometryN(0).getGeometryN(1).getGeometryN(1))); + + } + + @Test + public void affine2DGeom3D() { + //1 0 1, 1 1 1, 2 2 2, 1 0 1 + WKTWriter wktWriter = new WKTWriter(3); + LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray3d(1, 0, 1, 1, 1, 2, 1, 2, 2)); + String expected = wktWriter.write(GEOMETRY_FACTORY.createLineString(coordArray3d(6, 8, 1, 7, 11, 2, 8, 14, 2))); + String actual = wktWriter.write(Functions.affine(lineString, 1, 1, 2, 3, 5, 6)); + assertEquals(expected, actual); + } + + @Test + public void affine2DGeom2D() { + LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray(1, 0, 1, 1, 1, 2)); + String expected = GEOMETRY_FACTORY.createLineString(coordArray(6, 8, 7, 11, 8, 14)).toText(); + String actual = Functions.affine(lineString, 1, 1, 2, 3, 5, 6).toText(); + assertEquals(expected, actual); + } + + @Test + public void affine2DHybridGeomCollection() { + Point point3D = GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1)); + Polygon polygon1 = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); + Polygon polygon2 = GEOMETRY_FACTORY.createPolygon(coordArray(3, 4, 3, 5, 3, 7, 10, 7, 3, 4)); + MultiPolygon multiPolygon = GEOMETRY_FACTORY.createMultiPolygon(new Polygon[] {polygon1, polygon2}); + Geometry geomCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {point3D, multiPolygon})}); + Geometry actualGeomCollection = Functions.affine(geomCollection, 1, 2, 1, 2, 1, 2); + Point expectedPoint3D = GEOMETRY_FACTORY.createPoint(new Coordinate(4, 5)); + Polygon expectedPolygon1 = GEOMETRY_FACTORY.createPolygon(coordArray(2, 3, 4, 5, 5, 6, 3, 4, 2, 3)); + Polygon expectedPolygon2 = GEOMETRY_FACTORY.createPolygon(coordArray(12, 13, 14, 15, 18, 19, 25, 26, 12, 13)); + assertEquals(expectedPoint3D.toText(), actualGeomCollection.getGeometryN(0).getGeometryN(0).toText()); + assertEquals(expectedPolygon1.toText(), actualGeomCollection.getGeometryN(0).getGeometryN(1).getGeometryN(0).toText()); + assertEquals(expectedPolygon2.toText(), actualGeomCollection.getGeometryN(0).getGeometryN(1).getGeometryN(1).toText()); + } + } diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index fb68b55c93..11d86ff6e1 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -703,7 +703,7 @@ SELECT ST_NDims(ST_GeomFromEWKT('POINT(1 1 2)')) Output: `3` -Example with x,y coordinate: +Spark SQL example with x,y coordinate: ```sql SELECT ST_NDims(ST_GeomFromText('POINT(1 1)')) @@ -711,39 +711,6 @@ SELECT ST_NDims(ST_GeomFromText('POINT(1 1)')) Output: `2` -## ST_NRings - -Introduction: Returns the number of rings in a Polygon or MultiPolygon. Contrary to ST_NumInteriorRings, -this function also takes into account the number of exterior rings. - -This function returns 0 for an empty Polygon or MultiPolygon. -If the geometry is not a Polygon or MultiPolygon, an IllegalArgument Exception is thrown. - -Format: `ST_NRings(geom: geometry)` - -Since: `1.4.1` - - -Examples: - -Input: `POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))` - -Output: `1` - -Input: `'MULTIPOLYGON (((1 0, 1 6, 6 6, 6 0, 1 0), (2 1, 2 2, 3 2, 3 1, 2 1)), ((10 0, 10 6, 16 6, 16 0, 10 0), (12 1, 12 2, 13 2, 13 1, 12 1)))'` - -Output: `4` - -Input: `'POLYGON EMPTY'` - -Output: `0` - -Input: `'LINESTRING (1 0, 1 1, 2 1)'` - -Output: `Unsupported geometry type: LineString, only Polygon or MultiPolygon geometries are supported.` - - - ## ST_NumGeometries Introduction: Returns the number of Geometries. If geometry is a GEOMETRYCOLLECTION (or MULTI*) return the number of geometries, for single geometries will return 1. @@ -978,13 +945,13 @@ Format: `ST_Transform (A:geometry, SourceCRS:string, TargetCRS:string ,[Optional Since: `v1.2.0` -Example (simple): +Spark SQL example (simple): ```sql SELECT ST_Transform(polygondf.countyshape, 'epsg:4326','epsg:3857') FROM polygondf ``` -Example (with optional parameters): +Spark SQL example (with optional parameters): ```sql SELECT ST_Transform(polygondf.countyshape, 'epsg:4326','epsg:3857', false) FROM polygondf @@ -993,31 +960,6 @@ FROM polygondf !!!note The detailed EPSG information can be searched on [EPSG.io](https://epsg.io/). -## ST_Translate -Introduction: Returns the input geometry with its X, Y and Z coordinates (if present in the geometry) translated by deltaX, deltaY and deltaZ (if specified) - -If the geometry is 2D, and a deltaZ parameter is specified, no change is done to the Z coordinate of the geometry and the resultant geometry is also 2D. - -If the geometry is empty, no change is done to it. - -If the given geometry contains sub-geometries (GEOMETRY COLLECTION, MULTI POLYGON/LINE/POINT), all underlying geometries are individually translated. - -Format: `ST_Translate(geometry: geometry, deltaX: deltaX, deltaY: deltaY, deltaZ: deltaZ)` - -Since: `1.4.1` - -Example: - -Input: `ST_Translate(GEOMETRYCOLLECTION(MULTIPOLYGON (((1 0, 1 1, 2 1, 2 0, 1 0)), ((1 2, 3 4, 3 5, 1 2))), POINT(1, 1, 1), LINESTRING EMPTY), 2, 2, 3)` - -Output: `GEOMETRYCOLLECTION(MULTIPOLYGON (((3 2, 3 3, 4 3, 4 2, 3 2)), ((3 4, 5 6, 5 7, 3 4))), POINT(3, 3, 4), LINESTRING EMPTY)` - -Input: `ST_Translate(POINT(1, 3, 2), 1, 2)` - -Output: `POINT(2, 5, 2)` - - - ## ST_X Introduction: Returns X Coordinate of given Point, null otherwise. diff --git a/docs/api/sql/Function.md b/docs/api/sql/Function.md index 4ef78b148e..05265b5381 100644 --- a/docs/api/sql/Function.md +++ b/docs/api/sql/Function.md @@ -1108,37 +1108,6 @@ SELECT ST_NPoints(polygondf.countyshape) FROM polygondf ``` -## ST_NRings - -Introduction: Returns the number of rings in a Polygon or MultiPolygon. Contrary to ST_NumInteriorRings, -this function also takes into account the number of exterior rings. - -This function returns 0 for an empty Polygon or MultiPolygon. -If the geometry is not a Polygon or MultiPolygon, an IllegalArgument Exception is thrown. - -Format: `ST_NRings(geom: geometry)` - -Since: `1.4.1` - - -Examples: - -Input: `POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))` - -Output: `1` - -Input: `'MULTIPOLYGON (((1 0, 1 6, 6 6, 6 0, 1 0), (2 1, 2 2, 3 2, 3 1, 2 1)), ((10 0, 10 6, 16 6, 16 0, 10 0), (12 1, 12 2, 13 2, 13 1, 12 1)))'` - -Output: `4` - -Input: `'POLYGON EMPTY'` - -Output: `0` - -Input: `'LINESTRING (1 0, 1 1, 2 1)'` - -Output: `Unsupported geometry type: LineString, only Polygon or MultiPolygon geometries are supported.` - ## ST_NumGeometries Introduction: Returns the number of Geometries. If geometry is a GEOMETRYCOLLECTION (or MULTI*) return the number of geometries, for single geometries will return 1. @@ -1598,29 +1567,6 @@ FROM polygondf !!!note The detailed EPSG information can be searched on [EPSG.io](https://epsg.io/). - -## ST_Translate -Introduction: Returns the input geometry with its X, Y and Z coordinates (if present in the geometry) translated by deltaX, deltaY and deltaZ (if specified) - -If the geometry is 2D, and a deltaZ parameter is specified, no change is done to the Z coordinate of the geometry and the resultant geometry is also 2D. - -If the geometry is empty, no change is done to it. -If the given geometry contains sub-geometries (GEOMETRY COLLECTION, MULTI POLYGON/LINE/POINT), all underlying geometries are individually translated. - -Format: `ST_Translate(geometry: geometry, deltaX: deltaX, deltaY: deltaY, deltaZ: deltaZ)` - -Since: `1.4.1` - -Example: - -Input: `ST_Translate(GEOMETRYCOLLECTION(MULTIPOLYGON (((1 0, 1 1, 2 1, 2 0, 1 0)), ((1 2, 3 4, 3 5, 1 2))), POINT(1, 1, 1), LINESTRING EMPTY), 2, 2, 3)` - -Output: `GEOMETRYCOLLECTION(MULTIPOLYGON (((3 2, 3 3, 4 3, 4 2, 3 2)), ((3 4, 5 6, 5 7, 3 4))), POINT(3, 3, 4), LINESTRING EMPTY)` - -Input: `ST_Translate(POINT(1, 3, 2), 1, 2)` - -Output: `POINT(2, 5, 2)` - ## ST_Union Introduction: Return the union of geometry A and B diff --git a/docs/community/develop.md b/docs/community/develop.md index 9483322395..dd8c47d2e9 100644 --- a/docs/community/develop.md +++ b/docs/community/develop.md @@ -4,7 +4,7 @@ ### IDE -We recommend Intellij IDEA with Scala plugin installed. Please make sure that the IDE has JDK 1.8 set as project default. +We recommend Intellij IDEA with Scala plugin installed. ### Import the project @@ -51,10 +51,6 @@ Make sure you reload the POM.xml or reload the maven project. The IDE will ask y #### Run all unit tests In a terminal, go to the Sedona root folder. Run `mvn clean install`. All tests will take more than 15 minutes. To only build the project jars, run `mvn clean install -DskipTests`. -!!!Note - `mvn clean install` will compile Sedona with Spark 3.0 and Scala 2.12. If you have a different version of Spark in $SPARK_HOME, make sure to specify that using -Dspark command line arg. - For example, to compile sedona with Spark 3.4 and Scala 2.12, use: `mvn clean install -Dspark=3.4 -Dscala=2.12` - More details can be found on [Compile Sedona](../../setup/compile/) @@ -82,19 +78,7 @@ Re-run the test case. Do NOT right click the test case to re-run. Instead, click ## Python developers -#### Run all python tests - -To run all Python test cases, follow steps mentioned [here](../../setup/compile/#run-python-test). - -#### Run all python tests in a single test file -To run a particular python test file, specify the path of the .py file to pipenv. - -For example, to run all tests in `test_function.py` located in `python/tests/sql/`, use: `pipenv run pytest tests/sql/test_function.py`. - -#### Run a single test -To run a particular test in a particular .py test file, specify `file_name::class_name::test_name` to the pytest command. - -For example, to run the test on ST_Contains function located in sql/test_predicate.py, use: `pipenv run pytest tests/sql/test_predicate.py::TestPredicate::test_st_contains` +More details to come. ### IDE diff --git a/flink/src/main/java/org/apache/sedona/flink/Catalog.java b/flink/src/main/java/org/apache/sedona/flink/Catalog.java index 8d3559599e..ff3c76a84c 100644 --- a/flink/src/main/java/org/apache/sedona/flink/Catalog.java +++ b/flink/src/main/java/org/apache/sedona/flink/Catalog.java @@ -99,6 +99,7 @@ public static UserDefinedFunction[] getFuncs() { new Functions.ST_Force3D(), new Functions.ST_NRings(), new Functions.ST_Translate(), + new Functions.ST_Affine(), }; } diff --git a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java index 79adcc8d23..52cec2e87f 100644 --- a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java +++ b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java @@ -13,7 +13,6 @@ */ package org.apache.sedona.flink.expressions; -import org.apache.calcite.runtime.Geometries; import org.apache.flink.table.annotation.DataTypeHint; import org.apache.flink.table.functions.ScalarFunction; import org.locationtech.jts.geom.Geometry; @@ -623,4 +622,24 @@ public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.j } } + public static class ST_Affine extends ScalarFunction { + @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) + public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = Geometry.class) Object o, @DataTypeHint("Double") Double a, + @DataTypeHint("Double") Double b, @DataTypeHint("Double") Double d, @DataTypeHint("Double") Double e, @DataTypeHint("Double") Double xOff, @DataTypeHint("Double") Double yOff, @DataTypeHint("Double") Double c, + @DataTypeHint("Double") Double f, @DataTypeHint("Double") Double g, @DataTypeHint("Double") Double h, @DataTypeHint("Double") Double i, + @DataTypeHint("Double") Double zOff) { + Geometry geometry = (Geometry) o; + return org.apache.sedona.common.Functions.affine(geometry, a, b, d, e, xOff, yOff, c, f, g, h, i, zOff); + } + + @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) + public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o, @DataTypeHint("Double") Double a, + @DataTypeHint("Double") Double b, @DataTypeHint("Double") Double d, @DataTypeHint("Double") Double e, + @DataTypeHint("Double") Double xOff, @DataTypeHint("Double") Double yOff) { + Geometry geometry = (Geometry) o; + return org.apache.sedona.common.Functions.affine(geometry, a, b, d, e, xOff, yOff); + } + + } + } diff --git a/python/sedona/sql/st_functions.py b/python/sedona/sql/st_functions.py index 32a2f2728c..6dc5451909 100644 --- a/python/sedona/sql/st_functions.py +++ b/python/sedona/sql/st_functions.py @@ -111,7 +111,8 @@ "ST_NumPoints", "ST_Force3D", "ST_NRings", - "ST_Translate" + "ST_Translate", + "ST_Affine" ] @@ -1277,3 +1278,32 @@ def ST_Translate(geometry: ColumnOrName, deltaX: Union[ColumnOrName, float], del args = (geometry, deltaX, deltaY, deltaZ) return _call_st_function("ST_Translate", args) +@validate_argument_types +def ST_Affine(geometry: ColumnOrName, a: Union[ColumnOrName, float], b: Union[ColumnOrName, float], d: Union[ColumnOrName, float], + e: Union[ColumnOrName, float], xOff: Union[ColumnOrName, float], yOff: Union[ColumnOrName, float], c: Optional[Union[ColumnOrName, float]] = 0.0, f: Optional[Union[ColumnOrName, float]] = 0.0, + g: Optional[Union[ColumnOrName, float]] = 0.0, h: Optional[Union[ColumnOrName, float]] = 0.0, + i: Optional[Union[ColumnOrName, float]] = 0.0, zOff: Optional[Union[ColumnOrName, float]] = 0.0) -> Column: + """ + Apply a 3D/2D affine tranformation to the given geometry + x = a * x + b * y + c * z + xOff | x = a * x + b * y + xOff + y = d * x + e * y + f * z + yOff | y = d * x + e * y + yOff + z = g * x + h * y + i * z + zOff + :param geometry: Geometry to apply affine transformation to + :param a: + :param b: + :param c: Default 0.0 + :param d: + :param e: + :param f: Default 0.0 + :param g: Default 0.0 + :param h: Default 0.0 + :param i: Default 0.0 + :param xOff: + :param yOff: + :param zOff: Default 0.0 + :return: Geometry with affine transformation applied + """ + args = (geometry, a, b, c, d, e, f, g, h, i, xOff, yOff, zOff) + return _call_st_function("ST_Affine", args) + + diff --git a/python/tests/sql/test_function.py b/python/tests/sql/test_function.py index 6897fb8ee5..df519b1203 100644 --- a/python/tests/sql/test_function.py +++ b/python/tests/sql/test_function.py @@ -31,7 +31,6 @@ class TestPredicateJoin(TestBase): - geo_schema = StructType( [StructField("geom", GeometryType(), False)] ) @@ -184,29 +183,34 @@ def test_st_transform(self): polygon_df = self.spark.sql("select ST_GeomFromWKT(polygontable._c0) as countyshape from polygontable") polygon_df.createOrReplaceTempView("polygondf") polygon_df.show() - function_df = self.spark.sql("select ST_Transform(ST_FlipCoordinates(polygondf.countyshape), 'epsg:4326','epsg:3857', false) from polygondf") + function_df = self.spark.sql( + "select ST_Transform(ST_FlipCoordinates(polygondf.countyshape), 'epsg:4326','epsg:3857', false) from polygondf") function_df.show() def test_st_intersection_intersects_but_not_contains(self): - test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON((1 1, 8 1, 8 8, 1 8, 1 1))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b") + test_table = self.spark.sql( + "select ST_GeomFromWKT('POLYGON((1 1, 8 1, 8 8, 1 8, 1 1))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b") test_table.createOrReplaceTempView("testtable") intersect = self.spark.sql("select ST_Intersection(a,b) from testtable") assert intersect.take(1)[0][0].wkt == "POLYGON ((2 8, 8 8, 8 2, 2 2, 2 8))" def test_st_intersection_intersects_but_left_contains_right(self): - test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON((1 1, 1 5, 5 5, 1 1))') as a,ST_GeomFromWKT('POLYGON((2 2, 2 3, 3 3, 2 2))') as b") + test_table = self.spark.sql( + "select ST_GeomFromWKT('POLYGON((1 1, 1 5, 5 5, 1 1))') as a,ST_GeomFromWKT('POLYGON((2 2, 2 3, 3 3, 2 2))') as b") test_table.createOrReplaceTempView("testtable") intersects = self.spark.sql("select ST_Intersection(a,b) from testtable") assert intersects.take(1)[0][0].wkt == "POLYGON ((2 2, 2 3, 3 3, 2 2))" def test_st_intersection_intersects_but_right_contains_left(self): - test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON((2 2, 2 3, 3 3, 2 2))') as a,ST_GeomFromWKT('POLYGON((1 1, 1 5, 5 5, 1 1))') as b") + test_table = self.spark.sql( + "select ST_GeomFromWKT('POLYGON((2 2, 2 3, 3 3, 2 2))') as a,ST_GeomFromWKT('POLYGON((1 1, 1 5, 5 5, 1 1))') as b") test_table.createOrReplaceTempView("testtable") intersects = self.spark.sql("select ST_Intersection(a,b) from testtable") assert intersects.take(1)[0][0].wkt == "POLYGON ((2 2, 2 3, 3 3, 2 2))" def test_st_intersection_not_intersects(self): - test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON((40 21, 40 22, 40 23, 40 21))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b") + test_table = self.spark.sql( + "select ST_GeomFromWKT('POLYGON((40 21, 40 22, 40 23, 40 21))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b") test_table.createOrReplaceTempView("testtable") intersects = self.spark.sql("select ST_Intersection(a,b) from testtable") assert intersects.take(1)[0][0].wkt == "POLYGON EMPTY" @@ -255,16 +259,17 @@ def test_st_as_text(self): wkt_df = self.spark.sql("select ST_AsText(countyshape) as wkt from polygondf") assert polygon_df.take(1)[0]["countyshape"].wkt == loads(wkt_df.take(1)[0]["wkt"]).wkt - def test_st_astext_3d(self): input_df = self.spark.createDataFrame([ ("Point(21 52 87)",), ("Polygon((0 0 1, 0 1 1, 1 1 1, 1 0 1, 0 0 1))",), ("Linestring(0 0 1, 1 1 2, 1 0 3)",), ("MULTIPOINT ((10 40 66), (40 30 77), (20 20 88), (30 10 99))",), - ("MULTIPOLYGON (((30 20 11, 45 40 11, 10 40 11, 30 20 11)), ((15 5 11, 40 10 11, 10 20 11, 5 10 11, 15 5 11)))",), + ( + "MULTIPOLYGON (((30 20 11, 45 40 11, 10 40 11, 30 20 11)), ((15 5 11, 40 10 11, 10 20 11, 5 10 11, 15 5 11)))",), ("MULTILINESTRING ((10 10 11, 20 20 11, 10 40 11), (40 40 11, 30 30 11, 40 20 11, 30 10 11))",), - ("MULTIPOLYGON (((40 40 11, 20 45 11, 45 30 11, 40 40 11)), ((20 35 11, 10 30 11, 10 10 11, 30 5 11, 45 20 11, 20 35 11), (30 20 11, 20 15 11, 20 25 11, 30 20 11)))",), + ( + "MULTIPOLYGON (((40 40 11, 20 45 11, 45 30 11, 40 40 11)), ((20 35 11, 10 30 11, 10 10 11, 30 5 11, 45 20 11, 20 35 11), (30 20 11, 20 15 11, 20 25 11, 30 20 11)))",), ("POLYGON((0 0 11, 0 5 11, 5 5 11, 5 0 11, 0 0 11), (1 1 11, 2 1 11, 2 2 11, 1 2 11, 1 1 11))",), ], ["wkt"]) @@ -285,64 +290,78 @@ def test_st_as_text_3d(self): assert polygon_df.take(1)[0]["countyshape"].wkt == loads(wkt_df.take(1)[0]["wkt"]).wkt def test_st_n_points(self): - test = self.spark.sql("SELECT ST_NPoints(ST_GeomFromText('LINESTRING(77.29 29.07,77.42 29.26,77.27 29.31,77.29 29.07)'))") + test = self.spark.sql( + "SELECT ST_NPoints(ST_GeomFromText('LINESTRING(77.29 29.07,77.42 29.26,77.27 29.31,77.29 29.07)'))") def test_st_geometry_type(self): - test = self.spark.sql("SELECT ST_GeometryType(ST_GeomFromText('LINESTRING(77.29 29.07,77.42 29.26,77.27 29.31,77.29 29.07)'))") + test = self.spark.sql( + "SELECT ST_GeometryType(ST_GeomFromText('LINESTRING(77.29 29.07,77.42 29.26,77.27 29.31,77.29 29.07)'))") def test_st_difference_right_overlaps_left(self): - test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((0 -4, 4 -4, 4 4, 0 4, 0 -4))') as b") + test_table = self.spark.sql( + "select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((0 -4, 4 -4, 4 4, 0 4, 0 -4))') as b") test_table.createOrReplaceTempView("test_diff") diff = self.spark.sql("select ST_Difference(a,b) from test_diff") assert diff.take(1)[0][0].wkt == "POLYGON ((0 -3, -3 -3, -3 3, 0 3, 0 -3))" def test_st_difference_right_not_overlaps_left(self): - test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((5 -3, 7 -3, 7 -1, 5 -1, 5 -3))') as b") + test_table = self.spark.sql( + "select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((5 -3, 7 -3, 7 -1, 5 -1, 5 -3))') as b") test_table.createOrReplaceTempView("test_diff") diff = self.spark.sql("select ST_Difference(a,b) from test_diff") assert diff.take(1)[0][0].wkt == "POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))" def test_st_difference_left_contains_right(self): - test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as b") + test_table = self.spark.sql( + "select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as b") test_table.createOrReplaceTempView("test_diff") diff = self.spark.sql("select ST_Difference(a,b) from test_diff") assert diff.take(1)[0][0].wkt == "POLYGON ((-3 -3, -3 3, 3 3, 3 -3, -3 -3), (-1 -1, 1 -1, 1 1, -1 1, -1 -1))" def test_st_difference_right_not_overlaps_left(self): - test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as a,ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as b") + test_table = self.spark.sql( + "select ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as a,ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as b") test_table.createOrReplaceTempView("test_diff") diff = self.spark.sql("select ST_Difference(a,b) from test_diff") assert diff.take(1)[0][0].wkt == "POLYGON EMPTY" def test_st_sym_difference_part_of_right_overlaps_left(self): - test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as a,ST_GeomFromWKT('POLYGON ((0 -2, 2 -2, 2 0, 0 0, 0 -2))') as b") + test_table = self.spark.sql( + "select ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as a,ST_GeomFromWKT('POLYGON ((0 -2, 2 -2, 2 0, 0 0, 0 -2))') as b") test_table.createOrReplaceTempView("test_sym_diff") diff = self.spark.sql("select ST_SymDifference(a,b) from test_sym_diff") - assert diff.take(1)[0][0].wkt == "MULTIPOLYGON (((0 -1, -1 -1, -1 1, 1 1, 1 0, 0 0, 0 -1)), ((0 -1, 1 -1, 1 0, 2 0, 2 -2, 0 -2, 0 -1)))" + assert diff.take(1)[0][ + 0].wkt == "MULTIPOLYGON (((0 -1, -1 -1, -1 1, 1 1, 1 0, 0 0, 0 -1)), ((0 -1, 1 -1, 1 0, 2 0, 2 -2, 0 -2, 0 -1)))" def test_st_sym_difference_not_overlaps_left(self): - test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((5 -3, 7 -3, 7 -1, 5 -1, 5 -3))') as b") + test_table = self.spark.sql( + "select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((5 -3, 7 -3, 7 -1, 5 -1, 5 -3))') as b") test_table.createOrReplaceTempView("test_sym_diff") diff = self.spark.sql("select ST_SymDifference(a,b) from test_sym_diff") - assert diff.take(1)[0][0].wkt == "MULTIPOLYGON (((-3 -3, -3 3, 3 3, 3 -3, -3 -3)), ((5 -3, 5 -1, 7 -1, 7 -3, 5 -3)))" + assert diff.take(1)[0][ + 0].wkt == "MULTIPOLYGON (((-3 -3, -3 3, 3 3, 3 -3, -3 -3)), ((5 -3, 5 -1, 7 -1, 7 -3, 5 -3)))" def test_st_sym_difference_contains(self): - test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as b") + test_table = self.spark.sql( + "select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as b") test_table.createOrReplaceTempView("test_sym_diff") diff = self.spark.sql("select ST_SymDifference(a,b) from test_sym_diff") assert diff.take(1)[0][0].wkt == "POLYGON ((-3 -3, -3 3, 3 3, 3 -3, -3 -3), (-1 -1, 1 -1, 1 1, -1 1, -1 -1))" def test_st_union_part_of_right_overlaps_left(self): - test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a, ST_GeomFromWKT('POLYGON ((-2 1, 2 1, 2 4, -2 4, -2 1))') as b") + test_table = self.spark.sql( + "select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a, ST_GeomFromWKT('POLYGON ((-2 1, 2 1, 2 4, -2 4, -2 1))') as b") test_table.createOrReplaceTempView("test_union") union = self.spark.sql("select ST_Union(a,b) from test_union") assert union.take(1)[0][0].wkt == "POLYGON ((2 3, 3 3, 3 -3, -3 -3, -3 3, -2 3, -2 4, 2 4, 2 3))" def test_st_union_not_overlaps_left(self): - test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((5 -3, 7 -3, 7 -1, 5 -1, 5 -3))') as b") + test_table = self.spark.sql( + "select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((5 -3, 7 -3, 7 -1, 5 -1, 5 -3))') as b") test_table.createOrReplaceTempView("test_union") union = self.spark.sql("select ST_Union(a,b) from test_union") - assert union.take(1)[0][0].wkt == "MULTIPOLYGON (((-3 -3, -3 3, 3 3, 3 -3, -3 -3)), ((5 -3, 5 -1, 7 -1, 7 -3, 5 -3)))" + assert union.take(1)[0][ + 0].wkt == "MULTIPOLYGON (((-3 -3, -3 3, 3 3, 3 -3, -3 -3)), ((5 -3, 5 -1, 7 -1, 7 -3, 5 -3)))" def test_st_azimuth(self): sample_points = create_sample_points(20) @@ -385,11 +404,11 @@ def test_st_x(self): linestrings = linestring_df.selectExpr("ST_X(geom) as x").filter("x IS NOT NULL") - assert([point[0] for point in points] == [-71.064544, -88.331492, 88.331492, 1.0453, 32.324142]) + assert ([point[0] for point in points] == [-71.064544, -88.331492, 88.331492, 1.0453, 32.324142]) - assert(not linestrings.count()) + assert (not linestrings.count()) - assert(not polygons.count()) + assert (not polygons.count()) def test_st_y(self): point_df = create_sample_points_df(self.spark, 5) @@ -403,11 +422,11 @@ def test_st_y(self): linestrings = linestring_df.selectExpr("ST_Y(geom) as y").filter("y IS NOT NULL") - assert([point[0] for point in points] == [42.28787, 32.324142, 32.324142, 5.3324324, -88.331492]) + assert ([point[0] for point in points] == [42.28787, 32.324142, 32.324142, 5.3324324, -88.331492]) - assert(not linestrings.count()) + assert (not linestrings.count()) - assert(not polygons.count()) + assert (not polygons.count()) def test_st_z(self): point_df = self.spark.sql( @@ -427,27 +446,27 @@ def test_st_z(self): linestrings = linestring_df.selectExpr("ST_Z(geom) as z").filter("z IS NOT NULL") - assert([point[0] for point in points] == [3.3]) + assert ([point[0] for point in points] == [3.3]) - assert(not linestrings.count()) + assert (not linestrings.count()) - assert(not polygons.count()) + assert (not polygons.count()) def test_st_z_max(self): linestring_df = self.spark.sql("SELECT ST_GeomFromWKT('LINESTRING Z (0 0 1, 0 1 2)') as geom") linestring_row = [lnstr_row[0] for lnstr_row in linestring_df.selectExpr("ST_ZMax(geom)").collect()] - assert(linestring_row == [2.0]) + assert (linestring_row == [2.0]) def test_st_z_min(self): - linestring_df = self.spark.sql("SELECT ST_GeomFromWKT('POLYGON Z ((0 0 2, 0 1 1, 1 1 2, 1 0 2, 0 0 2))') as geom") + linestring_df = self.spark.sql( + "SELECT ST_GeomFromWKT('POLYGON Z ((0 0 2, 0 1 1, 1 1 2, 1 0 2, 0 0 2))') as geom") linestring_row = [lnstr_row[0] for lnstr_row in linestring_df.selectExpr("ST_ZMin(geom)").collect()] - assert(linestring_row == [1.0]) + assert (linestring_row == [1.0]) def test_st_n_dims(self): point_df = self.spark.sql("SELECT ST_GeomFromWKT('POINT(1 1 2)') as geom") point_row = [pt_row[0] for pt_row in point_df.selectExpr("ST_NDims(geom)").collect()] - assert(point_row == [3]) - + assert (point_row == [3]) def test_st_start_point(self): @@ -469,11 +488,11 @@ def test_st_start_point(self): linestrings = linestring_df.selectExpr("ST_StartPoint(geom) as geom").filter("geom IS NOT NULL") - assert([line[0] for line in linestrings.collect()] == [wkt.loads(el) for el in expected_points]) + assert ([line[0] for line in linestrings.collect()] == [wkt.loads(el) for el in expected_points]) - assert(not points.count()) + assert (not points.count()) - assert(not polygons.count()) + assert (not polygons.count()) def test_st_end_point(self): linestring_dataframe = create_sample_lines_df(self.spark, 5) @@ -493,10 +512,10 @@ def test_st_end_point(self): empty_dataframe = other_geometry_dataframe.selectExpr("ST_EndPoint(geom) as geom"). \ filter("geom IS NOT NULL") - assert([wkt_row[0] - for wkt_row in point_data_frame.selectExpr("ST_AsText(geom)").collect()] == expected_ending_points) + assert ([wkt_row[0] + for wkt_row in point_data_frame.selectExpr("ST_AsText(geom)").collect()] == expected_ending_points) - assert(empty_dataframe.count() == 0) + assert (empty_dataframe.count() == 0) def test_st_boundary(self): wkt_list = [ @@ -519,7 +538,7 @@ def test_st_boundary(self): boundary_table = geometry_table.selectExpr("ST_Boundary(geom) as geom") boundary_wkt = [wkt_row[0] for wkt_row in boundary_table.selectExpr("ST_AsText(geom)").collect()] - assert(boundary_wkt == [ + assert (boundary_wkt == [ "MULTIPOINT ((1 1), (-1 1))", "MULTIPOINT ((100 150), (160 170))", "MULTILINESTRING ((10 130, 50 190, 110 190, 140 150, 150 80, 100 10, 20 40, 10 130), (70 40, 100 50, 120 80, 80 110, 50 90, 70 40))", @@ -542,33 +561,34 @@ def test_st_exterior_ring(self): linestring_wkt = [wkt_row[0] for wkt_row in linestring_df.selectExpr("ST_AsText(geom)").collect()] - assert(linestring_wkt == ["LINESTRING (0 0, 0 1, 1 1, 1 0, 0 0)", "LINESTRING (0 0, 1 1, 1 2, 1 1, 0 0)"]) + assert (linestring_wkt == ["LINESTRING (0 0, 0 1, 1 1, 1 0, 0 0)", "LINESTRING (0 0, 1 1, 1 2, 1 1, 0 0)"]) - assert(not empty_df.count()) + assert (not empty_df.count()) def test_st_geometry_n(self): data_frame = self.__wkt_list_to_data_frame(["MULTIPOINT((1 2), (3 4), (5 6), (8 9))"]) wkts = [data_frame.selectExpr(f"ST_GeometryN(geom, {i}) as geom").selectExpr("st_asText(geom)").collect()[0][0] for i in range(0, 4)] - assert(wkts == ["POINT (1 2)", "POINT (3 4)", "POINT (5 6)", "POINT (8 9)"]) + assert (wkts == ["POINT (1 2)", "POINT (3 4)", "POINT (5 6)", "POINT (8 9)"]) def test_st_interior_ring_n(self): polygon_df = self.__wkt_list_to_data_frame( - ["POLYGON((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1), (1 3, 2 3, 2 4, 1 4, 1 3), (3 3, 4 3, 4 4, 3 4, 3 3))"] + [ + "POLYGON((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1), (1 3, 2 3, 2 4, 1 4, 1 3), (3 3, 4 3, 4 4, 3 4, 3 3))"] ) other_geometry = create_sample_points_df(self.spark, 5).union(create_sample_lines_df(self.spark, 5)) wholes = [polygon_df.selectExpr(f"ST_InteriorRingN(geom, {i}) as geom"). - selectExpr("ST_AsText(geom)").collect()[0][0] + selectExpr("ST_AsText(geom)").collect()[0][0] for i in range(3)] empty_df = other_geometry.selectExpr("ST_InteriorRingN(geom, 1) as geom").filter("geom IS NOT NULL") - assert(not empty_df.count()) - assert(wholes == ["LINESTRING (1 1, 2 1, 2 2, 1 2, 1 1)", - "LINESTRING (1 3, 2 3, 2 4, 1 4, 1 3)", - "LINESTRING (3 3, 4 3, 4 4, 3 4, 3 3)"]) + assert (not empty_df.count()) + assert (wholes == ["LINESTRING (1 1, 2 1, 2 2, 1 2, 1 1)", + "LINESTRING (1 3, 2 3, 2 4, 1 4, 1 3)", + "LINESTRING (3 3, 4 3, 4 4, 3 4, 3 3)"]) def test_st_dumps(self): expected_geometries = [ @@ -598,14 +618,14 @@ def test_st_dumps(self): dumped_geometries = geometry_df.selectExpr("ST_Dump(geom) as geom") - assert(dumped_geometries.select(explode(col("geom"))).count() == 14) + assert (dumped_geometries.select(explode(col("geom"))).count() == 14) collected_geometries = dumped_geometries \ .select(explode(col("geom")).alias("geom")) \ .selectExpr("ST_AsText(geom) as geom") \ .collect() - assert([geom_row[0] for geom_row in collected_geometries] == expected_geometries) + assert ([geom_row[0] for geom_row in collected_geometries] == expected_geometries) def test_st_dump_points(self): expected_points = [ @@ -625,10 +645,10 @@ def test_st_dump_points(self): dumped_points = geometry_df.selectExpr("ST_DumpPoints(geom) as geom") \ .select(explode(col("geom")).alias("geom")) - assert(dumped_points.count() == 10) + assert (dumped_points.count() == 10) collected_points = [geom_row[0] for geom_row in dumped_points.selectExpr("ST_AsText(geom)").collect()] - assert(collected_points == expected_points) + assert (collected_points == expected_points) def test_st_is_closed(self): expected_result = [ @@ -653,13 +673,14 @@ def test_st_is_closed(self): (7, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))"), (8, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10))"), (9, "MULTILINESTRING ((10 10, 20 20, 10 40), (40 40, 30 30, 40 20, 30 10))"), - (10, "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))") + (10, + "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))") ] geometry_df = self.__wkt_pair_list_with_index_to_data_frame(geometry_list) is_closed = geometry_df.selectExpr("index", "ST_IsClosed(geom)").collect() is_closed_collected = [[*row] for row in is_closed] - assert(is_closed_collected == expected_result) + assert (is_closed_collected == expected_result) def test_num_interior_ring(self): geometries = [ @@ -672,14 +693,15 @@ def test_num_interior_ring(self): (7, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))"), (8, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10))"), (9, "MULTILINESTRING ((10 10, 20 20, 10 40), (40 40, 30 30, 40 20, 30 10))"), - (10, "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))"), + (10, + "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))"), (11, "POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))")] geometry_df = self.__wkt_pair_list_with_index_to_data_frame(geometries) number_of_interior_rings = geometry_df.selectExpr("index", "ST_NumInteriorRings(geom) as num") collected_interior_rings = [[*row] for row in number_of_interior_rings.filter("num is not null").collect()] - assert(collected_interior_rings == [[2, 0], [11, 1]]) + assert (collected_interior_rings == [[2, 0], [11, 1]]) def test_st_add_point(self): geometry = [ @@ -692,7 +714,9 @@ def test_st_add_point(self): ("MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))", "Point(21 52)"), ("MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10))", "Point(21 52)"), ("MULTILINESTRING ((10 10, 20 20, 10 40), (40 40, 30 30, 40 20, 30 10))", "Point(21 52)"), - ("GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))", "Point(21 52)"), + ( + "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))", + "Point(21 52)"), ("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))", "Point(21 52)") ] geometry_df = self.__wkt_pairs_to_data_frame(geometry) @@ -700,7 +724,7 @@ def test_st_add_point(self): collected_geometries = [ row[0] for row in modified_geometries.filter("geom is not null").selectExpr("ST_AsText(geom)").collect() ] - assert(collected_geometries[0] == "LINESTRING (0 0, 1 1, 1 0, 21 52)") + assert (collected_geometries[0] == "LINESTRING (0 0, 1 1, 1 0, 21 52)") def test_st_remove_point(self): result_and_expected = [ @@ -711,11 +735,13 @@ def test_st_remove_point(self): [self.calculate_st_remove("POINT(0 1)", 3), None], [self.calculate_st_remove("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))", 3), None], [self.calculate_st_remove("GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40))", 0), None], - [self.calculate_st_remove("MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), ((15 5, 40 10, 10 20, 5 10, 15 5)))", 3), None], - [self.calculate_st_remove("MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))", 3), None] + [self.calculate_st_remove( + "MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), ((15 5, 40 10, 10 20, 5 10, 15 5)))", 3), None], + [self.calculate_st_remove( + "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))", 3), None] ] for actual, expected in result_and_expected: - assert(actual == expected) + assert (actual == expected) def test_st_is_ring(self): result_and_expected = [ @@ -726,7 +752,7 @@ def test_st_is_ring(self): [self.calculate_st_is_ring("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))"), None], ] for actual, expected in result_and_expected: - assert(actual == expected) + assert (actual == expected) def test_st_subdivide(self): # Given @@ -797,12 +823,11 @@ def test_st_make_polygon(self): geom_poly = geometry_df.withColumn("polygon", expr("ST_MakePolygon(geom)")) # Then only based on closed linestring geom is created - geom_poly.filter("polygon IS NOT NULL").selectExpr("ST_AsText(polygon)", "expected").\ + geom_poly.filter("polygon IS NOT NULL").selectExpr("ST_AsText(polygon)", "expected"). \ show() - result = geom_poly.filter("polygon IS NOT NULL").selectExpr("ST_AsText(polygon)", "expected").\ + result = geom_poly.filter("polygon IS NOT NULL").selectExpr("ST_AsText(polygon)", "expected"). \ collect() - assert result.__len__() == 1 for actual, expected in result: @@ -819,7 +844,7 @@ def test_st_geohash(self): ).select(expr("St_GeomFromText(_1)").alias("geom"), col("_2").alias("expected_hash")) # When - geohash_df = geometry_df.withColumn("geohash", expr("ST_GeoHash(geom, 10)")).\ + geohash_df = geometry_df.withColumn("geohash", expr("ST_GeoHash(geom, 10)")). \ select("geohash", "expected_hash") # Then @@ -886,7 +911,7 @@ def test_st_collect_on_array_type(self): geometry_df_collected = geometry_df.withColumn("collected", expr("ST_Collect(geom)")) # then result should be as expected - assert(set([el[0] for el in geometry_df_collected.selectExpr("ST_AsText(collected)").collect()]) == { + assert (set([el[0] for el in geometry_df_collected.selectExpr("ST_AsText(collected)").collect()]) == { "MULTILINESTRING ((1 2, 3 4), (3 4, 4 5))", "MULTIPOINT ((1 2), (-2 3))", "MULTIPOLYGON (((1 2, 1 4, 3 4, 3 2, 1 2)), ((0.5 0.5, 5 0, 5 5, 0 5, 0.5 0.5)))" @@ -904,7 +929,7 @@ def test_st_collect_on_multiple_columns(self): geometry_df_collected = geometry_df.withColumn("collected", expr("ST_Collect(geom_left, geom_right)")) # then result should be calculated - assert(set([el[0] for el in geometry_df_collected.selectExpr("ST_AsText(collected)").collect()]) == { + assert (set([el[0] for el in geometry_df_collected.selectExpr("ST_AsText(collected)").collect()]) == { "MULTILINESTRING ((1 2, 3 4), (3 4, 4 5))", "MULTIPOINT ((1 2), (-2 3))", "MULTIPOLYGON (((1 2, 1 4, 3 4, 3 2, 1 2)), ((0.5 0.5, 5 0, 5 5, 0 5, 0.5 0.5)))" @@ -959,30 +984,34 @@ def calculate_st_remove(self, wkt, index): return geometry_collected[0][0] if geometry_collected.__len__() != 0 else None def __wkt_pairs_to_data_frame(self, wkt_list: List) -> DataFrame: - return self.spark.createDataFrame([[wkt.loads(wkt_a), wkt.loads(wkt_b)] for wkt_a, wkt_b in wkt_list], self.geo_pair_schema) + return self.spark.createDataFrame([[wkt.loads(wkt_a), wkt.loads(wkt_b)] for wkt_a, wkt_b in wkt_list], + self.geo_pair_schema) def __wkt_list_to_data_frame(self, wkt_list: List) -> DataFrame: return self.spark.createDataFrame([[wkt.loads(given_wkt)] for given_wkt in wkt_list], self.geo_schema) def __wkt_pair_list_with_index_to_data_frame(self, wkt_list: List) -> DataFrame: - return self.spark.createDataFrame([[index, wkt.loads(given_wkt)] for index, given_wkt in wkt_list], self.geo_schema_with_index) + return self.spark.createDataFrame([[index, wkt.loads(given_wkt)] for index, given_wkt in wkt_list], + self.geo_schema_with_index) def test_st_pointonsurface(self): tests1 = { - "'POINT(0 5)'":"POINT (0 5)", - "'LINESTRING(0 5, 0 10)'":"POINT (0 5)", - "'POLYGON((0 0, 0 5, 5 5, 5 0, 0 0))'":"POINT (2.5 2.5)", - "'LINESTRING(0 5 1, 0 0 1, 0 10 2)'":"POINT Z(0 0 1)" + "'POINT(0 5)'": "POINT (0 5)", + "'LINESTRING(0 5, 0 10)'": "POINT (0 5)", + "'POLYGON((0 0, 0 5, 5 5, 5 0, 0 0))'": "POINT (2.5 2.5)", + "'LINESTRING(0 5 1, 0 0 1, 0 10 2)'": "POINT Z(0 0 1)" } for input_geom, expected_geom in tests1.items(): - pointOnSurface = self.spark.sql("select ST_AsText(ST_PointOnSurface(ST_GeomFromText({})))".format(input_geom)) + pointOnSurface = self.spark.sql( + "select ST_AsText(ST_PointOnSurface(ST_GeomFromText({})))".format(input_geom)) assert pointOnSurface.take(1)[0][0] == expected_geom - tests2 = { "'LINESTRING(0 5 1, 0 0 1, 0 10 2)'":"POINT Z(0 0 1)" } + tests2 = {"'LINESTRING(0 5 1, 0 0 1, 0 10 2)'": "POINT Z(0 0 1)"} for input_geom, expected_geom in tests2.items(): - pointOnSurface = self.spark.sql("select ST_AsEWKT(ST_PointOnSurface(ST_GeomFromWKT({})))".format(input_geom)) + pointOnSurface = self.spark.sql( + "select ST_AsEWKT(ST_PointOnSurface(ST_GeomFromWKT({})))".format(input_geom)) assert pointOnSurface.take(1)[0][0] == expected_geom def test_st_pointn(self): @@ -1058,7 +1087,8 @@ def test_st_line_from_multi_point(self): "LINESTRING Z(10 40 66, 40 30 77, 20 20 88, 30 10 99)" } for input_geom, expected_geom in test_cases.items(): - line_geometry = self.spark.sql("select ST_AsText(ST_LineFromMultiPoint(ST_GeomFromText({})))".format(input_geom)) + line_geometry = self.spark.sql( + "select ST_AsText(ST_LineFromMultiPoint(ST_GeomFromText({})))".format(input_geom)) assert line_geometry.take(1)[0][0] == expected_geom def test_st_s2_cell_ids(self): @@ -1094,7 +1124,14 @@ def test_nRings(self): def test_translate(self): expected = "POLYGON ((3 5, 3 6, 4 6, 4 5, 3 5))" - actualDf = self.spark.sql("SELECT ST_Translate(ST_GeomFromText('POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))'), 2, 5) AS geom") - actual = actualDf.selectExpr("ST_AsText(geom)").take(1)[0][0] + actual_df = self.spark.sql( + "SELECT ST_Translate(ST_GeomFromText('POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))'), 2, 5) AS geom") + actual = actual_df.selectExpr("ST_AsText(geom)").take(1)[0][0] assert expected == actual + def test_affine(self): + expected = "POLYGON Z((2 3 1, 4 5 1, 7 8 2 ,2 3 1))" + actual_df = self.spark.sql("SELECT ST_Affine(ST_GeomFromText('POLYGON ((1 0 1, 1 1 1, 2 2 2, 1 0 1))'), 1, 2, " + "1, 2, 1, 2) AS geom") + actual = actual_df.selectExpr("ST_AsText(geom)").take(1)[0][0] + assert expected == actual diff --git a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala index cbb1319307..15331d704e 100644 --- a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala +++ b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala @@ -151,6 +151,7 @@ object Catalog { function[ST_Force3D](0.0), function[ST_NRings](), function[ST_Translate](0.0), + function[ST_Affine](0.0, 0.0, 0.0, 0.0, 0.0, 0.0), // Expression for rasters function[RS_NormalizedDifference](), function[RS_Mean](), diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala index 41052806dd..ae7ff02304 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala @@ -1010,3 +1010,10 @@ case class ST_Translate(inputExpressions: Seq[Expression]) } } +case class ST_Affine(inputExpressions: Seq[Expression]) + extends InferredNaryExpression(Functions.affine) with FoldableExpression { + protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { + copy(inputExpressions = newChildren) + } +} + diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/NullSafeExpressions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/NullSafeExpressions.scala index f526baf0cf..604d1ac53e 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/NullSafeExpressions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/NullSafeExpressions.scala @@ -364,3 +364,97 @@ abstract class InferredQuarternaryExpression[A1: InferrableType, A2: InferrableT } } } + +abstract class InferredNaryExpression[A1: InferrableType, A2: InferrableType, A3: InferrableType, A4: InferrableType, + A5: InferrableType, A6: InferrableType, A7: InferrableType, A8: InferrableType, A9: InferrableType, + A10: InferrableType, A11: InferrableType, A12: InferrableType, A13: InferrableType, R: InferrableType] + (f: (A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13) => R) + (implicit val a1Tag: TypeTag[A1], implicit val a2Tag: TypeTag[A2], implicit val a3Tag: TypeTag[A3], implicit val a4Tag: TypeTag[A4], + implicit val a5Tag: TypeTag[A5], implicit val a6Tag: TypeTag[A6], implicit val a7Tag: TypeTag[A7], + implicit val a8Tag: TypeTag[A8], implicit val a9Tag: TypeTag[A9], implicit val a10Tag: TypeTag[A10], + implicit val a11Tag: TypeTag[A11], implicit val a12Tag: TypeTag[A12], implicit val a13Tag: TypeTag[A13], implicit val rTag: TypeTag[R]) + extends Expression with ImplicitCastInputTypes with CodegenFallback with Serializable with SerdeAware { + + import InferredTypes._ + + def inputExpressions: Seq[Expression] + + override def children: Seq[Expression] = inputExpressions + + override def toString: String = s" **${getClass.getName}** " + + override def inputTypes: Seq[AbstractDataType] = Seq(inferSparkType[A1], inferSparkType[A2], inferSparkType[A3], inferSparkType[A4], inferSparkType[A5], + inferSparkType[A6], inferSparkType[A7], inferSparkType[A8], inferSparkType[A9], inferSparkType[A10], + inferSparkType[A11], inferSparkType[A12], inferSparkType[A13]) + + override def nullable: Boolean = true + + override def dataType = inferSparkType[R] + + lazy val extractFirst = buildExtractor[A1](inputExpressions.head) + lazy val extractSecond = buildExtractor[A2](inputExpressions(1)) + lazy val extractThird = buildExtractor[A3](inputExpressions(2)) + lazy val extractFourth = buildExtractor[A4](inputExpressions(3)) + lazy val extractFifth = buildExtractor[A5](inputExpressions(4)) + lazy val extractSixth = buildExtractor[A6](inputExpressions(5)) + lazy val extractSeventh = buildExtractor[A7](inputExpressions(6)) + lazy val extractEighth = buildExtractor[A8](inputExpressions(7)) + lazy val extractNinth = buildExtractor[A9](inputExpressions(8)) + lazy val extractTenth = buildExtractor[A10](inputExpressions(9)) + lazy val extractEleventh = buildExtractor[A11](inputExpressions(10)) + lazy val extractTwelfth = buildExtractor[A12](inputExpressions(11)) + lazy val extractThirteenth = buildExtractor[A13](inputExpressions(12)) + + lazy val serialize = buildSerializer[R] + + override def eval(input: InternalRow): Any = { + val first = extractFirst(input) + val second = extractSecond(input) + val third = extractThird(input) + val fourth = extractFourth(input) + val fifth = extractFifth(input) + val sixth = extractSixth(input) + val seventh = extractSeventh(input) + val eighth = extractEighth(input) + val ninth = extractNinth(input) + val tenth = extractTenth(input) + val eleventh = extractEleventh(input) + val twelfth = extractTwelfth(input) + val thirteenth = extractThirteenth(input) + + if (first != null && second != null && third != null && fourth != null && + fifth != null & sixth != null & seventh != null & eighth != null && + ninth != null & tenth != null & eleventh != null & twelfth != null && + thirteenth != null) { + serialize(f(first, second, third, fourth, fifth, sixth, seventh, eighth, ninth, tenth, eleventh, twelfth, thirteenth)) + } else { + null + } + } + + override def evalWithoutSerialization(input: InternalRow): Any = { + val first = extractFirst(input) + val second = extractSecond(input) + val third = extractThird(input) + val fourth = extractFourth(input) + val fifth = extractFifth(input) + val sixth = extractSixth(input) + val seventh = extractSeventh(input) + val eighth = extractEighth(input) + val ninth = extractNinth(input) + val tenth = extractTenth(input) + val eleventh = extractEleventh(input) + val twelfth = extractTwelfth(input) + val thirteenth = extractThirteenth(input) + + if (first != null && second != null && third != null && fourth != null && + fifth != null & sixth != null & seventh != null & eighth != null && + ninth != null & tenth != null & eleventh != null & twelfth != null && + thirteenth != null) { + serialize(f(first, second, third, fourth, fifth, sixth, seventh, eighth, ninth, tenth, eleventh, twelfth, thirteenth)) + } else { + null + } + } +} + diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala index 6e110e6ad0..98734e197b 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala @@ -326,5 +326,16 @@ object st_functions extends DataFrameAPI { def ST_Translate(geometry: String, deltaX: Double, deltaY: Double): Column = wrapExpression[ST_Translate](geometry, deltaX, deltaY, 0.0) + def ST_Affine(geometry: Column, a: Column, b: Column, d: Column, e: Column, xOff: Column, yOff: Column, c: Column, f: Column, g: Column, h: Column, i: Column, zOff: Column): Column = + wrapExpression[ST_Affine](geometry, a, b, d, e, xOff, yOff, c, f, g, h, i, zOff) + + def ST_Affine(geometry: String, a: Double, b: Double, d: Double, e: Double, xOff: Double, yOff: Double, c: Double, f: Double, g: Double, h: Double, i: Double, zOff: Double): Column = + wrapExpression[ST_Affine](geometry, a, b, d, e, xOff, yOff, c, f, g, h, i, zOff) + + def ST_Affine(geometry: Column, a: Column, b: Column, d: Column, e: Column, xOff: Column, yOff: Column) = + wrapExpression[ST_Affine](geometry, a, b, d, e, xOff, yOff, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0) + + def ST_Affine(geometry: String, a: Double, b: Double, d: Double, e: Double, xOff: Double, yOff: Double) = + wrapExpression[ST_Affine](geometry, a, b, d, e, xOff, yOff, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0) } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala index a9aab8444e..e589b14662 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala @@ -994,5 +994,21 @@ class dataFrameAPITestScala extends TestBaseScala { val expectedDefaultValue = "POLYGON Z((3 3 1, 3 4 1, 4 4 1, 4 3 1, 3 3 1))" assert(expectedDefaultValue == actualDefaultValue) } + + it("Passed ST_Affine") { + val polyDf = sparkSession.sql("SELECT ST_GeomFromWKT('POLYGON ((2 3 1, 4 5 1, 7 8 2, 2 3 1))') AS geom") + //val df = polyDf.select(ST_Affine("geom", 1, 2, 3, 4, 1, 2, 3, 4, 1, 4, 2, 1)); + val dfDefaultValue = polyDf.select(ST_Affine("geom", 1, 2, 1, 2, 1, 2)) + val wKTWriter3D = new WKTWriter(3); + //val actualGeom = df.take(1)(0).get(0).asInstanceOf[Geometry] + val actualGeomDefaultValue = dfDefaultValue.take(1)(0).get(0).asInstanceOf[Geometry] + //val actual = wKTWriter3D.write(actualGeom) + val expected = "POLYGON Z((12 24 17, 18 38 27, 30 63 44, 12 24 17))" + val actualDefaultValue = wKTWriter3D.write(actualGeomDefaultValue) + val expectedDefaultValue = "POLYGON Z((9 10 1, 15 16 1, 24 25 2, 9 10 1))" + //assertEquals(expected, actual) + assertEquals(expectedDefaultValue, actualDefaultValue) + + } } } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index 2bab5e5413..da6ec321d0 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -1973,4 +1973,16 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample assertEquals(expectedDefaultValue, actualDefaultValue) } } + + it ("should pass ST_Affine") { + val geomTestCases = Map ( + ("'POLYGON ((1 0 1, 1 1 1, 2 2 2, 1 0 1))'")-> "'POLYGON Z((2 3 1, 4 5 1, 7 8 2 ,2 3 1))'" + ) + for (((geom), expectedResult) <- geomTestCases) { + val df = sparkSession.sql(s"SELECT ST_AsText(ST_Affine(ST_GeomFromWKT($geom), 1, 2, 1, 2, 1, 2)) AS geom, " + s"$expectedResult") + val actual = df.take(1)(0).get(0).asInstanceOf[String] + val expected = df.take(1)(0).get(1).asInstanceOf[String] + assertEquals(expected, actual) + } + } } From 959b35c1ca49d21ce195a9ea92892f202dded705 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Mon, 19 Jun 2023 11:28:20 -0700 Subject: [PATCH 32/41] core logic implementation of BoundingDiagonal (without fits parameter) --- .../org/apache/sedona/common/Functions.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index 7108ad84ee..d5afb4d7fa 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -946,4 +946,38 @@ public static Geometry geometricMedian(Geometry geometry) throws Exception { return geometricMedian(geometry, DEFAULT_TOLERANCE, DEFAULT_MAX_ITER, false); } + public static LineString boundingDiagonal(Geometry geometry) { + if (geometry.isEmpty()) { + return GEOMETRY_FACTORY.createLineString(); + }else { + Double startX = null, startY = null, startZ = null, endX = null, endY = null, endZ = null; + boolean is3d = Double.isNaN(geometry.getCoordinate().z); + Coordinate[] coordinates = geometry.getCoordinates(); + for (Coordinate currCoordinate : coordinates) { + Double geomX = currCoordinate.getX(), geomY = currCoordinate.getY(); + startX = startX == null ? currCoordinate.getX() : Math.min(startX, currCoordinate.getX()); + startY = startY == null ? currCoordinate.getY() : Math.min(startY, currCoordinate.getY()); + + endX = endX == null ? currCoordinate.getX() : Math.max(endX, currCoordinate.getX()); + endY = endY == null ? currCoordinate.getY() : Math.max(endY, currCoordinate.getY()); + + if (is3d) { + Double geomZ = currCoordinate.getZ(); + startZ = startZ == null ? currCoordinate.getZ() : Math.min(startZ, currCoordinate.getZ()); + endZ = endZ == null ? currCoordinate.getZ() : Math.max(endZ, currCoordinate.getZ()); + } + } + Coordinate startCoordinate; + Coordinate endCoordinate; + if (is3d) { + startCoordinate = new Coordinate(startX, startY, startZ); + endCoordinate = new Coordinate(endX, endY, endZ); + }else { + startCoordinate = new Coordinate(startX, startY); + endCoordinate = new Coordinate(endX, endY); + } + return GEOMETRY_FACTORY.createLineString(new Coordinate[] {startCoordinate, endCoordinate}); + } + } + } From 00d0e48a10f04994cbf09b1b21aa457b79bb9719 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Mon, 19 Jun 2023 21:01:25 -0700 Subject: [PATCH 33/41] Revert "temp affine commit" This reverts commit 19016ae966e68251fac7675f7e43945b50484223. --- .../org/apache/sedona/common/Functions.java | 15 -- .../apache/sedona/common/utils/GeomUtils.java | 18 +- .../apache/sedona/common/FunctionsTest.java | 90 +------- docs/api/flink/Function.md | 64 +++++- docs/api/sql/Function.md | 54 +++++ docs/community/develop.md | 20 +- .../java/org/apache/sedona/flink/Catalog.java | 1 - .../sedona/flink/expressions/Functions.java | 21 +- python/sedona/sql/st_functions.py | 32 +-- python/tests/sql/test_function.py | 207 +++++++----------- .../org/apache/sedona/sql/UDF/Catalog.scala | 1 - .../sedona_sql/expressions/Functions.scala | 7 - .../expressions/NullSafeExpressions.scala | 94 -------- .../sedona_sql/expressions/st_functions.scala | 11 - .../sedona/sql/dataFrameAPITestScala.scala | 16 -- .../apache/sedona/sql/functionTestScala.scala | 12 - 16 files changed, 222 insertions(+), 441 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index d5afb4d7fa..27e05d1d5c 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -896,21 +896,6 @@ public static Geometry translate(Geometry geometry, double deltaX, double deltaY return geometry; } - public static Geometry affine(Geometry geometry, double a, double b, double d, double e, double xOff, double yOff, double c, - double f, double g, double h, double i, double zOff) { - if (!geometry.isEmpty()) { - GeomUtils.affineGeom(geometry, a, b, d, e, xOff, yOff, c, f, g, h, i, zOff, true); - } - return geometry; - } - - public static Geometry affine(Geometry geometry, double a, double b, double d, double e, double xOff, double yOff) { - if (!geometry.isEmpty()) { - GeomUtils.affineGeom(geometry, a, b, d, e, xOff, yOff, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, false); - } - return geometry; - } - public static Geometry geometricMedian(Geometry geometry, double tolerance, int maxIter, boolean failIfNotConverged) throws Exception { String geometryType = geometry.getGeometryType(); if(!(Geometry.TYPENAME_POINT.equals(geometryType) || Geometry.TYPENAME_MULTIPOINT.equals(geometryType))) { diff --git a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java index 731843e776..8795f830ac 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java @@ -27,6 +27,7 @@ import org.locationtech.jts.operation.polygonize.Polygonizer; import org.locationtech.jts.operation.union.UnaryUnionOp; +import java.awt.*; import java.nio.ByteOrder; import java.util.*; import java.util.List; @@ -460,21 +461,4 @@ public static void translateGeom(Geometry geometry, double deltaX, double deltaY geometry.geometryChanged(); } } - - public static void affineGeom(Geometry geometry, double a, double b, double d, double e, double xOff, double yOff, double c, - double f, double g, double h, double i, double zOff, boolean set3d) { - Coordinate[] coordinates = geometry.getCoordinates(); - for (Coordinate currCoordinate: coordinates) { - double x = currCoordinate.getX(), y = currCoordinate.getY(), z = Double.isNaN(currCoordinate.getZ()) ? 0 : currCoordinate.getZ(); - double newX = a * x + b * y + c * z + xOff; - double newY = d * x + e * y + f * z + yOff; - currCoordinate.setX(newX); - currCoordinate.setY(newY); - if (set3d && !Double.isNaN(currCoordinate.getZ())) { - double newZ = g * x + h * y + i * z + zOff; - currCoordinate.setZ(newZ); - } - } - geometry.geometryChanged(); - } } diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index ab2980d82e..3a1745b6b4 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -24,6 +24,7 @@ import org.locationtech.jts.io.WKTReader; import org.locationtech.jts.io.WKTWriter; +import javax.sound.sampled.Line; import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -856,93 +857,4 @@ public void translateHybridGeomCollectionDeltaZ() { assertEquals(wktWriter3D.write(expectedPoint3D), wktWriter3D.write(actualGeometry.getGeometryN(0).getGeometryN(1))); assertEquals(emptyLineString.toText(), actualGeometry.getGeometryN(0).getGeometryN(2).toText()); } - - @Test - public void affineEmpty3D() { - LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); - String expected = emptyLineString.toText(); - String actual = Functions.affine(emptyLineString, 1, 1, 2, 3, 5, 6, 2, 3, 4, 4, 5, 6).toText(); - assertEquals(expected, actual); - } - - @Test - public void affineEmpty2D() { - LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); - String expected = emptyLineString.toText(); - String actual = Functions.affine(emptyLineString, 1, 2, 3, 4, 1, 2).toText(); - assertEquals(expected, actual); - } - - @Test - public void affine3DGeom2D() { - LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray(1, 0, 1, 1, 1, 2)); - String expected = GEOMETRY_FACTORY.createLineString(coordArray(6, 18, 7, 11, 8, 14)).toText(); - String actual = Functions.affine(lineString, 1, 1, 2, 3, 5, 6, 2, 3, 4, 4, 5, 6).toText(); - assertEquals(expected, actual); - } - - @Test - public void affine3DGeom3D() { - WKTWriter wktWriter = new WKTWriter(3); -// 2 3 1, 4 5 1, 7 8 2 ,2 3 1 - Polygon polygon3D = GEOMETRY_FACTORY.createPolygon(coordArray3d(2, 3, 1, 4, 5, 1, 7, 8, 2, 2, 3, 1)); - - LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray3d(1, 0, 1, 1, 1, 2, 1, 2, 2)); - String expected = wktWriter.write(GEOMETRY_FACTORY.createLineString(coordArray3d(8, 11, 15, 11, 17, 24, 12, 20, 28))); - String actual = wktWriter.write(Functions.affine(lineString, 1, 1, 2, 3, 5, 6, 2, 3, 4, 4, 5, 6)); - assertEquals(expected, actual); - } - - @Test - public void affine3DHybridGeomCollection() { - Point point3D = GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1, 1)); - Polygon polygon1 = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 0, 2, 1, 1, 2, 2, 1, 2, 2, 0, 2, 1, 0, 2)); - Polygon polygon2 = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 0, 1, 1, 1, 1, 2, 2, 2, 1, 0, 1)); - MultiPolygon multiPolygon = GEOMETRY_FACTORY.createMultiPolygon(new Polygon[] {polygon1, polygon2}); - Geometry geomCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {point3D, multiPolygon})}); - Geometry actualGeomCollection = Functions.affine(geomCollection, 1, 2, 1, 2, 1, 2, 3, 3, 1, 2, 3, 3); - WKTWriter wktWriter3D = new WKTWriter(3); - Point expectedPoint3D = GEOMETRY_FACTORY.createPoint(new Coordinate(7, 8, 9)); - Polygon expectedPolygon1 = GEOMETRY_FACTORY.createPolygon(coordArray3d(8, 9, 10, 10, 11, 12, 11, 12, 13, 9, 10, 11, 8, 9, 10)); - Polygon expectedPolygon2 = GEOMETRY_FACTORY.createPolygon(coordArray3d(5, 6, 7, 7, 8, 9, 13, 14, 15, 5, 6, 7)); - assertEquals(wktWriter3D.write(expectedPoint3D), wktWriter3D.write(actualGeomCollection.getGeometryN(0).getGeometryN(0))); - assertEquals(wktWriter3D.write(expectedPolygon1), wktWriter3D.write(actualGeomCollection.getGeometryN(0).getGeometryN(1).getGeometryN(0))); - assertEquals(wktWriter3D.write(expectedPolygon2), wktWriter3D.write(actualGeomCollection.getGeometryN(0).getGeometryN(1).getGeometryN(1))); - - } - - @Test - public void affine2DGeom3D() { - //1 0 1, 1 1 1, 2 2 2, 1 0 1 - WKTWriter wktWriter = new WKTWriter(3); - LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray3d(1, 0, 1, 1, 1, 2, 1, 2, 2)); - String expected = wktWriter.write(GEOMETRY_FACTORY.createLineString(coordArray3d(6, 8, 1, 7, 11, 2, 8, 14, 2))); - String actual = wktWriter.write(Functions.affine(lineString, 1, 1, 2, 3, 5, 6)); - assertEquals(expected, actual); - } - - @Test - public void affine2DGeom2D() { - LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray(1, 0, 1, 1, 1, 2)); - String expected = GEOMETRY_FACTORY.createLineString(coordArray(6, 8, 7, 11, 8, 14)).toText(); - String actual = Functions.affine(lineString, 1, 1, 2, 3, 5, 6).toText(); - assertEquals(expected, actual); - } - - @Test - public void affine2DHybridGeomCollection() { - Point point3D = GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1)); - Polygon polygon1 = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 0, 1, 0)); - Polygon polygon2 = GEOMETRY_FACTORY.createPolygon(coordArray(3, 4, 3, 5, 3, 7, 10, 7, 3, 4)); - MultiPolygon multiPolygon = GEOMETRY_FACTORY.createMultiPolygon(new Polygon[] {polygon1, polygon2}); - Geometry geomCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {point3D, multiPolygon})}); - Geometry actualGeomCollection = Functions.affine(geomCollection, 1, 2, 1, 2, 1, 2); - Point expectedPoint3D = GEOMETRY_FACTORY.createPoint(new Coordinate(4, 5)); - Polygon expectedPolygon1 = GEOMETRY_FACTORY.createPolygon(coordArray(2, 3, 4, 5, 5, 6, 3, 4, 2, 3)); - Polygon expectedPolygon2 = GEOMETRY_FACTORY.createPolygon(coordArray(12, 13, 14, 15, 18, 19, 25, 26, 12, 13)); - assertEquals(expectedPoint3D.toText(), actualGeomCollection.getGeometryN(0).getGeometryN(0).toText()); - assertEquals(expectedPolygon1.toText(), actualGeomCollection.getGeometryN(0).getGeometryN(1).getGeometryN(0).toText()); - assertEquals(expectedPolygon2.toText(), actualGeomCollection.getGeometryN(0).getGeometryN(1).getGeometryN(1).toText()); - } - } diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index 11d86ff6e1..fb68b55c93 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -703,7 +703,7 @@ SELECT ST_NDims(ST_GeomFromEWKT('POINT(1 1 2)')) Output: `3` -Spark SQL example with x,y coordinate: +Example with x,y coordinate: ```sql SELECT ST_NDims(ST_GeomFromText('POINT(1 1)')) @@ -711,6 +711,39 @@ SELECT ST_NDims(ST_GeomFromText('POINT(1 1)')) Output: `2` +## ST_NRings + +Introduction: Returns the number of rings in a Polygon or MultiPolygon. Contrary to ST_NumInteriorRings, +this function also takes into account the number of exterior rings. + +This function returns 0 for an empty Polygon or MultiPolygon. +If the geometry is not a Polygon or MultiPolygon, an IllegalArgument Exception is thrown. + +Format: `ST_NRings(geom: geometry)` + +Since: `1.4.1` + + +Examples: + +Input: `POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))` + +Output: `1` + +Input: `'MULTIPOLYGON (((1 0, 1 6, 6 6, 6 0, 1 0), (2 1, 2 2, 3 2, 3 1, 2 1)), ((10 0, 10 6, 16 6, 16 0, 10 0), (12 1, 12 2, 13 2, 13 1, 12 1)))'` + +Output: `4` + +Input: `'POLYGON EMPTY'` + +Output: `0` + +Input: `'LINESTRING (1 0, 1 1, 2 1)'` + +Output: `Unsupported geometry type: LineString, only Polygon or MultiPolygon geometries are supported.` + + + ## ST_NumGeometries Introduction: Returns the number of Geometries. If geometry is a GEOMETRYCOLLECTION (or MULTI*) return the number of geometries, for single geometries will return 1. @@ -945,13 +978,13 @@ Format: `ST_Transform (A:geometry, SourceCRS:string, TargetCRS:string ,[Optional Since: `v1.2.0` -Spark SQL example (simple): +Example (simple): ```sql SELECT ST_Transform(polygondf.countyshape, 'epsg:4326','epsg:3857') FROM polygondf ``` -Spark SQL example (with optional parameters): +Example (with optional parameters): ```sql SELECT ST_Transform(polygondf.countyshape, 'epsg:4326','epsg:3857', false) FROM polygondf @@ -960,6 +993,31 @@ FROM polygondf !!!note The detailed EPSG information can be searched on [EPSG.io](https://epsg.io/). +## ST_Translate +Introduction: Returns the input geometry with its X, Y and Z coordinates (if present in the geometry) translated by deltaX, deltaY and deltaZ (if specified) + +If the geometry is 2D, and a deltaZ parameter is specified, no change is done to the Z coordinate of the geometry and the resultant geometry is also 2D. + +If the geometry is empty, no change is done to it. + +If the given geometry contains sub-geometries (GEOMETRY COLLECTION, MULTI POLYGON/LINE/POINT), all underlying geometries are individually translated. + +Format: `ST_Translate(geometry: geometry, deltaX: deltaX, deltaY: deltaY, deltaZ: deltaZ)` + +Since: `1.4.1` + +Example: + +Input: `ST_Translate(GEOMETRYCOLLECTION(MULTIPOLYGON (((1 0, 1 1, 2 1, 2 0, 1 0)), ((1 2, 3 4, 3 5, 1 2))), POINT(1, 1, 1), LINESTRING EMPTY), 2, 2, 3)` + +Output: `GEOMETRYCOLLECTION(MULTIPOLYGON (((3 2, 3 3, 4 3, 4 2, 3 2)), ((3 4, 5 6, 5 7, 3 4))), POINT(3, 3, 4), LINESTRING EMPTY)` + +Input: `ST_Translate(POINT(1, 3, 2), 1, 2)` + +Output: `POINT(2, 5, 2)` + + + ## ST_X Introduction: Returns X Coordinate of given Point, null otherwise. diff --git a/docs/api/sql/Function.md b/docs/api/sql/Function.md index 05265b5381..4ef78b148e 100644 --- a/docs/api/sql/Function.md +++ b/docs/api/sql/Function.md @@ -1108,6 +1108,37 @@ SELECT ST_NPoints(polygondf.countyshape) FROM polygondf ``` +## ST_NRings + +Introduction: Returns the number of rings in a Polygon or MultiPolygon. Contrary to ST_NumInteriorRings, +this function also takes into account the number of exterior rings. + +This function returns 0 for an empty Polygon or MultiPolygon. +If the geometry is not a Polygon or MultiPolygon, an IllegalArgument Exception is thrown. + +Format: `ST_NRings(geom: geometry)` + +Since: `1.4.1` + + +Examples: + +Input: `POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))` + +Output: `1` + +Input: `'MULTIPOLYGON (((1 0, 1 6, 6 6, 6 0, 1 0), (2 1, 2 2, 3 2, 3 1, 2 1)), ((10 0, 10 6, 16 6, 16 0, 10 0), (12 1, 12 2, 13 2, 13 1, 12 1)))'` + +Output: `4` + +Input: `'POLYGON EMPTY'` + +Output: `0` + +Input: `'LINESTRING (1 0, 1 1, 2 1)'` + +Output: `Unsupported geometry type: LineString, only Polygon or MultiPolygon geometries are supported.` + ## ST_NumGeometries Introduction: Returns the number of Geometries. If geometry is a GEOMETRYCOLLECTION (or MULTI*) return the number of geometries, for single geometries will return 1. @@ -1567,6 +1598,29 @@ FROM polygondf !!!note The detailed EPSG information can be searched on [EPSG.io](https://epsg.io/). + +## ST_Translate +Introduction: Returns the input geometry with its X, Y and Z coordinates (if present in the geometry) translated by deltaX, deltaY and deltaZ (if specified) + +If the geometry is 2D, and a deltaZ parameter is specified, no change is done to the Z coordinate of the geometry and the resultant geometry is also 2D. + +If the geometry is empty, no change is done to it. +If the given geometry contains sub-geometries (GEOMETRY COLLECTION, MULTI POLYGON/LINE/POINT), all underlying geometries are individually translated. + +Format: `ST_Translate(geometry: geometry, deltaX: deltaX, deltaY: deltaY, deltaZ: deltaZ)` + +Since: `1.4.1` + +Example: + +Input: `ST_Translate(GEOMETRYCOLLECTION(MULTIPOLYGON (((1 0, 1 1, 2 1, 2 0, 1 0)), ((1 2, 3 4, 3 5, 1 2))), POINT(1, 1, 1), LINESTRING EMPTY), 2, 2, 3)` + +Output: `GEOMETRYCOLLECTION(MULTIPOLYGON (((3 2, 3 3, 4 3, 4 2, 3 2)), ((3 4, 5 6, 5 7, 3 4))), POINT(3, 3, 4), LINESTRING EMPTY)` + +Input: `ST_Translate(POINT(1, 3, 2), 1, 2)` + +Output: `POINT(2, 5, 2)` + ## ST_Union Introduction: Return the union of geometry A and B diff --git a/docs/community/develop.md b/docs/community/develop.md index dd8c47d2e9..9483322395 100644 --- a/docs/community/develop.md +++ b/docs/community/develop.md @@ -4,7 +4,7 @@ ### IDE -We recommend Intellij IDEA with Scala plugin installed. +We recommend Intellij IDEA with Scala plugin installed. Please make sure that the IDE has JDK 1.8 set as project default. ### Import the project @@ -51,6 +51,10 @@ Make sure you reload the POM.xml or reload the maven project. The IDE will ask y #### Run all unit tests In a terminal, go to the Sedona root folder. Run `mvn clean install`. All tests will take more than 15 minutes. To only build the project jars, run `mvn clean install -DskipTests`. +!!!Note + `mvn clean install` will compile Sedona with Spark 3.0 and Scala 2.12. If you have a different version of Spark in $SPARK_HOME, make sure to specify that using -Dspark command line arg. + For example, to compile sedona with Spark 3.4 and Scala 2.12, use: `mvn clean install -Dspark=3.4 -Dscala=2.12` + More details can be found on [Compile Sedona](../../setup/compile/) @@ -78,7 +82,19 @@ Re-run the test case. Do NOT right click the test case to re-run. Instead, click ## Python developers -More details to come. +#### Run all python tests + +To run all Python test cases, follow steps mentioned [here](../../setup/compile/#run-python-test). + +#### Run all python tests in a single test file +To run a particular python test file, specify the path of the .py file to pipenv. + +For example, to run all tests in `test_function.py` located in `python/tests/sql/`, use: `pipenv run pytest tests/sql/test_function.py`. + +#### Run a single test +To run a particular test in a particular .py test file, specify `file_name::class_name::test_name` to the pytest command. + +For example, to run the test on ST_Contains function located in sql/test_predicate.py, use: `pipenv run pytest tests/sql/test_predicate.py::TestPredicate::test_st_contains` ### IDE diff --git a/flink/src/main/java/org/apache/sedona/flink/Catalog.java b/flink/src/main/java/org/apache/sedona/flink/Catalog.java index ff3c76a84c..8d3559599e 100644 --- a/flink/src/main/java/org/apache/sedona/flink/Catalog.java +++ b/flink/src/main/java/org/apache/sedona/flink/Catalog.java @@ -99,7 +99,6 @@ public static UserDefinedFunction[] getFuncs() { new Functions.ST_Force3D(), new Functions.ST_NRings(), new Functions.ST_Translate(), - new Functions.ST_Affine(), }; } diff --git a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java index 52cec2e87f..79adcc8d23 100644 --- a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java +++ b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java @@ -13,6 +13,7 @@ */ package org.apache.sedona.flink.expressions; +import org.apache.calcite.runtime.Geometries; import org.apache.flink.table.annotation.DataTypeHint; import org.apache.flink.table.functions.ScalarFunction; import org.locationtech.jts.geom.Geometry; @@ -622,24 +623,4 @@ public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.j } } - public static class ST_Affine extends ScalarFunction { - @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) - public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = Geometry.class) Object o, @DataTypeHint("Double") Double a, - @DataTypeHint("Double") Double b, @DataTypeHint("Double") Double d, @DataTypeHint("Double") Double e, @DataTypeHint("Double") Double xOff, @DataTypeHint("Double") Double yOff, @DataTypeHint("Double") Double c, - @DataTypeHint("Double") Double f, @DataTypeHint("Double") Double g, @DataTypeHint("Double") Double h, @DataTypeHint("Double") Double i, - @DataTypeHint("Double") Double zOff) { - Geometry geometry = (Geometry) o; - return org.apache.sedona.common.Functions.affine(geometry, a, b, d, e, xOff, yOff, c, f, g, h, i, zOff); - } - - @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) - public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o, @DataTypeHint("Double") Double a, - @DataTypeHint("Double") Double b, @DataTypeHint("Double") Double d, @DataTypeHint("Double") Double e, - @DataTypeHint("Double") Double xOff, @DataTypeHint("Double") Double yOff) { - Geometry geometry = (Geometry) o; - return org.apache.sedona.common.Functions.affine(geometry, a, b, d, e, xOff, yOff); - } - - } - } diff --git a/python/sedona/sql/st_functions.py b/python/sedona/sql/st_functions.py index 6dc5451909..32a2f2728c 100644 --- a/python/sedona/sql/st_functions.py +++ b/python/sedona/sql/st_functions.py @@ -111,8 +111,7 @@ "ST_NumPoints", "ST_Force3D", "ST_NRings", - "ST_Translate", - "ST_Affine" + "ST_Translate" ] @@ -1278,32 +1277,3 @@ def ST_Translate(geometry: ColumnOrName, deltaX: Union[ColumnOrName, float], del args = (geometry, deltaX, deltaY, deltaZ) return _call_st_function("ST_Translate", args) -@validate_argument_types -def ST_Affine(geometry: ColumnOrName, a: Union[ColumnOrName, float], b: Union[ColumnOrName, float], d: Union[ColumnOrName, float], - e: Union[ColumnOrName, float], xOff: Union[ColumnOrName, float], yOff: Union[ColumnOrName, float], c: Optional[Union[ColumnOrName, float]] = 0.0, f: Optional[Union[ColumnOrName, float]] = 0.0, - g: Optional[Union[ColumnOrName, float]] = 0.0, h: Optional[Union[ColumnOrName, float]] = 0.0, - i: Optional[Union[ColumnOrName, float]] = 0.0, zOff: Optional[Union[ColumnOrName, float]] = 0.0) -> Column: - """ - Apply a 3D/2D affine tranformation to the given geometry - x = a * x + b * y + c * z + xOff | x = a * x + b * y + xOff - y = d * x + e * y + f * z + yOff | y = d * x + e * y + yOff - z = g * x + h * y + i * z + zOff - :param geometry: Geometry to apply affine transformation to - :param a: - :param b: - :param c: Default 0.0 - :param d: - :param e: - :param f: Default 0.0 - :param g: Default 0.0 - :param h: Default 0.0 - :param i: Default 0.0 - :param xOff: - :param yOff: - :param zOff: Default 0.0 - :return: Geometry with affine transformation applied - """ - args = (geometry, a, b, c, d, e, f, g, h, i, xOff, yOff, zOff) - return _call_st_function("ST_Affine", args) - - diff --git a/python/tests/sql/test_function.py b/python/tests/sql/test_function.py index df519b1203..6897fb8ee5 100644 --- a/python/tests/sql/test_function.py +++ b/python/tests/sql/test_function.py @@ -31,6 +31,7 @@ class TestPredicateJoin(TestBase): + geo_schema = StructType( [StructField("geom", GeometryType(), False)] ) @@ -183,34 +184,29 @@ def test_st_transform(self): polygon_df = self.spark.sql("select ST_GeomFromWKT(polygontable._c0) as countyshape from polygontable") polygon_df.createOrReplaceTempView("polygondf") polygon_df.show() - function_df = self.spark.sql( - "select ST_Transform(ST_FlipCoordinates(polygondf.countyshape), 'epsg:4326','epsg:3857', false) from polygondf") + function_df = self.spark.sql("select ST_Transform(ST_FlipCoordinates(polygondf.countyshape), 'epsg:4326','epsg:3857', false) from polygondf") function_df.show() def test_st_intersection_intersects_but_not_contains(self): - test_table = self.spark.sql( - "select ST_GeomFromWKT('POLYGON((1 1, 8 1, 8 8, 1 8, 1 1))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b") + test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON((1 1, 8 1, 8 8, 1 8, 1 1))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b") test_table.createOrReplaceTempView("testtable") intersect = self.spark.sql("select ST_Intersection(a,b) from testtable") assert intersect.take(1)[0][0].wkt == "POLYGON ((2 8, 8 8, 8 2, 2 2, 2 8))" def test_st_intersection_intersects_but_left_contains_right(self): - test_table = self.spark.sql( - "select ST_GeomFromWKT('POLYGON((1 1, 1 5, 5 5, 1 1))') as a,ST_GeomFromWKT('POLYGON((2 2, 2 3, 3 3, 2 2))') as b") + test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON((1 1, 1 5, 5 5, 1 1))') as a,ST_GeomFromWKT('POLYGON((2 2, 2 3, 3 3, 2 2))') as b") test_table.createOrReplaceTempView("testtable") intersects = self.spark.sql("select ST_Intersection(a,b) from testtable") assert intersects.take(1)[0][0].wkt == "POLYGON ((2 2, 2 3, 3 3, 2 2))" def test_st_intersection_intersects_but_right_contains_left(self): - test_table = self.spark.sql( - "select ST_GeomFromWKT('POLYGON((2 2, 2 3, 3 3, 2 2))') as a,ST_GeomFromWKT('POLYGON((1 1, 1 5, 5 5, 1 1))') as b") + test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON((2 2, 2 3, 3 3, 2 2))') as a,ST_GeomFromWKT('POLYGON((1 1, 1 5, 5 5, 1 1))') as b") test_table.createOrReplaceTempView("testtable") intersects = self.spark.sql("select ST_Intersection(a,b) from testtable") assert intersects.take(1)[0][0].wkt == "POLYGON ((2 2, 2 3, 3 3, 2 2))" def test_st_intersection_not_intersects(self): - test_table = self.spark.sql( - "select ST_GeomFromWKT('POLYGON((40 21, 40 22, 40 23, 40 21))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b") + test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON((40 21, 40 22, 40 23, 40 21))') as a,ST_GeomFromWKT('POLYGON((2 2, 9 2, 9 9, 2 9, 2 2))') as b") test_table.createOrReplaceTempView("testtable") intersects = self.spark.sql("select ST_Intersection(a,b) from testtable") assert intersects.take(1)[0][0].wkt == "POLYGON EMPTY" @@ -259,17 +255,16 @@ def test_st_as_text(self): wkt_df = self.spark.sql("select ST_AsText(countyshape) as wkt from polygondf") assert polygon_df.take(1)[0]["countyshape"].wkt == loads(wkt_df.take(1)[0]["wkt"]).wkt + def test_st_astext_3d(self): input_df = self.spark.createDataFrame([ ("Point(21 52 87)",), ("Polygon((0 0 1, 0 1 1, 1 1 1, 1 0 1, 0 0 1))",), ("Linestring(0 0 1, 1 1 2, 1 0 3)",), ("MULTIPOINT ((10 40 66), (40 30 77), (20 20 88), (30 10 99))",), - ( - "MULTIPOLYGON (((30 20 11, 45 40 11, 10 40 11, 30 20 11)), ((15 5 11, 40 10 11, 10 20 11, 5 10 11, 15 5 11)))",), + ("MULTIPOLYGON (((30 20 11, 45 40 11, 10 40 11, 30 20 11)), ((15 5 11, 40 10 11, 10 20 11, 5 10 11, 15 5 11)))",), ("MULTILINESTRING ((10 10 11, 20 20 11, 10 40 11), (40 40 11, 30 30 11, 40 20 11, 30 10 11))",), - ( - "MULTIPOLYGON (((40 40 11, 20 45 11, 45 30 11, 40 40 11)), ((20 35 11, 10 30 11, 10 10 11, 30 5 11, 45 20 11, 20 35 11), (30 20 11, 20 15 11, 20 25 11, 30 20 11)))",), + ("MULTIPOLYGON (((40 40 11, 20 45 11, 45 30 11, 40 40 11)), ((20 35 11, 10 30 11, 10 10 11, 30 5 11, 45 20 11, 20 35 11), (30 20 11, 20 15 11, 20 25 11, 30 20 11)))",), ("POLYGON((0 0 11, 0 5 11, 5 5 11, 5 0 11, 0 0 11), (1 1 11, 2 1 11, 2 2 11, 1 2 11, 1 1 11))",), ], ["wkt"]) @@ -290,78 +285,64 @@ def test_st_as_text_3d(self): assert polygon_df.take(1)[0]["countyshape"].wkt == loads(wkt_df.take(1)[0]["wkt"]).wkt def test_st_n_points(self): - test = self.spark.sql( - "SELECT ST_NPoints(ST_GeomFromText('LINESTRING(77.29 29.07,77.42 29.26,77.27 29.31,77.29 29.07)'))") + test = self.spark.sql("SELECT ST_NPoints(ST_GeomFromText('LINESTRING(77.29 29.07,77.42 29.26,77.27 29.31,77.29 29.07)'))") def test_st_geometry_type(self): - test = self.spark.sql( - "SELECT ST_GeometryType(ST_GeomFromText('LINESTRING(77.29 29.07,77.42 29.26,77.27 29.31,77.29 29.07)'))") + test = self.spark.sql("SELECT ST_GeometryType(ST_GeomFromText('LINESTRING(77.29 29.07,77.42 29.26,77.27 29.31,77.29 29.07)'))") def test_st_difference_right_overlaps_left(self): - test_table = self.spark.sql( - "select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((0 -4, 4 -4, 4 4, 0 4, 0 -4))') as b") + test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((0 -4, 4 -4, 4 4, 0 4, 0 -4))') as b") test_table.createOrReplaceTempView("test_diff") diff = self.spark.sql("select ST_Difference(a,b) from test_diff") assert diff.take(1)[0][0].wkt == "POLYGON ((0 -3, -3 -3, -3 3, 0 3, 0 -3))" def test_st_difference_right_not_overlaps_left(self): - test_table = self.spark.sql( - "select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((5 -3, 7 -3, 7 -1, 5 -1, 5 -3))') as b") + test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((5 -3, 7 -3, 7 -1, 5 -1, 5 -3))') as b") test_table.createOrReplaceTempView("test_diff") diff = self.spark.sql("select ST_Difference(a,b) from test_diff") assert diff.take(1)[0][0].wkt == "POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))" def test_st_difference_left_contains_right(self): - test_table = self.spark.sql( - "select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as b") + test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as b") test_table.createOrReplaceTempView("test_diff") diff = self.spark.sql("select ST_Difference(a,b) from test_diff") assert diff.take(1)[0][0].wkt == "POLYGON ((-3 -3, -3 3, 3 3, 3 -3, -3 -3), (-1 -1, 1 -1, 1 1, -1 1, -1 -1))" def test_st_difference_right_not_overlaps_left(self): - test_table = self.spark.sql( - "select ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as a,ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as b") + test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as a,ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as b") test_table.createOrReplaceTempView("test_diff") diff = self.spark.sql("select ST_Difference(a,b) from test_diff") assert diff.take(1)[0][0].wkt == "POLYGON EMPTY" def test_st_sym_difference_part_of_right_overlaps_left(self): - test_table = self.spark.sql( - "select ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as a,ST_GeomFromWKT('POLYGON ((0 -2, 2 -2, 2 0, 0 0, 0 -2))') as b") + test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as a,ST_GeomFromWKT('POLYGON ((0 -2, 2 -2, 2 0, 0 0, 0 -2))') as b") test_table.createOrReplaceTempView("test_sym_diff") diff = self.spark.sql("select ST_SymDifference(a,b) from test_sym_diff") - assert diff.take(1)[0][ - 0].wkt == "MULTIPOLYGON (((0 -1, -1 -1, -1 1, 1 1, 1 0, 0 0, 0 -1)), ((0 -1, 1 -1, 1 0, 2 0, 2 -2, 0 -2, 0 -1)))" + assert diff.take(1)[0][0].wkt == "MULTIPOLYGON (((0 -1, -1 -1, -1 1, 1 1, 1 0, 0 0, 0 -1)), ((0 -1, 1 -1, 1 0, 2 0, 2 -2, 0 -2, 0 -1)))" def test_st_sym_difference_not_overlaps_left(self): - test_table = self.spark.sql( - "select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((5 -3, 7 -3, 7 -1, 5 -1, 5 -3))') as b") + test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((5 -3, 7 -3, 7 -1, 5 -1, 5 -3))') as b") test_table.createOrReplaceTempView("test_sym_diff") diff = self.spark.sql("select ST_SymDifference(a,b) from test_sym_diff") - assert diff.take(1)[0][ - 0].wkt == "MULTIPOLYGON (((-3 -3, -3 3, 3 3, 3 -3, -3 -3)), ((5 -3, 5 -1, 7 -1, 7 -3, 5 -3)))" + assert diff.take(1)[0][0].wkt == "MULTIPOLYGON (((-3 -3, -3 3, 3 3, 3 -3, -3 -3)), ((5 -3, 5 -1, 7 -1, 7 -3, 5 -3)))" def test_st_sym_difference_contains(self): - test_table = self.spark.sql( - "select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as b") + test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))') as b") test_table.createOrReplaceTempView("test_sym_diff") diff = self.spark.sql("select ST_SymDifference(a,b) from test_sym_diff") assert diff.take(1)[0][0].wkt == "POLYGON ((-3 -3, -3 3, 3 3, 3 -3, -3 -3), (-1 -1, 1 -1, 1 1, -1 1, -1 -1))" def test_st_union_part_of_right_overlaps_left(self): - test_table = self.spark.sql( - "select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a, ST_GeomFromWKT('POLYGON ((-2 1, 2 1, 2 4, -2 4, -2 1))') as b") + test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a, ST_GeomFromWKT('POLYGON ((-2 1, 2 1, 2 4, -2 4, -2 1))') as b") test_table.createOrReplaceTempView("test_union") union = self.spark.sql("select ST_Union(a,b) from test_union") assert union.take(1)[0][0].wkt == "POLYGON ((2 3, 3 3, 3 -3, -3 -3, -3 3, -2 3, -2 4, 2 4, 2 3))" def test_st_union_not_overlaps_left(self): - test_table = self.spark.sql( - "select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((5 -3, 7 -3, 7 -1, 5 -1, 5 -3))') as b") + test_table = self.spark.sql("select ST_GeomFromWKT('POLYGON ((-3 -3, 3 -3, 3 3, -3 3, -3 -3))') as a,ST_GeomFromWKT('POLYGON ((5 -3, 7 -3, 7 -1, 5 -1, 5 -3))') as b") test_table.createOrReplaceTempView("test_union") union = self.spark.sql("select ST_Union(a,b) from test_union") - assert union.take(1)[0][ - 0].wkt == "MULTIPOLYGON (((-3 -3, -3 3, 3 3, 3 -3, -3 -3)), ((5 -3, 5 -1, 7 -1, 7 -3, 5 -3)))" + assert union.take(1)[0][0].wkt == "MULTIPOLYGON (((-3 -3, -3 3, 3 3, 3 -3, -3 -3)), ((5 -3, 5 -1, 7 -1, 7 -3, 5 -3)))" def test_st_azimuth(self): sample_points = create_sample_points(20) @@ -404,11 +385,11 @@ def test_st_x(self): linestrings = linestring_df.selectExpr("ST_X(geom) as x").filter("x IS NOT NULL") - assert ([point[0] for point in points] == [-71.064544, -88.331492, 88.331492, 1.0453, 32.324142]) + assert([point[0] for point in points] == [-71.064544, -88.331492, 88.331492, 1.0453, 32.324142]) - assert (not linestrings.count()) + assert(not linestrings.count()) - assert (not polygons.count()) + assert(not polygons.count()) def test_st_y(self): point_df = create_sample_points_df(self.spark, 5) @@ -422,11 +403,11 @@ def test_st_y(self): linestrings = linestring_df.selectExpr("ST_Y(geom) as y").filter("y IS NOT NULL") - assert ([point[0] for point in points] == [42.28787, 32.324142, 32.324142, 5.3324324, -88.331492]) + assert([point[0] for point in points] == [42.28787, 32.324142, 32.324142, 5.3324324, -88.331492]) - assert (not linestrings.count()) + assert(not linestrings.count()) - assert (not polygons.count()) + assert(not polygons.count()) def test_st_z(self): point_df = self.spark.sql( @@ -446,27 +427,27 @@ def test_st_z(self): linestrings = linestring_df.selectExpr("ST_Z(geom) as z").filter("z IS NOT NULL") - assert ([point[0] for point in points] == [3.3]) + assert([point[0] for point in points] == [3.3]) - assert (not linestrings.count()) + assert(not linestrings.count()) - assert (not polygons.count()) + assert(not polygons.count()) def test_st_z_max(self): linestring_df = self.spark.sql("SELECT ST_GeomFromWKT('LINESTRING Z (0 0 1, 0 1 2)') as geom") linestring_row = [lnstr_row[0] for lnstr_row in linestring_df.selectExpr("ST_ZMax(geom)").collect()] - assert (linestring_row == [2.0]) + assert(linestring_row == [2.0]) def test_st_z_min(self): - linestring_df = self.spark.sql( - "SELECT ST_GeomFromWKT('POLYGON Z ((0 0 2, 0 1 1, 1 1 2, 1 0 2, 0 0 2))') as geom") + linestring_df = self.spark.sql("SELECT ST_GeomFromWKT('POLYGON Z ((0 0 2, 0 1 1, 1 1 2, 1 0 2, 0 0 2))') as geom") linestring_row = [lnstr_row[0] for lnstr_row in linestring_df.selectExpr("ST_ZMin(geom)").collect()] - assert (linestring_row == [1.0]) + assert(linestring_row == [1.0]) def test_st_n_dims(self): point_df = self.spark.sql("SELECT ST_GeomFromWKT('POINT(1 1 2)') as geom") point_row = [pt_row[0] for pt_row in point_df.selectExpr("ST_NDims(geom)").collect()] - assert (point_row == [3]) + assert(point_row == [3]) + def test_st_start_point(self): @@ -488,11 +469,11 @@ def test_st_start_point(self): linestrings = linestring_df.selectExpr("ST_StartPoint(geom) as geom").filter("geom IS NOT NULL") - assert ([line[0] for line in linestrings.collect()] == [wkt.loads(el) for el in expected_points]) + assert([line[0] for line in linestrings.collect()] == [wkt.loads(el) for el in expected_points]) - assert (not points.count()) + assert(not points.count()) - assert (not polygons.count()) + assert(not polygons.count()) def test_st_end_point(self): linestring_dataframe = create_sample_lines_df(self.spark, 5) @@ -512,10 +493,10 @@ def test_st_end_point(self): empty_dataframe = other_geometry_dataframe.selectExpr("ST_EndPoint(geom) as geom"). \ filter("geom IS NOT NULL") - assert ([wkt_row[0] - for wkt_row in point_data_frame.selectExpr("ST_AsText(geom)").collect()] == expected_ending_points) + assert([wkt_row[0] + for wkt_row in point_data_frame.selectExpr("ST_AsText(geom)").collect()] == expected_ending_points) - assert (empty_dataframe.count() == 0) + assert(empty_dataframe.count() == 0) def test_st_boundary(self): wkt_list = [ @@ -538,7 +519,7 @@ def test_st_boundary(self): boundary_table = geometry_table.selectExpr("ST_Boundary(geom) as geom") boundary_wkt = [wkt_row[0] for wkt_row in boundary_table.selectExpr("ST_AsText(geom)").collect()] - assert (boundary_wkt == [ + assert(boundary_wkt == [ "MULTIPOINT ((1 1), (-1 1))", "MULTIPOINT ((100 150), (160 170))", "MULTILINESTRING ((10 130, 50 190, 110 190, 140 150, 150 80, 100 10, 20 40, 10 130), (70 40, 100 50, 120 80, 80 110, 50 90, 70 40))", @@ -561,34 +542,33 @@ def test_st_exterior_ring(self): linestring_wkt = [wkt_row[0] for wkt_row in linestring_df.selectExpr("ST_AsText(geom)").collect()] - assert (linestring_wkt == ["LINESTRING (0 0, 0 1, 1 1, 1 0, 0 0)", "LINESTRING (0 0, 1 1, 1 2, 1 1, 0 0)"]) + assert(linestring_wkt == ["LINESTRING (0 0, 0 1, 1 1, 1 0, 0 0)", "LINESTRING (0 0, 1 1, 1 2, 1 1, 0 0)"]) - assert (not empty_df.count()) + assert(not empty_df.count()) def test_st_geometry_n(self): data_frame = self.__wkt_list_to_data_frame(["MULTIPOINT((1 2), (3 4), (5 6), (8 9))"]) wkts = [data_frame.selectExpr(f"ST_GeometryN(geom, {i}) as geom").selectExpr("st_asText(geom)").collect()[0][0] for i in range(0, 4)] - assert (wkts == ["POINT (1 2)", "POINT (3 4)", "POINT (5 6)", "POINT (8 9)"]) + assert(wkts == ["POINT (1 2)", "POINT (3 4)", "POINT (5 6)", "POINT (8 9)"]) def test_st_interior_ring_n(self): polygon_df = self.__wkt_list_to_data_frame( - [ - "POLYGON((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1), (1 3, 2 3, 2 4, 1 4, 1 3), (3 3, 4 3, 4 4, 3 4, 3 3))"] + ["POLYGON((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1), (1 3, 2 3, 2 4, 1 4, 1 3), (3 3, 4 3, 4 4, 3 4, 3 3))"] ) other_geometry = create_sample_points_df(self.spark, 5).union(create_sample_lines_df(self.spark, 5)) wholes = [polygon_df.selectExpr(f"ST_InteriorRingN(geom, {i}) as geom"). - selectExpr("ST_AsText(geom)").collect()[0][0] + selectExpr("ST_AsText(geom)").collect()[0][0] for i in range(3)] empty_df = other_geometry.selectExpr("ST_InteriorRingN(geom, 1) as geom").filter("geom IS NOT NULL") - assert (not empty_df.count()) - assert (wholes == ["LINESTRING (1 1, 2 1, 2 2, 1 2, 1 1)", - "LINESTRING (1 3, 2 3, 2 4, 1 4, 1 3)", - "LINESTRING (3 3, 4 3, 4 4, 3 4, 3 3)"]) + assert(not empty_df.count()) + assert(wholes == ["LINESTRING (1 1, 2 1, 2 2, 1 2, 1 1)", + "LINESTRING (1 3, 2 3, 2 4, 1 4, 1 3)", + "LINESTRING (3 3, 4 3, 4 4, 3 4, 3 3)"]) def test_st_dumps(self): expected_geometries = [ @@ -618,14 +598,14 @@ def test_st_dumps(self): dumped_geometries = geometry_df.selectExpr("ST_Dump(geom) as geom") - assert (dumped_geometries.select(explode(col("geom"))).count() == 14) + assert(dumped_geometries.select(explode(col("geom"))).count() == 14) collected_geometries = dumped_geometries \ .select(explode(col("geom")).alias("geom")) \ .selectExpr("ST_AsText(geom) as geom") \ .collect() - assert ([geom_row[0] for geom_row in collected_geometries] == expected_geometries) + assert([geom_row[0] for geom_row in collected_geometries] == expected_geometries) def test_st_dump_points(self): expected_points = [ @@ -645,10 +625,10 @@ def test_st_dump_points(self): dumped_points = geometry_df.selectExpr("ST_DumpPoints(geom) as geom") \ .select(explode(col("geom")).alias("geom")) - assert (dumped_points.count() == 10) + assert(dumped_points.count() == 10) collected_points = [geom_row[0] for geom_row in dumped_points.selectExpr("ST_AsText(geom)").collect()] - assert (collected_points == expected_points) + assert(collected_points == expected_points) def test_st_is_closed(self): expected_result = [ @@ -673,14 +653,13 @@ def test_st_is_closed(self): (7, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))"), (8, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10))"), (9, "MULTILINESTRING ((10 10, 20 20, 10 40), (40 40, 30 30, 40 20, 30 10))"), - (10, - "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))") + (10, "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))") ] geometry_df = self.__wkt_pair_list_with_index_to_data_frame(geometry_list) is_closed = geometry_df.selectExpr("index", "ST_IsClosed(geom)").collect() is_closed_collected = [[*row] for row in is_closed] - assert (is_closed_collected == expected_result) + assert(is_closed_collected == expected_result) def test_num_interior_ring(self): geometries = [ @@ -693,15 +672,14 @@ def test_num_interior_ring(self): (7, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))"), (8, "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10))"), (9, "MULTILINESTRING ((10 10, 20 20, 10 40), (40 40, 30 30, 40 20, 30 10))"), - (10, - "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))"), + (10, "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))"), (11, "POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))")] geometry_df = self.__wkt_pair_list_with_index_to_data_frame(geometries) number_of_interior_rings = geometry_df.selectExpr("index", "ST_NumInteriorRings(geom) as num") collected_interior_rings = [[*row] for row in number_of_interior_rings.filter("num is not null").collect()] - assert (collected_interior_rings == [[2, 0], [11, 1]]) + assert(collected_interior_rings == [[2, 0], [11, 1]]) def test_st_add_point(self): geometry = [ @@ -714,9 +692,7 @@ def test_st_add_point(self): ("MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))", "Point(21 52)"), ("MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10))", "Point(21 52)"), ("MULTILINESTRING ((10 10, 20 20, 10 40), (40 40, 30 30, 40 20, 30 10))", "Point(21 52)"), - ( - "GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))", - "Point(21 52)"), + ("GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40), POLYGON ((40 40, 20 45, 45 30, 40 40)))", "Point(21 52)"), ("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))", "Point(21 52)") ] geometry_df = self.__wkt_pairs_to_data_frame(geometry) @@ -724,7 +700,7 @@ def test_st_add_point(self): collected_geometries = [ row[0] for row in modified_geometries.filter("geom is not null").selectExpr("ST_AsText(geom)").collect() ] - assert (collected_geometries[0] == "LINESTRING (0 0, 1 1, 1 0, 21 52)") + assert(collected_geometries[0] == "LINESTRING (0 0, 1 1, 1 0, 21 52)") def test_st_remove_point(self): result_and_expected = [ @@ -735,13 +711,11 @@ def test_st_remove_point(self): [self.calculate_st_remove("POINT(0 1)", 3), None], [self.calculate_st_remove("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))", 3), None], [self.calculate_st_remove("GEOMETRYCOLLECTION (POINT (40 10), LINESTRING (10 10, 20 20, 10 40))", 0), None], - [self.calculate_st_remove( - "MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), ((15 5, 40 10, 10 20, 5 10, 15 5)))", 3), None], - [self.calculate_st_remove( - "MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))", 3), None] + [self.calculate_st_remove("MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), ((15 5, 40 10, 10 20, 5 10, 15 5)))", 3), None], + [self.calculate_st_remove("MULTILINESTRING ((10 10, 20 20, 10 40, 10 10), (40 40, 30 30, 40 20, 30 10, 40 40))", 3), None] ] for actual, expected in result_and_expected: - assert (actual == expected) + assert(actual == expected) def test_st_is_ring(self): result_and_expected = [ @@ -752,7 +726,7 @@ def test_st_is_ring(self): [self.calculate_st_is_ring("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))"), None], ] for actual, expected in result_and_expected: - assert (actual == expected) + assert(actual == expected) def test_st_subdivide(self): # Given @@ -823,11 +797,12 @@ def test_st_make_polygon(self): geom_poly = geometry_df.withColumn("polygon", expr("ST_MakePolygon(geom)")) # Then only based on closed linestring geom is created - geom_poly.filter("polygon IS NOT NULL").selectExpr("ST_AsText(polygon)", "expected"). \ + geom_poly.filter("polygon IS NOT NULL").selectExpr("ST_AsText(polygon)", "expected").\ show() - result = geom_poly.filter("polygon IS NOT NULL").selectExpr("ST_AsText(polygon)", "expected"). \ + result = geom_poly.filter("polygon IS NOT NULL").selectExpr("ST_AsText(polygon)", "expected").\ collect() + assert result.__len__() == 1 for actual, expected in result: @@ -844,7 +819,7 @@ def test_st_geohash(self): ).select(expr("St_GeomFromText(_1)").alias("geom"), col("_2").alias("expected_hash")) # When - geohash_df = geometry_df.withColumn("geohash", expr("ST_GeoHash(geom, 10)")). \ + geohash_df = geometry_df.withColumn("geohash", expr("ST_GeoHash(geom, 10)")).\ select("geohash", "expected_hash") # Then @@ -911,7 +886,7 @@ def test_st_collect_on_array_type(self): geometry_df_collected = geometry_df.withColumn("collected", expr("ST_Collect(geom)")) # then result should be as expected - assert (set([el[0] for el in geometry_df_collected.selectExpr("ST_AsText(collected)").collect()]) == { + assert(set([el[0] for el in geometry_df_collected.selectExpr("ST_AsText(collected)").collect()]) == { "MULTILINESTRING ((1 2, 3 4), (3 4, 4 5))", "MULTIPOINT ((1 2), (-2 3))", "MULTIPOLYGON (((1 2, 1 4, 3 4, 3 2, 1 2)), ((0.5 0.5, 5 0, 5 5, 0 5, 0.5 0.5)))" @@ -929,7 +904,7 @@ def test_st_collect_on_multiple_columns(self): geometry_df_collected = geometry_df.withColumn("collected", expr("ST_Collect(geom_left, geom_right)")) # then result should be calculated - assert (set([el[0] for el in geometry_df_collected.selectExpr("ST_AsText(collected)").collect()]) == { + assert(set([el[0] for el in geometry_df_collected.selectExpr("ST_AsText(collected)").collect()]) == { "MULTILINESTRING ((1 2, 3 4), (3 4, 4 5))", "MULTIPOINT ((1 2), (-2 3))", "MULTIPOLYGON (((1 2, 1 4, 3 4, 3 2, 1 2)), ((0.5 0.5, 5 0, 5 5, 0 5, 0.5 0.5)))" @@ -984,34 +959,30 @@ def calculate_st_remove(self, wkt, index): return geometry_collected[0][0] if geometry_collected.__len__() != 0 else None def __wkt_pairs_to_data_frame(self, wkt_list: List) -> DataFrame: - return self.spark.createDataFrame([[wkt.loads(wkt_a), wkt.loads(wkt_b)] for wkt_a, wkt_b in wkt_list], - self.geo_pair_schema) + return self.spark.createDataFrame([[wkt.loads(wkt_a), wkt.loads(wkt_b)] for wkt_a, wkt_b in wkt_list], self.geo_pair_schema) def __wkt_list_to_data_frame(self, wkt_list: List) -> DataFrame: return self.spark.createDataFrame([[wkt.loads(given_wkt)] for given_wkt in wkt_list], self.geo_schema) def __wkt_pair_list_with_index_to_data_frame(self, wkt_list: List) -> DataFrame: - return self.spark.createDataFrame([[index, wkt.loads(given_wkt)] for index, given_wkt in wkt_list], - self.geo_schema_with_index) + return self.spark.createDataFrame([[index, wkt.loads(given_wkt)] for index, given_wkt in wkt_list], self.geo_schema_with_index) def test_st_pointonsurface(self): tests1 = { - "'POINT(0 5)'": "POINT (0 5)", - "'LINESTRING(0 5, 0 10)'": "POINT (0 5)", - "'POLYGON((0 0, 0 5, 5 5, 5 0, 0 0))'": "POINT (2.5 2.5)", - "'LINESTRING(0 5 1, 0 0 1, 0 10 2)'": "POINT Z(0 0 1)" + "'POINT(0 5)'":"POINT (0 5)", + "'LINESTRING(0 5, 0 10)'":"POINT (0 5)", + "'POLYGON((0 0, 0 5, 5 5, 5 0, 0 0))'":"POINT (2.5 2.5)", + "'LINESTRING(0 5 1, 0 0 1, 0 10 2)'":"POINT Z(0 0 1)" } for input_geom, expected_geom in tests1.items(): - pointOnSurface = self.spark.sql( - "select ST_AsText(ST_PointOnSurface(ST_GeomFromText({})))".format(input_geom)) + pointOnSurface = self.spark.sql("select ST_AsText(ST_PointOnSurface(ST_GeomFromText({})))".format(input_geom)) assert pointOnSurface.take(1)[0][0] == expected_geom - tests2 = {"'LINESTRING(0 5 1, 0 0 1, 0 10 2)'": "POINT Z(0 0 1)"} + tests2 = { "'LINESTRING(0 5 1, 0 0 1, 0 10 2)'":"POINT Z(0 0 1)" } for input_geom, expected_geom in tests2.items(): - pointOnSurface = self.spark.sql( - "select ST_AsEWKT(ST_PointOnSurface(ST_GeomFromWKT({})))".format(input_geom)) + pointOnSurface = self.spark.sql("select ST_AsEWKT(ST_PointOnSurface(ST_GeomFromWKT({})))".format(input_geom)) assert pointOnSurface.take(1)[0][0] == expected_geom def test_st_pointn(self): @@ -1087,8 +1058,7 @@ def test_st_line_from_multi_point(self): "LINESTRING Z(10 40 66, 40 30 77, 20 20 88, 30 10 99)" } for input_geom, expected_geom in test_cases.items(): - line_geometry = self.spark.sql( - "select ST_AsText(ST_LineFromMultiPoint(ST_GeomFromText({})))".format(input_geom)) + line_geometry = self.spark.sql("select ST_AsText(ST_LineFromMultiPoint(ST_GeomFromText({})))".format(input_geom)) assert line_geometry.take(1)[0][0] == expected_geom def test_st_s2_cell_ids(self): @@ -1124,14 +1094,7 @@ def test_nRings(self): def test_translate(self): expected = "POLYGON ((3 5, 3 6, 4 6, 4 5, 3 5))" - actual_df = self.spark.sql( - "SELECT ST_Translate(ST_GeomFromText('POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))'), 2, 5) AS geom") - actual = actual_df.selectExpr("ST_AsText(geom)").take(1)[0][0] + actualDf = self.spark.sql("SELECT ST_Translate(ST_GeomFromText('POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))'), 2, 5) AS geom") + actual = actualDf.selectExpr("ST_AsText(geom)").take(1)[0][0] assert expected == actual - def test_affine(self): - expected = "POLYGON Z((2 3 1, 4 5 1, 7 8 2 ,2 3 1))" - actual_df = self.spark.sql("SELECT ST_Affine(ST_GeomFromText('POLYGON ((1 0 1, 1 1 1, 2 2 2, 1 0 1))'), 1, 2, " - "1, 2, 1, 2) AS geom") - actual = actual_df.selectExpr("ST_AsText(geom)").take(1)[0][0] - assert expected == actual diff --git a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala index 15331d704e..cbb1319307 100644 --- a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala +++ b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala @@ -151,7 +151,6 @@ object Catalog { function[ST_Force3D](0.0), function[ST_NRings](), function[ST_Translate](0.0), - function[ST_Affine](0.0, 0.0, 0.0, 0.0, 0.0, 0.0), // Expression for rasters function[RS_NormalizedDifference](), function[RS_Mean](), diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala index ae7ff02304..41052806dd 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala @@ -1010,10 +1010,3 @@ case class ST_Translate(inputExpressions: Seq[Expression]) } } -case class ST_Affine(inputExpressions: Seq[Expression]) - extends InferredNaryExpression(Functions.affine) with FoldableExpression { - protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { - copy(inputExpressions = newChildren) - } -} - diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/NullSafeExpressions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/NullSafeExpressions.scala index 604d1ac53e..f526baf0cf 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/NullSafeExpressions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/NullSafeExpressions.scala @@ -364,97 +364,3 @@ abstract class InferredQuarternaryExpression[A1: InferrableType, A2: InferrableT } } } - -abstract class InferredNaryExpression[A1: InferrableType, A2: InferrableType, A3: InferrableType, A4: InferrableType, - A5: InferrableType, A6: InferrableType, A7: InferrableType, A8: InferrableType, A9: InferrableType, - A10: InferrableType, A11: InferrableType, A12: InferrableType, A13: InferrableType, R: InferrableType] - (f: (A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13) => R) - (implicit val a1Tag: TypeTag[A1], implicit val a2Tag: TypeTag[A2], implicit val a3Tag: TypeTag[A3], implicit val a4Tag: TypeTag[A4], - implicit val a5Tag: TypeTag[A5], implicit val a6Tag: TypeTag[A6], implicit val a7Tag: TypeTag[A7], - implicit val a8Tag: TypeTag[A8], implicit val a9Tag: TypeTag[A9], implicit val a10Tag: TypeTag[A10], - implicit val a11Tag: TypeTag[A11], implicit val a12Tag: TypeTag[A12], implicit val a13Tag: TypeTag[A13], implicit val rTag: TypeTag[R]) - extends Expression with ImplicitCastInputTypes with CodegenFallback with Serializable with SerdeAware { - - import InferredTypes._ - - def inputExpressions: Seq[Expression] - - override def children: Seq[Expression] = inputExpressions - - override def toString: String = s" **${getClass.getName}** " - - override def inputTypes: Seq[AbstractDataType] = Seq(inferSparkType[A1], inferSparkType[A2], inferSparkType[A3], inferSparkType[A4], inferSparkType[A5], - inferSparkType[A6], inferSparkType[A7], inferSparkType[A8], inferSparkType[A9], inferSparkType[A10], - inferSparkType[A11], inferSparkType[A12], inferSparkType[A13]) - - override def nullable: Boolean = true - - override def dataType = inferSparkType[R] - - lazy val extractFirst = buildExtractor[A1](inputExpressions.head) - lazy val extractSecond = buildExtractor[A2](inputExpressions(1)) - lazy val extractThird = buildExtractor[A3](inputExpressions(2)) - lazy val extractFourth = buildExtractor[A4](inputExpressions(3)) - lazy val extractFifth = buildExtractor[A5](inputExpressions(4)) - lazy val extractSixth = buildExtractor[A6](inputExpressions(5)) - lazy val extractSeventh = buildExtractor[A7](inputExpressions(6)) - lazy val extractEighth = buildExtractor[A8](inputExpressions(7)) - lazy val extractNinth = buildExtractor[A9](inputExpressions(8)) - lazy val extractTenth = buildExtractor[A10](inputExpressions(9)) - lazy val extractEleventh = buildExtractor[A11](inputExpressions(10)) - lazy val extractTwelfth = buildExtractor[A12](inputExpressions(11)) - lazy val extractThirteenth = buildExtractor[A13](inputExpressions(12)) - - lazy val serialize = buildSerializer[R] - - override def eval(input: InternalRow): Any = { - val first = extractFirst(input) - val second = extractSecond(input) - val third = extractThird(input) - val fourth = extractFourth(input) - val fifth = extractFifth(input) - val sixth = extractSixth(input) - val seventh = extractSeventh(input) - val eighth = extractEighth(input) - val ninth = extractNinth(input) - val tenth = extractTenth(input) - val eleventh = extractEleventh(input) - val twelfth = extractTwelfth(input) - val thirteenth = extractThirteenth(input) - - if (first != null && second != null && third != null && fourth != null && - fifth != null & sixth != null & seventh != null & eighth != null && - ninth != null & tenth != null & eleventh != null & twelfth != null && - thirteenth != null) { - serialize(f(first, second, third, fourth, fifth, sixth, seventh, eighth, ninth, tenth, eleventh, twelfth, thirteenth)) - } else { - null - } - } - - override def evalWithoutSerialization(input: InternalRow): Any = { - val first = extractFirst(input) - val second = extractSecond(input) - val third = extractThird(input) - val fourth = extractFourth(input) - val fifth = extractFifth(input) - val sixth = extractSixth(input) - val seventh = extractSeventh(input) - val eighth = extractEighth(input) - val ninth = extractNinth(input) - val tenth = extractTenth(input) - val eleventh = extractEleventh(input) - val twelfth = extractTwelfth(input) - val thirteenth = extractThirteenth(input) - - if (first != null && second != null && third != null && fourth != null && - fifth != null & sixth != null & seventh != null & eighth != null && - ninth != null & tenth != null & eleventh != null & twelfth != null && - thirteenth != null) { - serialize(f(first, second, third, fourth, fifth, sixth, seventh, eighth, ninth, tenth, eleventh, twelfth, thirteenth)) - } else { - null - } - } -} - diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala index 98734e197b..6e110e6ad0 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala @@ -326,16 +326,5 @@ object st_functions extends DataFrameAPI { def ST_Translate(geometry: String, deltaX: Double, deltaY: Double): Column = wrapExpression[ST_Translate](geometry, deltaX, deltaY, 0.0) - def ST_Affine(geometry: Column, a: Column, b: Column, d: Column, e: Column, xOff: Column, yOff: Column, c: Column, f: Column, g: Column, h: Column, i: Column, zOff: Column): Column = - wrapExpression[ST_Affine](geometry, a, b, d, e, xOff, yOff, c, f, g, h, i, zOff) - - def ST_Affine(geometry: String, a: Double, b: Double, d: Double, e: Double, xOff: Double, yOff: Double, c: Double, f: Double, g: Double, h: Double, i: Double, zOff: Double): Column = - wrapExpression[ST_Affine](geometry, a, b, d, e, xOff, yOff, c, f, g, h, i, zOff) - - def ST_Affine(geometry: Column, a: Column, b: Column, d: Column, e: Column, xOff: Column, yOff: Column) = - wrapExpression[ST_Affine](geometry, a, b, d, e, xOff, yOff, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0) - - def ST_Affine(geometry: String, a: Double, b: Double, d: Double, e: Double, xOff: Double, yOff: Double) = - wrapExpression[ST_Affine](geometry, a, b, d, e, xOff, yOff, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0) } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala index e589b14662..a9aab8444e 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala @@ -994,21 +994,5 @@ class dataFrameAPITestScala extends TestBaseScala { val expectedDefaultValue = "POLYGON Z((3 3 1, 3 4 1, 4 4 1, 4 3 1, 3 3 1))" assert(expectedDefaultValue == actualDefaultValue) } - - it("Passed ST_Affine") { - val polyDf = sparkSession.sql("SELECT ST_GeomFromWKT('POLYGON ((2 3 1, 4 5 1, 7 8 2, 2 3 1))') AS geom") - //val df = polyDf.select(ST_Affine("geom", 1, 2, 3, 4, 1, 2, 3, 4, 1, 4, 2, 1)); - val dfDefaultValue = polyDf.select(ST_Affine("geom", 1, 2, 1, 2, 1, 2)) - val wKTWriter3D = new WKTWriter(3); - //val actualGeom = df.take(1)(0).get(0).asInstanceOf[Geometry] - val actualGeomDefaultValue = dfDefaultValue.take(1)(0).get(0).asInstanceOf[Geometry] - //val actual = wKTWriter3D.write(actualGeom) - val expected = "POLYGON Z((12 24 17, 18 38 27, 30 63 44, 12 24 17))" - val actualDefaultValue = wKTWriter3D.write(actualGeomDefaultValue) - val expectedDefaultValue = "POLYGON Z((9 10 1, 15 16 1, 24 25 2, 9 10 1))" - //assertEquals(expected, actual) - assertEquals(expectedDefaultValue, actualDefaultValue) - - } } } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index da6ec321d0..2bab5e5413 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -1973,16 +1973,4 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample assertEquals(expectedDefaultValue, actualDefaultValue) } } - - it ("should pass ST_Affine") { - val geomTestCases = Map ( - ("'POLYGON ((1 0 1, 1 1 1, 2 2 2, 1 0 1))'")-> "'POLYGON Z((2 3 1, 4 5 1, 7 8 2 ,2 3 1))'" - ) - for (((geom), expectedResult) <- geomTestCases) { - val df = sparkSession.sql(s"SELECT ST_AsText(ST_Affine(ST_GeomFromWKT($geom), 1, 2, 1, 2, 1, 2)) AS geom, " + s"$expectedResult") - val actual = df.take(1)(0).get(0).asInstanceOf[String] - val expected = df.take(1)(0).get(1).asInstanceOf[String] - assertEquals(expected, actual) - } - } } From 0e23430f4af09797b2ed93d4eab710a4db72e59c Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Mon, 19 Jun 2023 21:39:05 -0700 Subject: [PATCH 34/41] Implement BoundingDiagonal --- .../org/apache/sedona/common/Functions.java | 16 ++--- .../apache/sedona/common/FunctionsTest.java | 61 +++++++++++++++++++ docs/api/flink/Function.md | 27 ++++++++ docs/api/sql/Function.md | 27 ++++++++ .../java/org/apache/sedona/flink/Catalog.java | 1 + .../sedona/flink/expressions/Functions.java | 9 +++ python/sedona/sql/st_functions.py | 13 +++- python/tests/sql/test_dataframe_api.py | 1 + python/tests/sql/test_function.py | 6 ++ .../org/apache/sedona/sql/UDF/Catalog.scala | 1 + .../sedona_sql/expressions/Functions.scala | 6 ++ .../sedona_sql/expressions/st_functions.scala | 6 ++ .../sedona/sql/dataFrameAPITestScala.scala | 9 +++ .../apache/sedona/sql/functionTestScala.scala | 14 +++++ 14 files changed, 186 insertions(+), 11 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index 27e05d1d5c..39338e3e9a 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -931,21 +931,17 @@ public static Geometry geometricMedian(Geometry geometry) throws Exception { return geometricMedian(geometry, DEFAULT_TOLERANCE, DEFAULT_MAX_ITER, false); } - public static LineString boundingDiagonal(Geometry geometry) { + public static Geometry boundingDiagonal(Geometry geometry) { if (geometry.isEmpty()) { return GEOMETRY_FACTORY.createLineString(); }else { - Double startX = null, startY = null, startZ = null, endX = null, endY = null, endZ = null; - boolean is3d = Double.isNaN(geometry.getCoordinate().z); + Envelope envelope = geometry.getEnvelopeInternal(); + if (envelope.isNull()) return GEOMETRY_FACTORY.createLineString(); + Double startX = envelope.getMinX(), startY = envelope.getMinY(), startZ = null, + endX = envelope.getMaxX(), endY = envelope.getMaxY(), endZ = null; + boolean is3d = !Double.isNaN(geometry.getCoordinate().z); Coordinate[] coordinates = geometry.getCoordinates(); for (Coordinate currCoordinate : coordinates) { - Double geomX = currCoordinate.getX(), geomY = currCoordinate.getY(); - startX = startX == null ? currCoordinate.getX() : Math.min(startX, currCoordinate.getX()); - startY = startY == null ? currCoordinate.getY() : Math.min(startY, currCoordinate.getY()); - - endX = endX == null ? currCoordinate.getX() : Math.max(endX, currCoordinate.getX()); - endY = endY == null ? currCoordinate.getY() : Math.max(endY, currCoordinate.getY()); - if (is3d) { Double geomZ = currCoordinate.getZ(); startZ = startZ == null ? currCoordinate.getZ() : Math.min(startZ, currCoordinate.getZ()); diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index 3a1745b6b4..1b92997656 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -15,6 +15,7 @@ import com.google.common.geometry.S2CellId; import com.google.common.math.DoubleMath; +import com.sun.org.apache.xpath.internal.operations.Mult; import org.apache.sedona.common.sphere.Haversine; import org.apache.sedona.common.sphere.Spheroid; import org.apache.sedona.common.utils.GeomUtils; @@ -857,4 +858,64 @@ public void translateHybridGeomCollectionDeltaZ() { assertEquals(wktWriter3D.write(expectedPoint3D), wktWriter3D.write(actualGeometry.getGeometryN(0).getGeometryN(1))); assertEquals(emptyLineString.toText(), actualGeometry.getGeometryN(0).getGeometryN(2).toText()); } + @Test + public void boundingDiagonalGeom2D() { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 0, 1, 1, 2, 1, 2, 2, 2, 0, 1, 0)); + String expected = "LINESTRING (1 0, 2 2)"; + String actual = Functions.boundingDiagonal(polygon).toText(); + assertEquals(expected, actual); + } + + @Test + public void boundingDiagonalGeom3D() { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 0, 1, 3, 2, 2, 2, 4, 5, 1, 1, 1, 1, 0, 1)); + WKTWriter wktWriter = new WKTWriter(3); + String expected = "LINESTRING Z(1 0 1, 3 4 5)"; + String actual = wktWriter.write(Functions.boundingDiagonal(polygon)); + assertEquals(expected, actual); + } + + @Test + public void boundingDiagonalGeomEmpty() { + LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); + String expected = "LINESTRING EMPTY"; + String actual = Functions.boundingDiagonal(emptyLineString).toText(); + assertEquals(expected, actual); + } + + @Test + public void boundingDiagonalGeomCollection2D() { + // ("'GEOMETRYCOLLECTION (MULTIPOLYGON (((1 1, 1 -1, 2 2, 2 9, 9 1, 1 1)), ((5 5, 4 4, 2 2 , 5 5))), POINT (-1 0))'") -> "'LINESTRING (-1 -1, 9 9)'" + Polygon polygon1 = GEOMETRY_FACTORY.createPolygon(coordArray(1, 1, 1, -1, 2, 2, 2, 9, 9, 1, 1, 1)); + Polygon polygon2 = GEOMETRY_FACTORY.createPolygon(coordArray(5, 5, 4, 4, 2, 2, 5, 5)); + MultiPolygon multiPolygon = GEOMETRY_FACTORY.createMultiPolygon(new Polygon[] {polygon1, polygon2}); + LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray(2, 2, 3, 3)); + Point point = GEOMETRY_FACTORY.createPoint(new Coordinate(-1, 0)); + Geometry geometryCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {multiPolygon, lineString, point}); + String expected = "LINESTRING (-1 -1, 9 9)"; + String actual = Functions.boundingDiagonal(geometryCollection).toText(); + assertEquals(expected, actual); + } + + @Test + public void boundingDiagonalGeomCollection3D() { + Polygon polygon1 = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 1, 4, 1, -1, 6, 2, 2, 4, 2, 9, 4, 9, 1, 0, 1, 1, 4)); + Polygon polygon2 = GEOMETRY_FACTORY.createPolygon(coordArray3d(5, 5, 1, 4, 4, 1, 2, 2, 2, 5, 5, 1)); + MultiPolygon multiPolygon = GEOMETRY_FACTORY.createMultiPolygon(new Polygon[] {polygon1, polygon2}); + LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray3d(2, 2, 9, 3, 3, -5)); + Point point = GEOMETRY_FACTORY.createPoint(new Coordinate(-1, 9, 1)); + Geometry geometryCollection = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {multiPolygon, lineString, point}); + String expected = "LINESTRING Z(-1 -1 -5, 9 9 9)"; + WKTWriter wktWriter = new WKTWriter(3); + String actual = wktWriter.write(Functions.boundingDiagonal(geometryCollection)); + assertEquals(expected, actual); + } + + @Test + public void boundingDiagonalSingleVertex() { + Point point = GEOMETRY_FACTORY.createPoint(new Coordinate(10, 5)); + String expected = "LINESTRING (10 5, 10 5)"; + String actual = Functions.boundingDiagonal(point).toText(); + assertEquals(expected, actual); + } } diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index fb68b55c93..dee06e6c16 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -208,6 +208,33 @@ SELECT ST_Boundary(ST_GeomFromText('POLYGON ((1 1, 0 0, -1 1, 1 1))')) Output: `LINEARRING (1 1, 0 0, -1 1, 1 1)` +## ST_BoundingDiagonal + +Introduction: Returns a linestring spanning minimum and maximum values of each dimension of the given geometry's coordinates as its start and end point respectively. +If an empty geometry is provided, the returned LineString is also empty. +If a single vertex (POINT) is provided, the returned LineString has both the start and end points same as the points coordinates + +Format: `ST_BoundingDiagonal(geom: geometry)` + +Since: `v1.4.1` + +Example: +```sql +SELECT ST_BoundingDiagonal(ST_GeomFromWKT(geom)) +``` + +Input: `POLYGON ((1 1 1, 3 3 3, 0 1 4, 4 4 0, 1 1 1))` + +Output: `LINESTRING Z(0 1 1, 4 4 4)` + +Input: `POINT (10 10)` + +Output: `LINESTRING (10 10, 10 10)` + +Input: `GEOMETRYCOLLECTION(POLYGON ((5 5 5, -1 2 3, -1 -1 0, 5 5 5)), POINT (10 3 3))` + +Output: `LINESTRING Z(-1 -1 0, 10 5 5)` + ## ST_Buffer Introduction: Returns a geometry/geography that represents all points whose distance from this Geometry/geography is less than or equal to distance. diff --git a/docs/api/sql/Function.md b/docs/api/sql/Function.md index 4ef78b148e..10aa260ee3 100644 --- a/docs/api/sql/Function.md +++ b/docs/api/sql/Function.md @@ -203,6 +203,33 @@ SELECT ST_Boundary(ST_GeomFromText('POLYGON((1 1,0 0, -1 1, 1 1))')) Output: `LINESTRING (1 1, 0 0, -1 1, 1 1)` +## ST_BoundingDiagonal + +Introduction: Returns a linestring spanning minimum and maximum values of each dimension of the given geometry's coordinates as its start and end point respectively. +If an empty geometry is provided, the returned LineString is also empty. +If a single vertex (POINT) is provided, the returned LineString has both the start and end points same as the points coordinates + +Format: `ST_BoundingDiagonal(geom: geometry)` + +Since: `v1.4.1` + +Spark SQL Example: +```sql +SELECT ST_BoundingDiagonal(ST_GeomFromWKT(geom)) +``` + +Input: `POLYGON ((1 1 1, 3 3 3, 0 1 4, 4 4 0, 1 1 1))` + +Output: `LINESTRING Z(0 1 1, 4 4 4)` + +Input: `POINT (10 10)` + +Output: `LINESTRING (10 10, 10 10)` + +Input: `GEOMETRYCOLLECTION(POLYGON ((5 5 5, -1 2 3, -1 -1 0, 5 5 5)), POINT (10 3 3))` + +Output: `LINESTRING Z(-1 -1 0, 10 5 5)` + ## ST_Buffer Introduction: Returns a geometry/geography that represents all points whose distance from this Geometry/geography is less than or equal to distance. diff --git a/flink/src/main/java/org/apache/sedona/flink/Catalog.java b/flink/src/main/java/org/apache/sedona/flink/Catalog.java index 8d3559599e..51b208ea1d 100644 --- a/flink/src/main/java/org/apache/sedona/flink/Catalog.java +++ b/flink/src/main/java/org/apache/sedona/flink/Catalog.java @@ -99,6 +99,7 @@ public static UserDefinedFunction[] getFuncs() { new Functions.ST_Force3D(), new Functions.ST_NRings(), new Functions.ST_Translate(), + new Functions.ST_BoundingDiagonal(), }; } diff --git a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java index 79adcc8d23..b3e92290eb 100644 --- a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java +++ b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java @@ -623,4 +623,13 @@ public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.j } } + public static class ST_BoundingDiagonal extends ScalarFunction { + + @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) + public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o) { + Geometry geometry = (Geometry) o; + return org.apache.sedona.common.Functions.boundingDiagonal(geometry); + } + + } } diff --git a/python/sedona/sql/st_functions.py b/python/sedona/sql/st_functions.py index 32a2f2728c..be60c6e677 100644 --- a/python/sedona/sql/st_functions.py +++ b/python/sedona/sql/st_functions.py @@ -111,7 +111,8 @@ "ST_NumPoints", "ST_Force3D", "ST_NRings", - "ST_Translate" + "ST_Translate", + "ST_BoundingDiagonal" ] @@ -1277,3 +1278,13 @@ def ST_Translate(geometry: ColumnOrName, deltaX: Union[ColumnOrName, float], del args = (geometry, deltaX, deltaY, deltaZ) return _call_st_function("ST_Translate", args) +@validate_argument_types +def ST_BoundingDiagonal(geometry: ColumnOrName) -> Column: + """ + Returns a LineString with the min/max values of each dimension of the bounding box of the given geometry as its + start/end coordinates. + :param geometry: Geometry to return bounding diagonal of. + :return: LineString spanning min and max values of each dimension of the given geometry + """ + + return _call_st_function("ST_BoundingDiagonal", geometry) diff --git a/python/tests/sql/test_dataframe_api.py b/python/tests/sql/test_dataframe_api.py index 56229b0e02..6aae5aef29 100644 --- a/python/tests/sql/test_dataframe_api.py +++ b/python/tests/sql/test_dataframe_api.py @@ -65,6 +65,7 @@ (stf.ST_Boundary, ("geom",), "triangle_geom", "", "LINESTRING (0 0, 1 0, 1 1, 0 0)"), (stf.ST_Buffer, ("point", 1.0), "point_geom", "ST_PrecisionReduce(geom, 2)", "POLYGON ((0.98 0.8, 0.92 0.62, 0.83 0.44, 0.71 0.29, 0.56 0.17, 0.38 0.08, 0.2 0.02, 0 0, -0.2 0.02, -0.38 0.08, -0.56 0.17, -0.71 0.29, -0.83 0.44, -0.92 0.62, -0.98 0.8, -1 1, -0.98 1.2, -0.92 1.38, -0.83 1.56, -0.71 1.71, -0.56 1.83, -0.38 1.92, -0.2 1.98, 0 2, 0.2 1.98, 0.38 1.92, 0.56 1.83, 0.71 1.71, 0.83 1.56, 0.92 1.38, 0.98 1.2, 1 1, 0.98 0.8))"), (stf.ST_BuildArea, ("geom",), "multiline_geom", "ST_Normalize(geom)", "POLYGON ((0 0, 1 1, 1 0, 0 0))"), + (stf.ST_BoundingDiagonal, ("geom",), "square_geom", "ST_BoundingDiagonal(geom)", "LINESTRING (1 0, 2 1)"), (stf.ST_Centroid, ("geom",), "triangle_geom", "ST_PrecisionReduce(geom, 2)", "POINT (0.67 0.33)"), (stf.ST_Collect, (lambda: f.expr("array(a, b)"),), "two_points", "", "MULTIPOINT Z (0 0 0, 3 0 4)"), (stf.ST_Collect, ("a", "b"), "two_points", "", "MULTIPOINT Z (0 0 0, 3 0 4)"), diff --git a/python/tests/sql/test_function.py b/python/tests/sql/test_function.py index 6897fb8ee5..18b28984ef 100644 --- a/python/tests/sql/test_function.py +++ b/python/tests/sql/test_function.py @@ -1098,3 +1098,9 @@ def test_translate(self): actual = actualDf.selectExpr("ST_AsText(geom)").take(1)[0][0] assert expected == actual + def test_boundingDiagonal(self): + expected = "LINESTRING (1 0, 2 1)" + actual_df = self.spark.sql("SELECT ST_BoundingDiagonal(ST_GeomFromText('POLYGON ((1 0, 1 1, 2 1, 2 0, " + "1 0))')) AS geom") + actual = actual_df.selectExpr("ST_AsText(geom)").take(1)[0][0] + assert expected == actual diff --git a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala index cbb1319307..d5c94c7f70 100644 --- a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala +++ b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala @@ -151,6 +151,7 @@ object Catalog { function[ST_Force3D](0.0), function[ST_NRings](), function[ST_Translate](0.0), + function[ST_BoundingDiagonal](), // Expression for rasters function[RS_NormalizedDifference](), function[RS_Mean](), diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala index 41052806dd..cc90ca01df 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala @@ -1010,3 +1010,9 @@ case class ST_Translate(inputExpressions: Seq[Expression]) } } +case class ST_BoundingDiagonal(inputExpressions: Seq[Expression]) + extends InferredUnaryExpression(Functions.boundingDiagonal) with FoldableExpression { + protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { + copy(inputExpressions = newChildren) + } +} diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala index 6e110e6ad0..f603cf278e 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala @@ -327,4 +327,10 @@ object st_functions extends DataFrameAPI { def ST_Translate(geometry: String, deltaX: Double, deltaY: Double): Column = wrapExpression[ST_Translate](geometry, deltaX, deltaY, 0.0) + def ST_BoundingDiagonal(geometry: Column) = + wrapExpression[ST_BoundingDiagonal](geometry) + + def ST_BoundingDiagonal(geometry: String) = + wrapExpression[ST_BoundingDiagonal](geometry) + } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala index a9aab8444e..8e6ed18bd8 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala @@ -994,5 +994,14 @@ class dataFrameAPITestScala extends TestBaseScala { val expectedDefaultValue = "POLYGON Z((3 3 1, 3 4 1, 4 4 1, 4 3 1, 3 3 1))" assert(expectedDefaultValue == actualDefaultValue) } + + it("Passed ST_BoundingDiagonal") { + val polyDf = sparkSession.sql("SELECT ST_GeomFromWKT('POLYGON ((1 0 1, 2 3 2, 5 0 1, 5 2 9, 1 0 1))') AS geom") + val df = polyDf.select(ST_BoundingDiagonal("geom")) + val wKTWriter = new WKTWriter(3); + val expected = "LINESTRING Z(1 0 1, 5 3 9)" + val actual = wKTWriter.write(df.take(1)(0).get(0).asInstanceOf[Geometry]) + assertEquals(expected, actual) + } } } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index 2bab5e5413..fa3cfc26c8 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -1973,4 +1973,18 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample assertEquals(expectedDefaultValue, actualDefaultValue) } } + + it ("should pass ST_BoundingDiagonal") { + val geomTestCases = Map ( + ("'POINT (10 10)'")-> "'LINESTRING (10 10, 10 10)'", + ("'POLYGON ((1 1 1, 4 4 4, 0 9 3, 0 9 9, 1 1 1))'") -> "'LINESTRING Z(0 1 1, 4 9 9)'", + ("'GEOMETRYCOLLECTION (MULTIPOLYGON (((1 1, 1 -1, 2 2, 2 9, 9 1, 1 1)), ((5 5, 4 4, 2 2 , 5 5))), POINT (-1 0))'") -> "'LINESTRING (-1 -1, 9 9)'" + ) + for (((geom), expectedResult) <- geomTestCases) { + val df = sparkSession.sql(s"SELECT ST_AsText(ST_BoundingDiagonal(ST_GeomFromWKT($geom))) AS geom, " + s"$expectedResult") + val actual = df.take(1)(0).get(0).asInstanceOf[String] + val expected = df.take(1)(0).get(1).asInstanceOf[String] + assertEquals(expected, actual) + } + } } From f18099e5666961d23af0acb6019a80d014c4e2a4 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Mon, 19 Jun 2023 21:46:20 -0700 Subject: [PATCH 35/41] Add flink test case --- .../test/java/org/apache/sedona/flink/FunctionTest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java b/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java index 0bf2536d7f..e49172a3a3 100644 --- a/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java +++ b/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java @@ -736,4 +736,13 @@ public void testTranslate() { assertEquals(expected, actual); } + @Test + public void testBoundingDiagonal() { + Table polyTable = tableEnv.sqlQuery("SELECT ST_BoundingDiagonal(ST_GeomFromWKT('POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))'))" +" AS " + polygonColNames[0]); + polyTable = polyTable.select(call(Functions.ST_AsText.class.getSimpleName(), $(polygonColNames[0]))); + String expected = "LINESTRING (1 0, 2 1)"; + String actual = (String) first(polyTable).getField(0); + assertEquals(expected, actual); + } + } From ea6ea55bcc1a878f6e73c6606f767f9c49c37c1c Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Tue, 20 Jun 2023 12:23:36 -0700 Subject: [PATCH 36/41] Reduce loops in ST_BoundingDiagonal Revert accidental incorrect removals from Catalog while merging with master --- .../java/org/apache/sedona/common/Functions.java | 16 ++++++++++------ .../org/apache/sedona/sql/UDF/Catalog.scala | 4 ++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index 39338e3e9a..000180ba5c 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -935,13 +935,17 @@ public static Geometry boundingDiagonal(Geometry geometry) { if (geometry.isEmpty()) { return GEOMETRY_FACTORY.createLineString(); }else { - Envelope envelope = geometry.getEnvelopeInternal(); - if (envelope.isNull()) return GEOMETRY_FACTORY.createLineString(); - Double startX = envelope.getMinX(), startY = envelope.getMinY(), startZ = null, - endX = envelope.getMaxX(), endY = envelope.getMaxY(), endZ = null; + //Envelope envelope = geometry.getEnvelopeInternal(); + // if (envelope.isNull()) return GEOMETRY_FACTORY.createLineString(); + Double startX = null, startY = null, startZ = null, + endX = null, endY = null, endZ = null; boolean is3d = !Double.isNaN(geometry.getCoordinate().z); - Coordinate[] coordinates = geometry.getCoordinates(); - for (Coordinate currCoordinate : coordinates) { + for (Coordinate currCoordinate : geometry.getCoordinates()) { + startX = startX == null ? currCoordinate.getX() : Math.min(startX, currCoordinate.getX()); + startY = startY == null ? currCoordinate.getY() : Math.min(startY, currCoordinate.getY()); + + endX = endX == null ? currCoordinate.getX() : Math.max(endX, currCoordinate.getX()); + endY = endY == null ? currCoordinate.getY() : Math.max(endY, currCoordinate.getY()); if (is3d) { Double geomZ = currCoordinate.getZ(); startZ = startZ == null ? currCoordinate.getZ() : Math.min(startZ, currCoordinate.getZ()); diff --git a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala index d5c94c7f70..f9929c56ad 100644 --- a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala +++ b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala @@ -179,10 +179,14 @@ object Catalog { function[RS_Array](), function[RS_Normalize](), function[RS_Append](), + function[RS_AddBandFromArray](), + function[RS_BandAsArray](), function[RS_FromArcInfoAsciiGrid](), function[RS_FromGeoTiff](), + function[RS_MakeEmptyRaster](java.lang.Integer.MAX_VALUE, 0.0, 0.0, 0), function[RS_Envelope](), function[RS_NumBands](), + function[RS_Metadata](), function[RS_SetSRID](), function[RS_SRID](), function[RS_Value](1), From 73e28c0e6699aad2ec411fa9738f83da02e6cd68 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Thu, 22 Jun 2023 13:54:45 -0700 Subject: [PATCH 37/41] Add ST_HausdorffDistance --- .../org/apache/sedona/common/Functions.java | 9 +++ .../apache/sedona/common/utils/GeomUtils.java | 10 +++ .../apache/sedona/common/FunctionsTest.java | 71 +++++++++++++++++++ docs/api/flink/Function.md | 40 +++++++++++ docs/api/sql/Function.md | 38 ++++++++++ .../java/org/apache/sedona/flink/Catalog.java | 1 + .../sedona/flink/expressions/Functions.java | 19 ++++- .../org/apache/sedona/flink/FunctionTest.java | 13 ++++ python/sedona/sql/st_functions.py | 13 ++++ python/tests/sql/test_dataframe_api.py | 3 + python/tests/sql/test_function.py | 16 +++++ .../org/apache/sedona/sql/UDF/Catalog.scala | 1 + .../sedona_sql/expressions/Functions.scala | 7 ++ .../sedona_sql/expressions/st_functions.scala | 10 +++ .../sedona/sql/dataFrameAPITestScala.scala | 12 ++++ .../apache/sedona/sql/functionTestScala.scala | 21 ++++++ 16 files changed, 283 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/sedona/common/Functions.java b/common/src/main/java/org/apache/sedona/common/Functions.java index 000180ba5c..7c44f67064 100644 --- a/common/src/main/java/org/apache/sedona/common/Functions.java +++ b/common/src/main/java/org/apache/sedona/common/Functions.java @@ -36,6 +36,7 @@ import org.locationtech.jts.operation.valid.IsValidOp; import org.locationtech.jts.precision.GeometryPrecisionReducer; import org.locationtech.jts.simplify.TopologyPreservingSimplifier; +import org.locationtech.jts.algorithm.distance.DiscreteHausdorffDistance; import org.opengis.referencing.FactoryException; import org.opengis.referencing.NoSuchAuthorityCodeException; import org.opengis.referencing.crs.CoordinateReferenceSystem; @@ -965,4 +966,12 @@ public static Geometry boundingDiagonal(Geometry geometry) { } } + public static Double hausdorffDistance(Geometry g1, Geometry g2, double densityFrac) throws Exception { + return GeomUtils.getHausdorffDistance(g1, g2, densityFrac); + } + + public static Double hausdorffDistance(Geometry g1, Geometry g2) throws Exception{ + return GeomUtils.getHausdorffDistance(g1, g2, -1); + } + } diff --git a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java index 8795f830ac..a47dc5102f 100644 --- a/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java +++ b/common/src/main/java/org/apache/sedona/common/utils/GeomUtils.java @@ -26,6 +26,7 @@ import org.locationtech.jts.io.WKTWriter; import org.locationtech.jts.operation.polygonize.Polygonizer; import org.locationtech.jts.operation.union.UnaryUnionOp; +import org.locationtech.jts.algorithm.distance.DiscreteHausdorffDistance; import java.awt.*; import java.nio.ByteOrder; @@ -461,4 +462,13 @@ public static void translateGeom(Geometry geometry, double deltaX, double deltaY geometry.geometryChanged(); } } + + public static Double getHausdorffDistance(Geometry g1, Geometry g2, double densityFrac) throws Exception { + if (g1.isEmpty() || g2.isEmpty()) return 0.0; + DiscreteHausdorffDistance hausdorffDistanceObj = new DiscreteHausdorffDistance(g1, g2); + if (densityFrac != -1) { + hausdorffDistanceObj.setDensifyFraction(densityFrac); + } + return hausdorffDistanceObj.distance(); + } } diff --git a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java index 1b92997656..fb18ccb1c0 100644 --- a/common/src/test/java/org/apache/sedona/common/FunctionsTest.java +++ b/common/src/test/java/org/apache/sedona/common/FunctionsTest.java @@ -918,4 +918,75 @@ public void boundingDiagonalSingleVertex() { String actual = Functions.boundingDiagonal(point).toText(); assertEquals(expected, actual); } + + @Test + public void hausdorffDistanceDefaultGeom2D() throws Exception { + Polygon polygon1 = GEOMETRY_FACTORY.createPolygon(coordArray3d(1, 0, 1, 1, 1, 2, 2, 1, 5, 2, 0, 1, 1, 0, 1)); + Polygon polygon2 = GEOMETRY_FACTORY.createPolygon(coordArray3d(4, 0, 4, 6, 1, 4, 6, 4, 9, 6, 1, 3, 4, 0, 4)); + Double expected = 5.0; + Double actual = Functions.hausdorffDistance(polygon1, polygon2); + assertEquals(expected, actual); + } + + @Test + public void hausdorffDistanceGeom2D() throws Exception { + Point point = GEOMETRY_FACTORY.createPoint(new Coordinate(10, 34)); + LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray(1, 2, 1, 5, 2, 6, 1, 2)); + Double expected = 33.24154027718932; + Double actual = Functions.hausdorffDistance(point, lineString, 0.33); + assertEquals(expected, actual); + } + + @Test + public void hausdorffDistanceInvalidDensityFrac() throws Exception { + Point point = GEOMETRY_FACTORY.createPoint(new Coordinate(10, 34)); + LineString lineString = GEOMETRY_FACTORY.createLineString(coordArray(1, 2, 1, 5, 2, 6, 1, 2)); + Exception e = assertThrows(IllegalArgumentException.class, () -> Functions.hausdorffDistance(point, lineString, 3)); + String expected = "Fraction is not in range (0.0 - 1.0]"; + String actual = e.getMessage(); + assertEquals(expected, actual); + } + + @Test + public void hausdorffDistanceDefaultGeomCollection() throws Exception { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 2, 2, 1, 2, 0, 4, 1, 1, 2)); + Geometry point1 = GEOMETRY_FACTORY.createPoint(new Coordinate(1, 0)); + Geometry point2 = GEOMETRY_FACTORY.createPoint(new Coordinate(40, 10)); + Geometry point3 = GEOMETRY_FACTORY.createPoint(new Coordinate(-10, -40)); + GeometryCollection multiPoint = GEOMETRY_FACTORY.createGeometryCollection(new Geometry[] {point1, point2, point3}); + Double actual = Functions.hausdorffDistance(polygon, multiPoint); + Double expected = 41.7612260356422; + assertEquals(expected, actual); + } + + @Test + public void hausdorffDistanceGeomCollection() throws Exception { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 2, 2, 1, 2, 0, 4, 1, 1, 2)); + LineString lineString1 = GEOMETRY_FACTORY.createLineString(coordArray(1, 1, 2, 1, 4, 4, 5, 5)); + LineString lineString2 = GEOMETRY_FACTORY.createLineString(coordArray(10, 10, 11, 11, 12, 12, 14, 14)); + LineString lineString3 = GEOMETRY_FACTORY.createLineString(coordArray(-11, -20, -11, -21, -15, -19)); + MultiLineString multiLineString = GEOMETRY_FACTORY.createMultiLineString(new LineString[] {lineString1, lineString2, lineString3}); + Double actual = Functions.hausdorffDistance(polygon, multiLineString, 0.0000001); + Double expected = 25.495097567963924; + assertEquals(expected, actual); + } + + @Test + public void hausdorffDistanceEmptyGeom() throws Exception { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 2, 2, 1, 2, 0, 4, 1, 1, 2)); + LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); + Double expected = 0.0; + Double actual = Functions.hausdorffDistance(polygon, emptyLineString, 0.00001); + assertEquals(expected, actual); + } + + @Test + public void hausdorffDistanceDefaultEmptyGeom() throws Exception { + Polygon polygon = GEOMETRY_FACTORY.createPolygon(coordArray(1, 2, 2, 1, 2, 0, 4, 1, 1, 2)); + LineString emptyLineString = GEOMETRY_FACTORY.createLineString(); + Double expected = 0.0; + Double actual = Functions.hausdorffDistance(polygon, emptyLineString); + assertEquals(expected, actual); + } + } diff --git a/docs/api/flink/Function.md b/docs/api/flink/Function.md index dee06e6c16..8f7281b9f0 100644 --- a/docs/api/flink/Function.md +++ b/docs/api/flink/Function.md @@ -533,6 +533,46 @@ SELECT ST_GeometryN(ST_GeomFromText('MULTIPOINT((1 2), (3 4), (5 6), (8 9))'), 1 Output: `POINT (3 4)` +## ST_HausdorffDistance + +Introduction: Returns a discretized (and hence approximate) [Hausdorff distance](https://en.wikipedia.org/wiki/Hausdorff_distance) between the given 2 geometries. +Optionally, a densityFraction parameter can be specified, which gives more accurate results by densifying segments before computing hausdorff distance between them. +Each segment is broken down into equal-length subsegments whose ratio with segment length is closest to the given density fraction. + +Hence, the lower the densityFrac value, the more accurate is the computed hausdorff distance, and the more time it takes to compute it. + +If any of the geometry is empty, 0.0 is returned. + + +!!!Note + Accepted range of densityFrac is (0.0, 1.0], if any other value is provided, ST_HausdorffDistance throws an IllegalArgumentException + + +!!!Note + Even though the function accepts 3D geometry, the z ordinate is ignored and the computed hausdorff distance is equivalent to the geometries not having the z ordinate. + +Format: `ST_HausdorffDistance(g1: geometry, g2: geometry, densityFrac)` + +Since: `v1.5.0` + +Example: +```sql +SELECT ST_HausdorffDistance(g1, g2, 0.1) +``` + +Input: `g1: POINT (0.0 1.0), g2: LINESTRING (0 0, 1 0, 2 0, 3 0, 4 0, 5 0)` + +Output: `5.0990195135927845` + +```sql +SELECT ST_HausdorffDistance(ST_GeomFromText(), ST_GeomFromText()) +``` + +Input: `g1: POLYGON Z((1 0 1, 1 1 2, 2 1 5, 2 0 1, 1 0 1)), g2: POLYGON Z((4 0 4, 6 1 4, 6 4 9, 6 1 3, 4 0 4))` + +Output: `5.0` + + ## ST_InteriorRingN Introduction: Returns the Nth interior linestring ring of the polygon geometry. Returns NULL if the geometry is not a polygon or the given N is out of range diff --git a/docs/api/sql/Function.md b/docs/api/sql/Function.md index 10aa260ee3..ac0edbb9bd 100644 --- a/docs/api/sql/Function.md +++ b/docs/api/sql/Function.md @@ -731,6 +731,44 @@ SELECT ST_GeometryType(polygondf.countyshape) FROM polygondf ``` +## ST_HausdorffDistance + +Introduction: Returns a discretized (and hence approximate) [Hausdorff distance](https://en.wikipedia.org/wiki/Hausdorff_distance) between the given 2 geometries. +Optionally, a densityFraction parameter can be specified, which gives more accurate results by densifying segments before computing hausdorff distance between them. +Each segment is broken down into equal-length subsegments whose ratio with segment length is closest to the given density fraction. + +Hence, the lower the densityFrac value, the more accurate is the computed hausdorff distance, and the more time it takes to compute it. + +If any of the geometry is empty, 0.0 is returned. + +!!!Note + Accepted range of densityFrac is (0.0, 1.0], if any other value is provided, ST_HausdorffDistance throws an IllegalArgumentException + + +!!!Note + Even though the function accepts 3D geometry, the z ordinate is ignored and the computed hausdorff distance is equivalent to the geometries not having the z ordinate. + +Format: `ST_HausdorffDistance(g1: geometry, g2: geometry, densityFrac)` + +Since: `v1.5.0` + +Example: +```sql +SELECT ST_HausdorffDistance(g1, g2, 0.1) +``` + +Input: `g1: POINT (0.0 1.0), g2: LINESTRING (0 0, 1 0, 2 0, 3 0, 4 0, 5 0)` + +Output: `5.0990195135927845` + +```sql +SELECT ST_HausdorffDistance(ST_GeomFromText(), ST_GeomFromText()) +``` + +Input: `g1: POLYGON Z((1 0 1, 1 1 2, 2 1 5, 2 0 1, 1 0 1)), g2: POLYGON Z((4 0 4, 6 1 4, 6 4 9, 6 1 3, 4 0 4))` + +Output: `5.0` + ## ST_InteriorRingN Introduction: Returns the Nth interior linestring ring of the polygon geometry. Returns NULL if the geometry is not a polygon or the given N is out of range diff --git a/flink/src/main/java/org/apache/sedona/flink/Catalog.java b/flink/src/main/java/org/apache/sedona/flink/Catalog.java index 51b208ea1d..dc992028fc 100644 --- a/flink/src/main/java/org/apache/sedona/flink/Catalog.java +++ b/flink/src/main/java/org/apache/sedona/flink/Catalog.java @@ -100,6 +100,7 @@ public static UserDefinedFunction[] getFuncs() { new Functions.ST_NRings(), new Functions.ST_Translate(), new Functions.ST_BoundingDiagonal(), + new Functions.ST_HausdorffDistance(), }; } diff --git a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java index b3e92290eb..c117c3e40b 100644 --- a/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java +++ b/flink/src/main/java/org/apache/sedona/flink/expressions/Functions.java @@ -624,12 +624,29 @@ public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.j } public static class ST_BoundingDiagonal extends ScalarFunction { - @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) public Geometry eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o) { Geometry geometry = (Geometry) o; return org.apache.sedona.common.Functions.boundingDiagonal(geometry); } + } + public static class ST_HausdorffDistance extends ScalarFunction { + @DataTypeHint("Double") + public Double eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object g1, + @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object g2, + @DataTypeHint("Double") Double densityFrac) throws Exception { + Geometry geom1 = (Geometry) g1; + Geometry geom2 = (Geometry) g2; + return org.apache.sedona.common.Functions.hausdorffDistance(geom1, geom2, densityFrac); + } + + @DataTypeHint("Double") + public Double eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object g1, + @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object g2) throws Exception { + Geometry geom1 = (Geometry) g1; + Geometry geom2 = (Geometry) g2; + return org.apache.sedona.common.Functions.hausdorffDistance(geom1, geom2); + } } } diff --git a/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java b/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java index e49172a3a3..2158648438 100644 --- a/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java +++ b/flink/src/test/java/org/apache/sedona/flink/FunctionTest.java @@ -745,4 +745,17 @@ public void testBoundingDiagonal() { assertEquals(expected, actual); } + @Test + public void testHausdorffDistance() { + Table polyTable = tableEnv.sqlQuery("SELECT ST_GeomFromWKT('POINT (0.0 1.0)') AS g1, ST_GeomFromWKT('LINESTRING (0 0, 1 0, 2 0, 3 0, 4 0, 5 0)') AS g2"); + Table actualTable = polyTable.select(call(Functions.ST_HausdorffDistance.class.getSimpleName(), $("g1"), $("g2"), 0.4)); + Table actualTableDefault = polyTable.select(call(Functions.ST_HausdorffDistance.class.getSimpleName(), $("g1"), $("g2"))); + Double expected = 5.0990195135927845; + Double expectedDefault = 5.0990195135927845; + Double actual = (Double) first(actualTable).getField(0); + Double actualDefault = (Double) first(actualTableDefault).getField(0); + assertEquals(expected, actual); + assertEquals(expectedDefault, actualDefault); + } + } diff --git a/python/sedona/sql/st_functions.py b/python/sedona/sql/st_functions.py index be60c6e677..fc85ee1799 100644 --- a/python/sedona/sql/st_functions.py +++ b/python/sedona/sql/st_functions.py @@ -1288,3 +1288,16 @@ def ST_BoundingDiagonal(geometry: ColumnOrName) -> Column: """ return _call_st_function("ST_BoundingDiagonal", geometry) + +@validate_argument_types +def ST_HausdorffDistance(g1: ColumnOrName, g2: ColumnOrName, densityFrac: Optional[Union[ColumnOrName, float]] = -1) -> Column: + """ + Returns discretized (and hence approximate) hausdorff distance between two given geometries. + Optionally, a distance fraction can also be provided which decreases the gap between actual and discretized hausforff distance + :param g1: + :param g2: + :param densityFrac: Optional + :return: + """ + args = (g1, g2, densityFrac) + return _call_st_function("ST_HausdorffDistance", args) diff --git a/python/tests/sql/test_dataframe_api.py b/python/tests/sql/test_dataframe_api.py index 6aae5aef29..7fce30bcd5 100644 --- a/python/tests/sql/test_dataframe_api.py +++ b/python/tests/sql/test_dataframe_api.py @@ -90,6 +90,7 @@ (stf.ST_GeometricMedian, ("multipoint",), "multipoint_geom", "", "POINT (22.500002656424286 21.250001168173426)"), (stf.ST_GeometryN, ("geom", 0), "multipoint", "", "POINT (0 0)"), (stf.ST_GeometryType, ("point",), "point_geom", "", "ST_Point"), + (stf.ST_HausdorffDistance, ("point", "line",), "point_and_line", "", 5.0990195135927845), (stf.ST_InteriorRingN, ("geom", 0), "geom_with_hole", "", "LINESTRING (1 1, 2 2, 2 1, 1 1)"), (stf.ST_Intersection, ("a", "b"), "overlapping_polys", "", "POLYGON ((2 0, 1 0, 1 1, 2 1, 2 0))"), (stf.ST_IsClosed, ("geom",), "closed_linestring_geom", "", True), @@ -391,6 +392,8 @@ def base_df(self, request): return TestDataFrameAPI.spark.sql("SELECT ST_GeomFromWKT('LINESTRING (0 0, 2 1)') AS line, ST_GeomFromWKT('POLYGON ((1 0, 2 0, 2 2, 1 2, 1 0))') AS poly") elif request.param == "square_geom": return TestDataFrameAPI.spark.sql("SELECT ST_GeomFromWKT('POLYGON ((1 0, 1 1, 2 1, 2 0, 1 0))') AS geom") + elif request.param == "point_and_line": + return TestDataFrameAPI.spark.sql("SELECT ST_GeomFromWKT('POINT (0.0 1.0)') AS point, ST_GeomFromWKT('LINESTRING (0 0, 1 0, 2 0, 3 0, 4 0, 5 0)') AS line") raise ValueError(f"Invalid base_df name passed: {request.param}") def _id_test_configuration(val): diff --git a/python/tests/sql/test_function.py b/python/tests/sql/test_function.py index 18b28984ef..a5d661623c 100644 --- a/python/tests/sql/test_function.py +++ b/python/tests/sql/test_function.py @@ -1104,3 +1104,19 @@ def test_boundingDiagonal(self): "1 0))')) AS geom") actual = actual_df.selectExpr("ST_AsText(geom)").take(1)[0][0] assert expected == actual + + def test_hausdorffDistance(self): + expected = 5.0 + actual_df = self.spark.sql("SELECT ST_HausdorffDistance(ST_GeomFromText('POLYGON ((1 0 1, 1 1 2, 2 1 5, " + "2 0 1, 1 0 1))'), ST_GeomFromText('POLYGON ((4 0 4, 6 1 4, 6 4 9, 6 1 3, " + "4 0 4))'), 0.5)") + actual_df_default = self.spark.sql("SELECT ST_HausdorffDistance(ST_GeomFromText('POLYGON ((1 0 1, 1 1 2, " + "2 1 5, " + "2 0 1, 1 0 1))'), ST_GeomFromText('POLYGON ((4 0 4, 6 1 4, 6 4 9, 6 1 3, " + "4 0 4))'))") + actual = actual_df.take(1)[0][0] + actual_default = actual_df_default.take(1)[0][0] + assert expected == actual + assert expected == actual_default + + diff --git a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala index f9929c56ad..50b5556736 100644 --- a/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala +++ b/sql/common/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala @@ -152,6 +152,7 @@ object Catalog { function[ST_NRings](), function[ST_Translate](0.0), function[ST_BoundingDiagonal](), + function[ST_HausdorffDistance](-1), // Expression for rasters function[RS_NormalizedDifference](), function[RS_Mean](), diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala index cc90ca01df..8f39b0659b 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala @@ -1016,3 +1016,10 @@ case class ST_BoundingDiagonal(inputExpressions: Seq[Expression]) copy(inputExpressions = newChildren) } } + +case class ST_HausdorffDistance(inputExpressions: Seq[Expression]) + extends InferredTernaryExpression(Functions.hausdorffDistance) with FoldableExpression { + protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { + copy(inputExpressions = newChildren) + } +} diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala index f603cf278e..13e6bd23d2 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/st_functions.scala @@ -333,4 +333,14 @@ object st_functions extends DataFrameAPI { def ST_BoundingDiagonal(geometry: String) = wrapExpression[ST_BoundingDiagonal](geometry) + def ST_HausdorffDistance(g1: Column, g2: Column) = wrapExpression[ST_HausdorffDistance](g1, g2, -1) + + def ST_HausdorffDistance(g1: String, g2: String) = wrapExpression[ST_HausdorffDistance](g1, g2, -1); + + def ST_HausdorffDistance(g1: Column, g2: Column, densityFrac: Column) = wrapExpression[ST_HausdorffDistance](g1, g2, densityFrac); + + def ST_HausdorffDistance(g1: String, g2: String, densityFrac: Double) = wrapExpression[ST_HausdorffDistance](g1, g2, densityFrac); + + + } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala index 8e6ed18bd8..b5b788ed7e 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/dataFrameAPITestScala.scala @@ -1003,5 +1003,17 @@ class dataFrameAPITestScala extends TestBaseScala { val actual = wKTWriter.write(df.take(1)(0).get(0).asInstanceOf[Geometry]) assertEquals(expected, actual) } + + it("Passed ST_HausdorffDistance") { + val polyDf = sparkSession.sql("SELECT ST_GeomFromWKT('POLYGON ((1 2, 2 1, 2 0, 4 1, 1 2))') AS g1, " + + "ST_GeomFromWKT('MULTILINESTRING ((1 1, 2 1, 4 4, 5 5), (10 10, 11 11, 12 12, 14 14), (-11 -20, -11 -21, -15 -19))') AS g2") + val df = polyDf.select(ST_HausdorffDistance("g1", "g2", 0.05)) + val dfDefaultValue = polyDf.select(ST_HausdorffDistance("g1", "g2")) + val expected = 25.495097567963924 + val actual = df.take(1)(0).get(0).asInstanceOf[Double] + val actualDefaultValue = dfDefaultValue.take(1)(0).get(0).asInstanceOf[Double] + assert(expected == actual) + assert(expected == actualDefaultValue) + } } } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala index fa3cfc26c8..8a35315901 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functionTestScala.scala @@ -1987,4 +1987,25 @@ class functionTestScala extends TestBaseScala with Matchers with GeometrySample assertEquals(expected, actual) } } + + it ("should pass ST_HausdorffDistance") { + val geomTestCases = Map ( + ("'LINESTRING (1 2, 1 5, 2 6, 1 2)'", "'POINT (10 34)'", 0.34) -> (33.24154027718932, 33.24154027718932), + ("'LINESTRING (1 0, 1 1, 2 1, 2 0, 1 0)'", "'POINT EMPTY'", 0.23) -> (0.0, 0.0), + ("'POLYGON ((1 2, 2 1, 2 0, 4 1, 1 2))'", "'MULTIPOINT ((1 0), (40 10), (-10 -40))'", 0.0001) -> (41.7612260356422, 41.7612260356422) + ) + for (((geom), expectedResult) <- geomTestCases) { + val geom1 = geom._1 + val geom2 = geom._2 + val densityFrac = geom._3 + val df = sparkSession.sql(s"SELECT ST_HausdorffDistance(ST_GeomFromWKT($geom1), ST_GeomFromWKT($geom2), $densityFrac) AS dist") + val dfDefaultValue = sparkSession.sql(s"SELECT ST_HausdorffDistance(ST_GeomFromWKT($geom1), ST_GeomFromWKT($geom2)) as dist") + val actual = df.take(1)(0).get(0).asInstanceOf[Double] + val actualDefaultValue = dfDefaultValue.take(1)(0).get(0).asInstanceOf[Double] + val expected = expectedResult._1 + val expectedDefaultValue = expectedResult._2 + assert(expected == actual) + assert(expectedDefaultValue == actualDefaultValue) + } + } } From be962589fc7d4acb85847129270509b18a9aa3a3 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Wed, 28 Jun 2023 13:34:58 -0700 Subject: [PATCH 38/41] Support optimized join for ST_HausdorffDistance --- .../sedona_sql/expressions/Functions.scala | 8 +--- .../strategy/join/JoinQueryDetector.scala | 26 +++++++++++++ .../sedona/sql/BroadcastIndexJoinSuite.scala | 37 +++++++++++++++++++ .../org/apache/sedona/sql/TestBaseScala.scala | 22 +++++++++++ .../sedona/sql/predicateJoinTestScala.scala | 34 +++++++++++++++++ 5 files changed, 120 insertions(+), 7 deletions(-) diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala index 26b3a50593..a1344cfa1a 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/Functions.scala @@ -1036,7 +1036,7 @@ case class ST_BoundingDiagonal(inputExpressions: Seq[Expression]) } case class ST_HausdorffDistance(inputExpressions: Seq[Expression]) - extends InferredExpression(inferrableFunction3(Functions.hausdorffDistance)) { + extends InferredExpression(inferrableFunction3(Functions.hausdorffDistance), inferrableFunction2(Functions.hausdorffDistance)) { protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { copy(inputExpressions = newChildren) } @@ -1049,9 +1049,3 @@ case class GeometryType(inputExpressions: Seq[Expression]) } } -case class ST_HausdorffDistance(inputExpressions: Seq[Expression]) - extends InferredTernaryExpression(Functions.hausdorffDistance) with FoldableExpression { - protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = { - copy(inputExpressions = newChildren) - } -} diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/JoinQueryDetector.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/JoinQueryDetector.scala index 8a8f411b0d..d6d4a2bccc 100644 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/JoinQueryDetector.scala +++ b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/JoinQueryDetector.scala @@ -149,6 +149,32 @@ class JoinQueryDetector(sparkSession: SparkSession) extends Strategy { Some(JoinQueryDetection(left, right, leftShape, rightShape, SpatialPredicate.INTERSECTS, true, condition, Some(distance))) case Some(And(_, LessThan(ST_DistanceSpheroid(Seq(leftShape, rightShape)), distance))) => Some(JoinQueryDetection(left, right, leftShape, rightShape, SpatialPredicate.INTERSECTS, true, condition, Some(distance))) + //ST_HausdorffDistanceDefault + case Some(LessThanOrEqual(ST_HausdorffDistance(Seq(leftShape, rightShape)), distance)) => + Some(JoinQueryDetection(left, right, leftShape, rightShape, SpatialPredicate.INTERSECTS, false, condition, Some(distance))) + case Some(And(LessThanOrEqual(ST_HausdorffDistance(Seq(leftShape, rightShape)), distance), _)) => + Some(JoinQueryDetection(left, right, leftShape, rightShape, SpatialPredicate.INTERSECTS, false, condition, Some(distance))) + case Some(And(_, LessThanOrEqual(ST_HausdorffDistance(Seq(leftShape, rightShape)), distance))) => + Some(JoinQueryDetection(left, right, leftShape, rightShape, SpatialPredicate.INTERSECTS, false, condition, Some(distance))) + case Some(LessThan(ST_HausdorffDistance(Seq(leftShape, rightShape)), distance)) => + Some(JoinQueryDetection(left, right, leftShape, rightShape, SpatialPredicate.INTERSECTS, false, condition, Some(distance))) + case Some(And(LessThan(ST_HausdorffDistance(Seq(leftShape, rightShape)), distance), _)) => + Some(JoinQueryDetection(left, right, leftShape, rightShape, SpatialPredicate.INTERSECTS, false, condition, Some(distance))) + case Some(And(_, LessThan(ST_HausdorffDistance(Seq(leftShape, rightShape)), distance))) => + Some(JoinQueryDetection(left, right, leftShape, rightShape, SpatialPredicate.INTERSECTS, false, condition, Some(distance))) + //ST_HausdorffDistanceDensityFrac + case Some(LessThanOrEqual(ST_HausdorffDistance(Seq(leftShape, rightShape, densityFrac)), distance)) => + Some(JoinQueryDetection(left, right, leftShape, rightShape, SpatialPredicate.INTERSECTS, false, condition, Some(distance))) + case Some(And(LessThanOrEqual(ST_HausdorffDistance(Seq(leftShape, rightShape, densityFrac)), distance), _)) => + Some(JoinQueryDetection(left, right, leftShape, rightShape, SpatialPredicate.INTERSECTS, false, condition, Some(distance))) + case Some(And(_, LessThanOrEqual(ST_HausdorffDistance(Seq(leftShape, rightShape, densityFrac)), distance))) => + Some(JoinQueryDetection(left, right, leftShape, rightShape, SpatialPredicate.INTERSECTS, false, condition, Some(distance))) + case Some(LessThan(ST_HausdorffDistance(Seq(leftShape, rightShape, densityFrac)), distance)) => + Some(JoinQueryDetection(left, right, leftShape, rightShape, SpatialPredicate.INTERSECTS, false, condition, Some(distance))) + case Some(And(LessThan(ST_HausdorffDistance(Seq(leftShape, rightShape, densityFrac)), distance), _)) => + Some(JoinQueryDetection(left, right, leftShape, rightShape, SpatialPredicate.INTERSECTS, false, condition, Some(distance))) + case Some(And(_, LessThan(ST_HausdorffDistance(Seq(leftShape, rightShape, densityFrac)), distance))) => + Some(JoinQueryDetection(left, right, leftShape, rightShape, SpatialPredicate.INTERSECTS, false, condition, Some(distance))) case _ => None } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/BroadcastIndexJoinSuite.scala b/sql/common/src/test/scala/org/apache/sedona/sql/BroadcastIndexJoinSuite.scala index 6f5691280e..d2232278fd 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/BroadcastIndexJoinSuite.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/BroadcastIndexJoinSuite.scala @@ -350,6 +350,43 @@ class BroadcastIndexJoinSuite extends TestBaseScala { assert(distanceJoinDf.count() == expected) }) } + + it("Passed ST_HausdorffDistance with densityFrac <= distance in a broadcast join") { + val sampleCount = 100 + val distance = 1.0 + val densityFrac = 0.5 + val polygonDf = buildPolygonDf.limit(sampleCount).repartition(3) + val pointDf = buildPointDf.limit(sampleCount).repartition(5) + val expected = bruteForceDistanceJoinHausdorff(sampleCount, distance, 0.5, true) + + var distanceJoinDF = pointDf.alias("pointDf").join( + broadcast(polygonDf).alias("polygonDF"), expr(s"ST_HausdorffDistance(pointDf.pointshape, polygonDf.polygonshape, $densityFrac) <= $distance")) + assert(distanceJoinDF.queryExecution.sparkPlan.collect{case p: BroadcastIndexJoinExec => p}.size == 1) + assert(distanceJoinDF.count() == expected) + + distanceJoinDF = broadcast(pointDf).alias("pointDf").join(polygonDf.alias("polygonDf"), expr(s"ST_HausdorffDistance(pointDf.pointshape, polygonDf.polygonshape, $densityFrac) <= $distance")) + + assert(distanceJoinDF.queryExecution.sparkPlan.collect { case p: BroadcastIndexJoinExec => p }.size == 1) + assert(distanceJoinDF.count() == expected) + } + + it("Passed ST_HausdorffDistance <= distance in a broadcast join") { + val sampleCount = 200 + val distance = 2.0 + val polygonDf = buildPolygonDf.limit(sampleCount).repartition(3) + val pointDf = buildPointDf.limit(sampleCount).repartition(5) + val expected = bruteForceDistanceJoinHausdorff(sampleCount, distance, 0, true) + + var distanceJoinDF = pointDf.alias("pointDf").join( + broadcast(polygonDf).alias("polygonDF"), expr(s"ST_HausdorffDistance(pointDf.pointshape, polygonDf.polygonshape) <= $distance")) + assert(distanceJoinDF.queryExecution.sparkPlan.collect { case p: BroadcastIndexJoinExec => p }.size == 1) + assert(distanceJoinDF.count() == expected) + + distanceJoinDF = broadcast(pointDf).alias("pointDf").join(polygonDf.alias("polygonDf"), expr(s"ST_HausdorffDistance(pointDf.pointshape, polygonDf.polygonshape) <= $distance")) + + assert(distanceJoinDF.queryExecution.sparkPlan.collect { case p: BroadcastIndexJoinExec => p }.size == 1) + assert(distanceJoinDF.count() == expected) + } } describe("Sedona-SQL Broadcast Index Join Test for left semi joins") { diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala index 21f31a3b0f..dc6936ddda 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala @@ -21,6 +21,7 @@ package org.apache.sedona.sql import com.google.common.math.DoubleMath import org.apache.log4j.{Level, Logger} import org.apache.sedona.common.sphere.{Haversine, Spheroid} +import org.apache.sedona.common.Functions.hausdorffDistance import org.apache.sedona.spark.SedonaContext import org.apache.spark.sql.DataFrame import org.locationtech.jts.geom.{CoordinateSequence, CoordinateSequenceComparator} @@ -117,4 +118,25 @@ trait TestBaseScala extends FunSpec with BeforeAndAfterAll { }).sum } + protected def bruteForceDistanceJoinHausdorff(sampleCount: Int, distance: Double, densityFrac: Double, intersects: Boolean): Int = { + val inputPolygon = buildPolygonDf.limit(sampleCount).collect() + val inputPoint = buildPointDf.limit(sampleCount).collect() + inputPoint.map(row => { + val point = row.getAs[org.locationtech.jts.geom.Point](0) + inputPolygon.map(row => { + val polygon = row.getAs[org.locationtech.jts.geom.Polygon](0) + if (densityFrac == 0) { + if (intersects) + if (hausdorffDistance(point, polygon) <= distance) 1 else 0 + else + if (hausdorffDistance(point, polygon) < distance) 1 else 0 + } else { + if (intersects) + if (hausdorffDistance(point, polygon, densityFrac) <= distance) 1 else 0 + else + if (hausdorffDistance(point, polygon, densityFrac) < distance) 1 else 0 + } + }).sum + }).sum + } } diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/predicateJoinTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/predicateJoinTestScala.scala index 66ee94caad..2271e56288 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/predicateJoinTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/predicateJoinTestScala.scala @@ -403,7 +403,41 @@ class predicateJoinTestScala extends TestBaseScala { assert(distanceJoinDf.queryExecution.sparkPlan.collect { case p: DistanceJoinExec => p }.size === 1) assert(distanceJoinDf.count() == expected) }) + } + + it("Passed ST_HausdorffDistance in a spatial join") { + val sampleCount = 100 + val distanceCandidates = Seq(1, 2) + val densityFrac = 0.6 + val inputPoint = buildPointDf.limit(sampleCount).repartition(5) + val inputPolygon = buildPolygonDf.limit(sampleCount).repartition(3) + distanceCandidates.foreach(distance => { + + //DensityFrac specified, <= distance + val expectedDensityIntersects = bruteForceDistanceJoinHausdorff(sampleCount, distance, densityFrac, true) + val distanceDensityIntersectsDF = inputPoint.alias("pointDF").join(inputPolygon.alias("polygonDF"), expr(s"ST_HausdorffDistance(pointDF.pointshape, polygonDF.polygonshape, $densityFrac) <= $distance")) + assert(distanceDensityIntersectsDF.queryExecution.sparkPlan.collect { case p: DistanceJoinExec => p }.size === 1) + assert(distanceDensityIntersectsDF.count() == expectedDensityIntersects) + + //DensityFrac specified, < distance + val expectedDensityNoIntersect = bruteForceDistanceJoinHausdorff(sampleCount, distance, densityFrac, false) + val distanceDensityNoIntersectDF = inputPoint.alias("pointDF").join(inputPolygon.alias("polygonDF"), expr(s"ST_HausdorffDistance(pointDF.pointshape, polygonDF.polygonshape, $densityFrac) <= $distance")) + assert(distanceDensityNoIntersectDF.queryExecution.sparkPlan.collect { case p: DistanceJoinExec => p }.size === 1) + assert(distanceDensityNoIntersectDF.count() == expectedDensityNoIntersect) + + //DensityFrac not specified, <= distance + val expectedDefaultIntersects = bruteForceDistanceJoinHausdorff(sampleCount, distance, 0.0, true) + val distanceDefaultIntersectsDF = inputPoint.alias("pointDF").join(inputPolygon.alias("polygonDF"), expr(s"ST_HausdorffDistance(pointDF.pointshape, polygonDF.polygonshape, $densityFrac) <= $distance")) + assert(distanceDefaultIntersectsDF.queryExecution.sparkPlan.collect { case p: DistanceJoinExec => p }.size === 1) + assert(distanceDefaultIntersectsDF.count() == expectedDefaultIntersects) + + //DensityFrac not specified, < distance + val expectedDefaultNoIntersects = bruteForceDistanceJoinHausdorff(sampleCount, distance, 0.0, false) + val distanceDefaultNoIntersectsDF = inputPoint.alias("pointDF").join(inputPolygon.alias("polygonDF"), expr(s"ST_HausdorffDistance(pointDF.pointshape, polygonDF.polygonshape, $densityFrac) <= $distance")) + assert(distanceDefaultNoIntersectsDF.queryExecution.sparkPlan.collect { case p: DistanceJoinExec => p }.size === 1) + assert(distanceDefaultIntersectsDF.count() == expectedDefaultNoIntersects) + }) } } } From 0a4e1e767c7e722312b9d1e220a00178c702eb64 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Wed, 28 Jun 2023 14:30:59 -0700 Subject: [PATCH 39/41] Updated optimizer documentation for hausdorffDistance --- docs/api/sql/Optimizer.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/api/sql/Optimizer.md b/docs/api/sql/Optimizer.md index 3fa0242bb0..1fe54ef742 100644 --- a/docs/api/sql/Optimizer.md +++ b/docs/api/sql/Optimizer.md @@ -46,7 +46,7 @@ RangeJoin polygonshape#20: geometry, pointshape#43: geometry, false ## Distance join -Introduction: Find geometries from A and geometries from B such that the distance of each geometry pair is less or equal than a certain distance. It supports the planar Euclidean distance calculator `ST_Distance` and the meter-based geodesic distance calculators `ST_DistanceSpheroid` and `ST_DistanceSphere`. +Introduction: Find geometries from A and geometries from B such that the distance of each geometry pair is less or equal than a certain distance. It supports the planar Euclidean distance calculators `ST_Distance` and `ST_HausdorffDistance` and the meter-based geodesic distance calculators `ST_DistanceSpheroid` and `ST_DistanceSphere`. Spark SQL Example for planar Euclidean distance: @@ -57,6 +57,12 @@ FROM pointdf1, pointdf2 WHERE ST_Distance(pointdf1.pointshape1,pointdf2.pointshape2) < 2 ``` +```sql +SELECT * +FROM pointDf, polygonDF +WHERE ST_HausdorffDistance(pointDf.pointshape, polygonDf.polygonshape, 0.3) < 2 +``` + *Consider ==intersects within a certain distance==* ```sql SELECT * @@ -64,6 +70,12 @@ FROM pointdf1, pointdf2 WHERE ST_Distance(pointdf1.pointshape1,pointdf2.pointshape2) <= 2 ``` +```sql +SELECT * +FROM pointDf, polygonDF +WHERE ST_HausdorffDistance(pointDf.pointshape, polygonDf.polygonshape) <= 2 +``` + Spark SQL Physical plan: ``` == Physical Plan == @@ -75,7 +87,7 @@ DistanceJoin pointshape1#12: geometry, pointshape2#33: geometry, 2.0, true ``` !!!warning - If you use `ST_Distance` as the predicate, Sedona doesn't control the distance's unit (degree or meter). It is same with the geometry. If your coordinates are in the longitude and latitude system, the unit of `distance` should be degree instead of meter or mile. To change the geometry's unit, please either transform the coordinate reference system to a meter-based system. See [ST_Transform](Function.md#st_transform). If you don't want to transform your data, please consider using `ST_DistanceSpheroid` or `ST_DistanceSphere`. + If you use planar euclidean distance functions like `ST_Distance` or `ST_HausdorffDistance` as the predicate, Sedona doesn't control the distance's unit (degree or meter). It is same with the geometry. If your coordinates are in the longitude and latitude system, the unit of `distance` should be degree instead of meter or mile. To change the geometry's unit, please either transform the coordinate reference system to a meter-based system. See [ST_Transform](Function.md#st_transform). If you don't want to transform your data, please consider using `ST_DistanceSpheroid` or `ST_DistanceSphere`. Spark SQL Example for meter-based geodesic distance `ST_DistanceSpheroid` (works for `ST_DistanceSphere` too): @@ -126,7 +138,7 @@ BroadcastIndexJoin pointshape#52: geometry, BuildRight, BuildRight, false ST_Con +- FileScan csv ``` -This also works for distance joins with `ST_Distance`, `ST_DistanceSpheroid` or `ST_DistanceSphere`: +This also works for distance joins with `ST_Distance`, `ST_DistanceSpheroid`, `ST_DistanceSphere` or `ST_HausdorffDistance`: ```scala pointDf1.alias("pointDf1").join(broadcast(pointDf2).alias("pointDf2"), expr("ST_Distance(pointDf1.pointshape, pointDf2.pointshape) <= 2")) From b20716f141bb674bbe19548ff221cf878e7a70ba Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Thu, 29 Jun 2023 11:04:44 -0700 Subject: [PATCH 40/41] fix query mistake --- .../scala/org/apache/sedona/sql/predicateJoinTestScala.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/predicateJoinTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/predicateJoinTestScala.scala index 2271e56288..0f3f869fdf 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/predicateJoinTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/predicateJoinTestScala.scala @@ -422,7 +422,7 @@ class predicateJoinTestScala extends TestBaseScala { //DensityFrac specified, < distance val expectedDensityNoIntersect = bruteForceDistanceJoinHausdorff(sampleCount, distance, densityFrac, false) - val distanceDensityNoIntersectDF = inputPoint.alias("pointDF").join(inputPolygon.alias("polygonDF"), expr(s"ST_HausdorffDistance(pointDF.pointshape, polygonDF.polygonshape, $densityFrac) <= $distance")) + val distanceDensityNoIntersectDF = inputPoint.alias("pointDF").join(inputPolygon.alias("polygonDF"), expr(s"ST_HausdorffDistance(pointDF.pointshape, polygonDF.polygonshape, $densityFrac) < $distance")) assert(distanceDensityNoIntersectDF.queryExecution.sparkPlan.collect { case p: DistanceJoinExec => p }.size === 1) assert(distanceDensityNoIntersectDF.count() == expectedDensityNoIntersect) @@ -434,7 +434,7 @@ class predicateJoinTestScala extends TestBaseScala { //DensityFrac not specified, < distance val expectedDefaultNoIntersects = bruteForceDistanceJoinHausdorff(sampleCount, distance, 0.0, false) - val distanceDefaultNoIntersectsDF = inputPoint.alias("pointDF").join(inputPolygon.alias("polygonDF"), expr(s"ST_HausdorffDistance(pointDF.pointshape, polygonDF.polygonshape, $densityFrac) <= $distance")) + val distanceDefaultNoIntersectsDF = inputPoint.alias("pointDF").join(inputPolygon.alias("polygonDF"), expr(s"ST_HausdorffDistance(pointDF.pointshape, polygonDF.polygonshape, $densityFrac) < $distance")) assert(distanceDefaultNoIntersectsDF.queryExecution.sparkPlan.collect { case p: DistanceJoinExec => p }.size === 1) assert(distanceDefaultIntersectsDF.count() == expectedDefaultNoIntersects) }) From 7e777d86a189f419bde8392e14bf908712746c15 Mon Sep 17 00:00:00 2001 From: Nilesh Gajwani Date: Thu, 29 Jun 2023 11:33:59 -0700 Subject: [PATCH 41/41] Added distance candidates with results > 100 --- .../scala/org/apache/sedona/sql/predicateJoinTestScala.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/predicateJoinTestScala.scala b/sql/common/src/test/scala/org/apache/sedona/sql/predicateJoinTestScala.scala index 0f3f869fdf..efa7280b65 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/predicateJoinTestScala.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/predicateJoinTestScala.scala @@ -407,7 +407,7 @@ class predicateJoinTestScala extends TestBaseScala { it("Passed ST_HausdorffDistance in a spatial join") { val sampleCount = 100 - val distanceCandidates = Seq(1, 2) + val distanceCandidates = Seq(1, 2, 5, 10) val densityFrac = 0.6 val inputPoint = buildPointDf.limit(sampleCount).repartition(5) val inputPolygon = buildPolygonDf.limit(sampleCount).repartition(3)