Skip to content

Commit

Permalink
add testVectorToRasterTemporal Open-EO/openeo-geopyspark-driver#663
Browse files Browse the repository at this point in the history
  • Loading branch information
@JeroenVerstraelen
JeroenVerstraelen committed Mar 4, 2024
1 parent 04ec90f commit 7d6ead6
Show file tree
Hide file tree
Showing 2 changed files with 165 additions and 15 deletions.
76 changes: 76 additions & 0 deletions ...t/resources/org/openeo/geotrellis/geometries/input_vector_cube_temporal.geojson
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
{
"type": "FeatureCollection",
"features": [{
"id": "0",
"type": "Feature",
"properties": {
"2017-09-25T11:37:00Z~0": 1,
"2017-09-25T11:37:00Z~1": 2,
"2017-09-30T00:37:00Z~0": null,
"2017-09-30T00:37:00Z~1": null,
"2017-10-25T11:37:00Z~0": 2,
"2017-10-25T11:37:00Z~1": 1
},
"geometry": {
"type": "Polygon",
"coordinates": [
[
[5.040, 51.230],
[5.040, 51.235],
[5.050, 51.235],
[5.050, 51.230],
[5.040, 51.230]
]
]
}
},
{
"id": "1",
"type": "Feature",
"properties": {
"2017-09-25T11:37:00Z~0": 3,
"2017-09-25T11:37:00Z~1": 4,
"2017-09-30T00:37:00Z~0": null,
"2017-09-30T00:37:00Z~1": null,
"2017-10-25T11:37:00Z~0": 3,
"2017-10-25T11:37:00Z~1": 4
},
"geometry": {
"type": "Polygon",
"coordinates": [
[
[5.020, 51.225],
[5.020, 51.220],
[5.030, 51.220],
[5.030, 51.225],
[5.020, 51.225]
]
]
}
},
{
"id": "2",
"type": "Feature",
"properties": {
"2017-09-25T11:37:00Z~0": 5,
"2017-09-25T11:37:00Z~1": 6,
"2017-09-30T00:37:00Z~0": null,
"2017-09-30T00:37:00Z~1": null,
"2017-10-25T11:37:00Z~0": 5,
"2017-10-25T11:37:00Z~1": 6
},
"geometry": {
"type": "Polygon",
"coordinates": [
[
[5.020, 51.245],
[5.020, 51.230],
[5.030, 51.230],
[5.030, 51.245],
[5.020, 51.245]
]
]
}
}
]
}
104 changes: 89 additions & 15 deletions openeo-geotrellis/src/test/scala/org/openeo/geotrellis/vector/VectorCubeMethodsTest.scala
View file
Original file line number Diff line number Diff line change
@@ -1,33 +1,72 @@
package org.openeo.geotrellis.vector

import geotrellis.layer.SpatialKey
import geotrellis.layer.{SpaceTimeKey, SpatialKey}
import geotrellis.raster.MultibandTile
import geotrellis.spark.MultibandTileLayerRDD
import geotrellis.spark.util.SparkUtils
import geotrellis.vector.{Feature, Geometry}
import org.apache.hadoop.hdfs.HdfsConfiguration
import org.apache.hadoop.security.UserGroupInformation
import org.apache.spark.{SparkConf, SparkContext}
import org.junit.Assert.assertEquals
import org.junit.jupiter.api.{AfterAll, BeforeAll}
import org.junit.{AfterClass, BeforeClass, Test}
import org.openeo.geotrellis.LayerFixtures
import org.openeo.geotrellis.LayerFixtures.sentinel2B04Layer
import org.openeo.geotrellis.netcdf.NetCDFRDDWriter.{saveSingleNetCDFSpatial}
import org.openeo.geotrellis.vector.VectorCubeMethods.extractFeatures

import java.time.ZonedDateTime
import scala.collection.mutable

object VectorCubeMethodsTest {

private var sc: SparkContext = _
private var _sc: Option[SparkContext] = None

@BeforeClass
def setUpSpark(): Unit = {
sc = {
private def sc: SparkContext = {
if (_sc.isEmpty) {
val config = new HdfsConfiguration
//config.set("hadoop.security.authentication", "kerberos")
UserGroupInformation.setConfiguration(config)

val conf = new SparkConf().set("spark.driver.bindAddress", "127.0.0.1")
SparkUtils.createLocalSparkContext(sparkMaster = "local[2]", appName = getClass.getSimpleName, conf)
val conf = new SparkConf().setMaster("local[2]") //.set("spark.driver.bindAddress", "127.0.0.1")
.set("spark.kryoserializer.buffer.max", "512m")
.set("spark.rdd.compress", "true")
//conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
_sc = Some(SparkUtils.createLocalSparkContext(sparkMaster = "local[2]", appName = getClass.getSimpleName, conf))
}
_sc.get
}

@BeforeClass
def setUpSpark_BeforeClass(): Unit = sc

@BeforeAll
def setUpSpark_BeforeAll(): Unit = sc

var gotAfterAll = false

@AfterAll
def tearDownSpark_AfterAll(): Unit = {
gotAfterAll = true
maybeStopSpark()
}

var gotAfterClass = false

@AfterClass
def tearDownSpark(): Unit = {
sc.stop()
def tearDownSpark_AfterClass(): Unit = {
gotAfterClass = true;
maybeStopSpark()
}

def maybeStopSpark(): Unit = {
if (gotAfterAll && gotAfterClass) {
if (_sc.isDefined) {
_sc.get.stop()
_sc = None
}
}
}
}

Expand All @@ -37,19 +76,54 @@ class VectorCubeMethodsTest {
@Test def testVectorToRaster(): Unit = {
val path = getClass.getResource("/org/openeo/geotrellis/geometries/input_vector_cube.geojson").getPath
val targetDataCube = sentinel2B04Layer
val cube: MultibandTileLayerRDD[SpatialKey] = VectorCubeMethods.vectorToRaster(path, targetDataCube)
val cube: MultibandTileLayerRDD[SpatialKey] = VectorCubeMethods.vectorToRasterSpatial(path, targetDataCube)
assertEquals(targetDataCube.metadata.crs, cube.metadata.crs)
assertEquals(targetDataCube.metadata.cellheight.toInt, cube.metadata.cellheight.toInt)
assertEquals(targetDataCube.metadata.cellwidth.toInt, cube.metadata.cellwidth.toInt)
assertEquals(targetDataCube.metadata.bounds.get.toSpatial, cube.metadata.bounds)
assertEquals(targetDataCube.metadata.extent, cube.metadata.extent)

val cubeTiles: Array[MultibandTile] = cube.collect().map(_._2)
val features: Seq[Feature[Geometry, Double]] = extractFeatures(path, targetDataCube.metadata.crs, targetDataCube.metadata.layout)
val actualValues: Set[Int] = cubeTiles.map(_.band(0).toArray().toSet).toSet.flatten.filter(_ != -2147483648)
val expectedValues = features.map(_.data).toSet
assertEquals(expectedValues, actualValues)
// saveSingleNetCDFSpatial(cube, "/tmp/testVectorToRaster.nc", new util.ArrayList(util.Arrays.asList("Band1")), null, null, 1)
val features: Map[String, Seq[(String, mutable.Buffer[Feature[Geometry, Double]])]] = extractFeatures(path, targetDataCube.metadata.crs, targetDataCube.metadata.layout)

assert(features.keys.size == 1)
assert(features.keys.head == "")
val featureBands: Seq[(String, mutable.Buffer[Feature[Geometry, Double]])] = features("")
for (tile <- cubeTiles) {
val featureBandValues: Seq[Set[Double]] = featureBands.map(_._2.map(_.data).toSet.filter(!_.isNaN)).toSeq
val cubeBandValues: Seq[Set[Int]] = tile.bands.map(_.toArray().toSet.filter(_ != -2147483648))
assertEquals(featureBandValues, cubeBandValues)
}

// saveSingleNetCDFSpatial(cube, "/tmp/testVectorToRasterSpatial.nc", new java.util.ArrayList(java.util.Arrays.asList("Band1")), null, null, 1)
}

@Test
def testVectorToRasterTemporal(): Unit = {
val path = getClass.getResource("/org/openeo/geotrellis/geometries/input_vector_cube_temporal.geojson").getPath
val targetDataCube = LayerFixtures.sentinel2B04Layer
val cube: MultibandTileLayerRDD[SpaceTimeKey] = VectorCubeMethods.vectorToRasterTemporal(path, targetDataCube)
assertEquals(targetDataCube.metadata.crs, cube.metadata.crs)
assertEquals(targetDataCube.metadata.cellheight.toInt, cube.metadata.cellheight.toInt)
assertEquals(targetDataCube.metadata.cellwidth.toInt, cube.metadata.cellwidth.toInt)
assertEquals(targetDataCube.metadata.bounds.get.toSpatial, cube.metadata.bounds.get.toSpatial)
assertEquals(targetDataCube.metadata.extent, cube.metadata.extent)

val cubeTiles: Array[(SpaceTimeKey, MultibandTile)] = cube.collect()
val features: Map[String, Seq[(String, mutable.Buffer[Feature[Geometry, Double]])]] = extractFeatures(path, targetDataCube.metadata.crs, targetDataCube.metadata.layout)
val featureDates = features.keys
val format = java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'")
val cubeDates = cubeTiles.map(_._1.time.format(format))
assertEquals(featureDates.toSet, cubeDates.toSet)

for (tile <- cubeTiles) {
val date: ZonedDateTime = tile._1.time
val featureBands: Seq[(String, mutable.Buffer[Feature[Geometry, Double]])] = features(date.format(format))
val featureBandValues: Seq[Set[Double]] = featureBands.map(_._2.map(_.data).toSet.filter(!_.isNaN)).toSeq
val cubeBandValues: Seq[Set[Int]] = tile._2.bands.map(_.toArray().toSet.filter(_ != -2147483648))
assertEquals(featureBandValues, cubeBandValues)
}

// saveSingleNetCDF(cube, "/tmp/testVectorToRasterTemporal.nc", new java.util.ArrayList(java.util.Arrays.asList("Band1", "Band2")), null, null, 1)
}
}

0 comments on commit 7d6ead6

Please sign in to comment.