Merge pull request #177 from aodn/bug/6092-use-geopoint

Bug/6092 use geo_shape

indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerController.java

@@ -120,9 +120,30 @@ public ResponseEntity<String> deleteDocumentByUUID(@PathVariable("uuid") String
         return indexerMetadata.deleteDocumentByUUID(uuid);
     }
 
+    @PostMapping(path="/all-dataset", produces = "application/json")
+    @Operation(security = {@SecurityRequirement(name = "X-API-Key") }, description = "Index a dataset by UUID")
+    public SseEmitter indexAllCOData() {
+        final SseEmitter emitter = new SseEmitter(0L); // 0L means no timeout;
+        final IndexService.Callback callback = createCallback(emitter);
+
+        new Thread(() -> {
+            try {
+                indexCloudOptimizedData.indexAllCloudOptimizedData(callback);
+            }
+            catch (IOException ioe) {
+                emitter.completeWithError(ioe);
+            }
+            finally {
+                emitter.complete();
+            }
+        }).start();
+
+        return emitter;
+    }
+
     @PostMapping(path="/{uuid}/dataset", produces = "application/json")
     @Operation(security = {@SecurityRequirement(name = "X-API-Key") }, description = "Index a dataset by UUID")
-    public SseEmitter indexDatasetByUUID(@PathVariable("uuid") String uuid)  {
+    public SseEmitter indexCODataByUUID(@PathVariable("uuid") String uuid)  {
 
         final SseEmitter emitter = new SseEmitter(0L); // 0L means no timeout;
         final IndexService.Callback callback = createCallback(emitter);

indexer/src/main/java/au/org/aodn/esindexer/service/DataAccessServiceImpl.java

@@ -137,10 +137,17 @@ protected List<StacItemModel> toStacItemModel(String uuid, Map<? extends CloudOp
                                             d.getKey().getDepth().toString()
                                     )
                             )
-                            .geometry(GeometryUtils.createGeoJson(d.getKey().getLongitude(), d.getKey().getLatitude(), d.getKey().getDepth()))
+                            // The elastic query cannot sort by geo_shape or geo_point, so need to flatten value in properties
+                            // this geometry is use for filtering
+                            .geometry(GeometryUtils.createGeoShapeJson(d.getKey().getLongitude(), d.getKey().getLatitude()))
                             .properties(Map.of(
+                                    // Fields dup here is use for aggregation, you must have the geo_shape to do spatial search
+                                    "depth", d.getKey().getDepth().doubleValue(),
+                                    "lng", d.getKey().getLongitude().doubleValue(),
+                                    "lat", d.getKey().getLatitude().doubleValue(),
                                     "count", d.getValue(),
-                                    "time", d.getKey().getZonedDateTime().format(DateTimeFormatter.ISO_ZONED_DATE_TIME)))
+                                    "time", d.getKey().getZonedDateTime().format(DateTimeFormatter.ISO_ZONED_DATE_TIME))
+                            )
                             .build()
                 )
                 .toList();

indexer/src/main/java/au/org/aodn/esindexer/service/IndexCloudOptimizedService.java

@@ -2,10 +2,11 @@
 
 import co.elastic.clients.elasticsearch.core.BulkResponse;
 
+import java.io.IOException;
 import java.time.LocalDate;
 import java.util.List;
 
 public interface IndexCloudOptimizedService extends IndexService {
     List<BulkResponse> indexCloudOptimizedData(String uuid, LocalDate startDate, LocalDate endDate, IndexService.Callback callback);
-
+    List<BulkResponse> indexAllCloudOptimizedData(IndexService.Callback callback) throws IOException;
 }

indexer/src/main/java/au/org/aodn/esindexer/service/IndexCloudOptimizedServiceImpl.java

@@ -1,5 +1,6 @@
 package au.org.aodn.esindexer.service;
 
+import au.org.aodn.esindexer.configuration.AppConstants;
 import au.org.aodn.esindexer.model.DatasetProvider;
 import au.org.aodn.stac.model.StacItemModel;
 import co.elastic.clients.elasticsearch.ElasticsearchClient;
@@ -14,6 +15,7 @@
 import org.springframework.context.annotation.ScopedProxyMode;
 import org.springframework.stereotype.Service;
 
+import java.io.IOException;
 import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.List;
@@ -27,6 +29,7 @@ public class IndexCloudOptimizedServiceImpl extends IndexServiceImpl implements
     protected DataAccessService dataAccessService;
     protected ObjectMapper indexerObjectMapper;
     protected String indexName;
+    protected ElasticSearchIndexService elasticSearchIndexService;
 
     @Lazy
     @Autowired
@@ -37,13 +40,22 @@ public IndexCloudOptimizedServiceImpl(
             @Value("${elasticsearch.cloud_optimized_index.name}") String indexName,
             @Qualifier("portalElasticsearchClient") ElasticsearchClient elasticsearchClient,
             ObjectMapper indexerObjectMapper,
-            DataAccessService dataAccessService) {
+            DataAccessService dataAccessService,
+            ElasticSearchIndexService elasticSearchIndexService) {
 
         super(elasticsearchClient, indexerObjectMapper);
 
         this.indexName = indexName;
         this.indexerObjectMapper = indexerObjectMapper;
         this.dataAccessService = dataAccessService;
+        this.elasticSearchIndexService = elasticSearchIndexService;
+    }
+
+    @Override
+    public List<BulkResponse> indexAllCloudOptimizedData(IndexService.Callback callback) throws IOException {
+        elasticSearchIndexService.createIndexFromMappingJSONFile(AppConstants.DATASET_INDEX_MAPPING_JSON_FILE, indexName);
+        // TODO: Right now just clear schema, we need to query all UUID and iterate one by one
+        return null;
     }
     /**
      * Index the cloud optimized data

indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java

@@ -107,41 +107,17 @@ public static synchronized void init() {
      * a properties call depth.
      * @param lng - lng
      * @param lat - lat
-     * @param depth - depth
      * @return - The map that represent the geoJson
      */
-    public static Map<?,?> createGeoJson(BigDecimal lng, BigDecimal lat, BigDecimal depth) {
+    public static Map<?,?> createGeoShapeJson(BigDecimal lng, BigDecimal lat) {
         Point point = factory.createPoint(new Coordinate(lng.doubleValue(), lat.doubleValue()));
-
-        try (StringWriter writer = new StringWriter()) {
-            geometryJson.write(point, writer);
-
-            Map<?, ?> values = objectMapper.readValue(writer.toString(), HashMap.class);
-
-            if(values == null)  {
-                logger.warn("Convert geometry to JSON result in null, {}", writer.toString());
-            }
-
-            Map<String, Object> feature = new HashMap<>();
-            Map<String, Object> properties = new HashMap<>();
-
-            properties.put("depth", depth);
-
-            feature.put("type", "Feature");
-            feature.put("properties", properties);
-            feature.put("geometry", values);
-
-            return feature;
-        }
-        catch(Exception e) {
-            return null;
-        }
+        return createGeoShapeJson(List.of(List.of(point)));
     }
     /**
      * @param polygons - Assume to be EPSG:4326, as GeoJson always use this encoding.
      * @return - Map that represent the geojson
      */
-    protected static Map<?,?> createGeoJson(List<List<Geometry>> polygons) {
+    protected static Map<?,?> createGeoShapeJson(List<List<Geometry>> polygons) {
 
         if(!polygons.isEmpty()) {
             // Convert list<list<polygon>> to list<polygon>
@@ -366,7 +342,7 @@ protected static List<List<Geometry>> createGeometryWithoutLand(List<List<Abstra
      */
     public static Map<?, ?> createGeometryNoLandFrom(List<List<AbstractEXGeographicExtentType>> rawInput, Integer gridSize) {
         List<List<Geometry>> polygon = createGeometryWithoutLand(rawInput);
-        return (polygon != null && !polygon.isEmpty()) ? createGeoJson(polygon) : null;
+        return (polygon != null && !polygon.isEmpty()) ? createGeoShapeJson(polygon) : null;
     }
     /**
      * Create the spatial extents area given the XML info, it will not remove land area for speed reason. Otherwise,
@@ -383,6 +359,6 @@ protected static List<List<Geometry>> createGeometryWithoutLand(List<List<Abstra
         // List<List<Geometry>> polygon = createGeometryWithoutLand(rawInput);
 
         List<List<Geometry>> polygon = GeometryBase.findPolygonsFrom(GeometryBase.COORDINATE_SYSTEM_CRS84, rawInput);
-        return (polygon != null && !polygon.isEmpty()) ? createGeoJson(polygon) : null;
+        return (polygon != null && !polygon.isEmpty()) ? createGeoShapeJson(polygon) : null;
     }
 }

indexer/src/main/resources/config_files/data_index_schema.json

@@ -19,6 +19,9 @@
       "properties" : {
         "type": "nested",
         "properties" : {
+          "lng": { "type": "double" },
+          "lat": { "type": "double" },
+          "depth": { "type": "double" },
           "count": { "type": "double" },
           "time": { "type": "date" }
         }

indexer/src/test/java/au/org/aodn/esindexer/service/DataAccessServiceIT.java

@@ -84,7 +84,7 @@ public void verifyConversion1() throws IOException, JSONException, InterruptedEx
                     .andExpect(method(HttpMethod.GET))
                     .andRespond(withSuccess("[]", MediaType.APPLICATION_JSON));
 
-            SseEmitter emitter = controller.indexDatasetByUUID("35234913-aa3c-48ec-b9a4-77f822f66ef8");
+            SseEmitter emitter = controller.indexCODataByUUID("35234913-aa3c-48ec-b9a4-77f822f66ef8");
 
             CountDownLatch latch = new CountDownLatch(1);
             latch.await(5, TimeUnit.SECONDS);

indexer/src/test/java/au/org/aodn/esindexer/service/DataAccessServiceTest.java

@@ -75,12 +75,10 @@ public void verifyToStacItemModel() throws IOException {
         Assertions.assertTrue(t.isPresent(), "Target found");
         Assertions.assertEquals(15L, t.get().getProperties().get("count"));
 
-        Assertions.assertNotNull(t.get().getGeometry().get("properties"), "geometry properties no null");
-        Assertions.assertInstanceOf(Map.class, t.get().getGeometry().get("properties"));
-
-        Map<?, ?> properties = (Map<?,?>)t.get().getGeometry().get("properties");
+        Map<?, ?> properties = t.get().getProperties();
         // The depth is a BigDecimal, so we do a toString() will force it print the .00 which is what we want
         // to check it contains two decimal
-        Assertions.assertEquals("530.00", properties.get("depth").toString());
+        Assertions.assertEquals(530.0, properties.get("depth"));
+        Assertions.assertEquals(170.33, properties.get("lng"));
     }
 }

indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java

@@ -139,29 +139,25 @@ public void verifyLandStrippedFromSpatialExtentsWithReducerOn() throws IOExcepti
         assertEquals(118.0, ncoors[4].getX(), 0.00);
         assertEquals(-36.0, ncoors[4].getY(), 0.00);
     }
-
+    /**
+     * Given a point call this function return a GeometryCollection contain a single point
+     */
     @Test
     public void verifyCreateJsonPoint() {
-        Map<?,?> item = GeometryUtils.createGeoJson(
+        Map<?,?> item = GeometryUtils.createGeoShapeJson(
                 BigDecimal.valueOf(1.2),
-                BigDecimal.valueOf(2.2),
-                BigDecimal.valueOf(3.0)
+                BigDecimal.valueOf(2.2)
         );
 
         Assertions.assertNotNull(item);
-        Assertions.assertEquals("Feature", item.get("type"));
-        Assertions.assertInstanceOf(Map.class, item.get("geometry"));
+        Assertions.assertEquals("GeometryCollection", item.get("type"));
+        Assertions.assertInstanceOf(List.class, item.get("geometries"));
 
-        Map<?, ?> geometry = (Map<?,?>)item.get("geometry");
-        Assertions.assertInstanceOf(List.class, geometry.get("coordinates"));
+        List<Map<?,?>> geometries = (List<Map<?,?>>)item.get("geometries");
+        Assertions.assertInstanceOf(List.class, geometries.get(0).get("coordinates"));
 
-        Assertions.assertInstanceOf(List.class, geometry.get("coordinates"));
-        List<?> coors = (List<?>)geometry.get("coordinates");
+        List<?> coors = (List<?>)geometries.get(0).get("coordinates");
         Assertions.assertEquals(1.2, coors.get(0));
         Assertions.assertEquals(2.2, coors.get(1));
-
-        Assertions.assertInstanceOf(Map.class, item.get("properties"));
-        Map<?, ?> properties = (Map<?,?>)item.get("properties");
-        Assertions.assertEquals(BigDecimal.valueOf(3.0), properties.get("depth"));
     }
 }

indexer/src/test/resources/canned/dataservice/35234913-aa3c-48ec-b9a4-77f822f66ef8/StacItem1.json

@@ -1,21 +1,23 @@
 {
   "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2024-02|170.33|-33.87|530.00",
   "geometry": {
-    "geometry": {
-      "coordinates": [
-        170.33,
-        -33.87
-      ],
-      "type": "Point"
-    },
-    "type": "Feature",
-    "properties": {
-      "depth": 530.0
-    }
+    "geometries": [
+      {
+        "type": "Point",
+        "coordinates": [
+          170.33,
+          -33.87
+        ]
+      }
+    ],
+    "type": "GeometryCollection"
   },
   "properties": {
+    "time": "2024-02-01T00:00:00Z",
     "count": 15,
-    "time": "2024-02-01T00:00:00Z"
+    "depth": 530.0,
+    "lat": -33.87,
+    "lng": 170.33
   },
   "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8",
   "stac_version": "1.0.0",