diff --git a/indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerController.java b/indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerController.java index cd0a2eae..98cbb930 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerController.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerController.java @@ -120,9 +120,30 @@ public ResponseEntity deleteDocumentByUUID(@PathVariable("uuid") String return indexerMetadata.deleteDocumentByUUID(uuid); } + @PostMapping(path="/all-dataset", produces = "application/json") + @Operation(security = {@SecurityRequirement(name = "X-API-Key") }, description = "Index a dataset by UUID") + public SseEmitter indexAllCOData() { + final SseEmitter emitter = new SseEmitter(0L); // 0L means no timeout; + final IndexService.Callback callback = createCallback(emitter); + + new Thread(() -> { + try { + indexCloudOptimizedData.indexAllCloudOptimizedData(callback); + } + catch (IOException ioe) { + emitter.completeWithError(ioe); + } + finally { + emitter.complete(); + } + }).start(); + + return emitter; + } + @PostMapping(path="/{uuid}/dataset", produces = "application/json") @Operation(security = {@SecurityRequirement(name = "X-API-Key") }, description = "Index a dataset by UUID") - public SseEmitter indexDatasetByUUID(@PathVariable("uuid") String uuid) { + public SseEmitter indexCODataByUUID(@PathVariable("uuid") String uuid) { final SseEmitter emitter = new SseEmitter(0L); // 0L means no timeout; final IndexService.Callback callback = createCallback(emitter); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/model/DatasetEsEntry.java b/indexer/src/main/java/au/org/aodn/esindexer/model/DatasetEsEntry.java deleted file mode 100644 index f8514fe6..00000000 --- a/indexer/src/main/java/au/org/aodn/esindexer/model/DatasetEsEntry.java +++ /dev/null @@ -1,10 +0,0 @@ -package au.org.aodn.esindexer.model; - -import java.util.List; - -public record DatasetEsEntry( - String uuid, - String yearMonth, - List data) { - -} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/DataAccessServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/DataAccessServiceImpl.java index 34bb68f9..6dfc3ec3 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/DataAccessServiceImpl.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/DataAccessServiceImpl.java @@ -137,10 +137,17 @@ protected List toStacItemModel(String uuid, Map indexCloudOptimizedData(String uuid, LocalDate startDate, LocalDate endDate, IndexService.Callback callback); - + List indexAllCloudOptimizedData(IndexService.Callback callback) throws IOException; } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexCloudOptimizedServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexCloudOptimizedServiceImpl.java index ca083613..c72cc5c8 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/IndexCloudOptimizedServiceImpl.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/IndexCloudOptimizedServiceImpl.java @@ -1,5 +1,6 @@ package au.org.aodn.esindexer.service; +import au.org.aodn.esindexer.configuration.AppConstants; import au.org.aodn.esindexer.model.DatasetProvider; import au.org.aodn.stac.model.StacItemModel; import co.elastic.clients.elasticsearch.ElasticsearchClient; @@ -14,6 +15,7 @@ import org.springframework.context.annotation.ScopedProxyMode; import org.springframework.stereotype.Service; +import java.io.IOException; import java.time.LocalDate; import java.util.ArrayList; import java.util.List; @@ -27,6 +29,7 @@ public class IndexCloudOptimizedServiceImpl extends IndexServiceImpl implements protected DataAccessService dataAccessService; protected ObjectMapper indexerObjectMapper; protected String indexName; + protected ElasticSearchIndexService elasticSearchIndexService; @Lazy @Autowired @@ -37,13 +40,22 @@ public IndexCloudOptimizedServiceImpl( @Value("${elasticsearch.cloud_optimized_index.name}") String indexName, @Qualifier("portalElasticsearchClient") ElasticsearchClient elasticsearchClient, ObjectMapper indexerObjectMapper, - DataAccessService dataAccessService) { + DataAccessService dataAccessService, + ElasticSearchIndexService elasticSearchIndexService) { super(elasticsearchClient, indexerObjectMapper); this.indexName = indexName; this.indexerObjectMapper = indexerObjectMapper; this.dataAccessService = dataAccessService; + this.elasticSearchIndexService = elasticSearchIndexService; + } + + @Override + public List indexAllCloudOptimizedData(IndexService.Callback callback) throws IOException { + elasticSearchIndexService.createIndexFromMappingJSONFile(AppConstants.DATASET_INDEX_MAPPING_JSON_FILE, indexName); + // TODO: Right now just clear schema, we need to query all UUID and iterate one by one + return null; } /** * Index the cloud optimized data diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java index d4f8ff77..122676aa 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java @@ -107,41 +107,17 @@ public static synchronized void init() { * a properties call depth. * @param lng - lng * @param lat - lat - * @param depth - depth * @return - The map that represent the geoJson */ - public static Map createGeoJson(BigDecimal lng, BigDecimal lat, BigDecimal depth) { + public static Map createGeoShapeJson(BigDecimal lng, BigDecimal lat) { Point point = factory.createPoint(new Coordinate(lng.doubleValue(), lat.doubleValue())); - - try (StringWriter writer = new StringWriter()) { - geometryJson.write(point, writer); - - Map values = objectMapper.readValue(writer.toString(), HashMap.class); - - if(values == null) { - logger.warn("Convert geometry to JSON result in null, {}", writer.toString()); - } - - Map feature = new HashMap<>(); - Map properties = new HashMap<>(); - - properties.put("depth", depth); - - feature.put("type", "Feature"); - feature.put("properties", properties); - feature.put("geometry", values); - - return feature; - } - catch(Exception e) { - return null; - } + return createGeoShapeJson(List.of(List.of(point))); } /** * @param polygons - Assume to be EPSG:4326, as GeoJson always use this encoding. * @return - Map that represent the geojson */ - protected static Map createGeoJson(List> polygons) { + protected static Map createGeoShapeJson(List> polygons) { if(!polygons.isEmpty()) { // Convert list> to list @@ -366,7 +342,7 @@ protected static List> createGeometryWithoutLand(List createGeometryNoLandFrom(List> rawInput, Integer gridSize) { List> polygon = createGeometryWithoutLand(rawInput); - return (polygon != null && !polygon.isEmpty()) ? createGeoJson(polygon) : null; + return (polygon != null && !polygon.isEmpty()) ? createGeoShapeJson(polygon) : null; } /** * Create the spatial extents area given the XML info, it will not remove land area for speed reason. Otherwise, @@ -383,6 +359,6 @@ protected static List> createGeometryWithoutLand(List> polygon = createGeometryWithoutLand(rawInput); List> polygon = GeometryBase.findPolygonsFrom(GeometryBase.COORDINATE_SYSTEM_CRS84, rawInput); - return (polygon != null && !polygon.isEmpty()) ? createGeoJson(polygon) : null; + return (polygon != null && !polygon.isEmpty()) ? createGeoShapeJson(polygon) : null; } } diff --git a/indexer/src/main/resources/config_files/data_index_schema.json b/indexer/src/main/resources/config_files/data_index_schema.json index 0b6da0e0..28ad4c00 100644 --- a/indexer/src/main/resources/config_files/data_index_schema.json +++ b/indexer/src/main/resources/config_files/data_index_schema.json @@ -19,6 +19,9 @@ "properties" : { "type": "nested", "properties" : { + "lng": { "type": "double" }, + "lat": { "type": "double" }, + "depth": { "type": "double" }, "count": { "type": "double" }, "time": { "type": "date" } } diff --git a/indexer/src/test/java/au/org/aodn/esindexer/service/DataAccessServiceIT.java b/indexer/src/test/java/au/org/aodn/esindexer/service/DataAccessServiceIT.java index 8c8b265a..6f843b17 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/service/DataAccessServiceIT.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/service/DataAccessServiceIT.java @@ -84,7 +84,7 @@ public void verifyConversion1() throws IOException, JSONException, InterruptedEx .andExpect(method(HttpMethod.GET)) .andRespond(withSuccess("[]", MediaType.APPLICATION_JSON)); - SseEmitter emitter = controller.indexDatasetByUUID("35234913-aa3c-48ec-b9a4-77f822f66ef8"); + SseEmitter emitter = controller.indexCODataByUUID("35234913-aa3c-48ec-b9a4-77f822f66ef8"); CountDownLatch latch = new CountDownLatch(1); latch.await(5, TimeUnit.SECONDS); diff --git a/indexer/src/test/java/au/org/aodn/esindexer/service/DataAccessServiceTest.java b/indexer/src/test/java/au/org/aodn/esindexer/service/DataAccessServiceTest.java index 6b5d498d..f64f224a 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/service/DataAccessServiceTest.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/service/DataAccessServiceTest.java @@ -75,12 +75,10 @@ public void verifyToStacItemModel() throws IOException { Assertions.assertTrue(t.isPresent(), "Target found"); Assertions.assertEquals(15L, t.get().getProperties().get("count")); - Assertions.assertNotNull(t.get().getGeometry().get("properties"), "geometry properties no null"); - Assertions.assertInstanceOf(Map.class, t.get().getGeometry().get("properties")); - - Map properties = (Map)t.get().getGeometry().get("properties"); + Map properties = t.get().getProperties(); // The depth is a BigDecimal, so we do a toString() will force it print the .00 which is what we want // to check it contains two decimal - Assertions.assertEquals("530.00", properties.get("depth").toString()); + Assertions.assertEquals(530.0, properties.get("depth")); + Assertions.assertEquals(170.33, properties.get("lng")); } } diff --git a/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java b/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java index 0d26d668..ec3e8ede 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java @@ -139,29 +139,25 @@ public void verifyLandStrippedFromSpatialExtentsWithReducerOn() throws IOExcepti assertEquals(118.0, ncoors[4].getX(), 0.00); assertEquals(-36.0, ncoors[4].getY(), 0.00); } - + /** + * Given a point call this function return a GeometryCollection contain a single point + */ @Test public void verifyCreateJsonPoint() { - Map item = GeometryUtils.createGeoJson( + Map item = GeometryUtils.createGeoShapeJson( BigDecimal.valueOf(1.2), - BigDecimal.valueOf(2.2), - BigDecimal.valueOf(3.0) + BigDecimal.valueOf(2.2) ); Assertions.assertNotNull(item); - Assertions.assertEquals("Feature", item.get("type")); - Assertions.assertInstanceOf(Map.class, item.get("geometry")); + Assertions.assertEquals("GeometryCollection", item.get("type")); + Assertions.assertInstanceOf(List.class, item.get("geometries")); - Map geometry = (Map)item.get("geometry"); - Assertions.assertInstanceOf(List.class, geometry.get("coordinates")); + List> geometries = (List>)item.get("geometries"); + Assertions.assertInstanceOf(List.class, geometries.get(0).get("coordinates")); - Assertions.assertInstanceOf(List.class, geometry.get("coordinates")); - List coors = (List)geometry.get("coordinates"); + List coors = (List)geometries.get(0).get("coordinates"); Assertions.assertEquals(1.2, coors.get(0)); Assertions.assertEquals(2.2, coors.get(1)); - - Assertions.assertInstanceOf(Map.class, item.get("properties")); - Map properties = (Map)item.get("properties"); - Assertions.assertEquals(BigDecimal.valueOf(3.0), properties.get("depth")); } } diff --git a/indexer/src/test/resources/canned/dataservice/35234913-aa3c-48ec-b9a4-77f822f66ef8/StacItem1.json b/indexer/src/test/resources/canned/dataservice/35234913-aa3c-48ec-b9a4-77f822f66ef8/StacItem1.json index 4884862b..8f829120 100644 --- a/indexer/src/test/resources/canned/dataservice/35234913-aa3c-48ec-b9a4-77f822f66ef8/StacItem1.json +++ b/indexer/src/test/resources/canned/dataservice/35234913-aa3c-48ec-b9a4-77f822f66ef8/StacItem1.json @@ -1,21 +1,23 @@ { "id": "35234913-aa3c-48ec-b9a4-77f822f66ef8|2024-02|170.33|-33.87|530.00", "geometry": { - "geometry": { - "coordinates": [ - 170.33, - -33.87 - ], - "type": "Point" - }, - "type": "Feature", - "properties": { - "depth": 530.0 - } + "geometries": [ + { + "type": "Point", + "coordinates": [ + 170.33, + -33.87 + ] + } + ], + "type": "GeometryCollection" }, "properties": { + "time": "2024-02-01T00:00:00Z", "count": 15, - "time": "2024-02-01T00:00:00Z" + "depth": 530.0, + "lat": -33.87, + "lng": 170.33 }, "collection": "35234913-aa3c-48ec-b9a4-77f822f66ef8", "stac_version": "1.0.0",