Merge branch 'main' into move_shortcircuit_parameters

ayolab · web-flow · commit c67a7da3d0f4 · 2024-05-31T10:22:32.000+02:00
diff --git a/pom.xml b/pom.xml
@@ -182,7 +182,7 @@
             <groupId>org.springdoc</groupId>
             <artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
         </dependency>
-        
+
         <dependency>
             <groupId>com.jayway.jsonpath</groupId>
             <artifactId>json-path-assert</artifactId>
@@ -227,6 +227,11 @@
             <artifactId>spring-boot-starter-actuator</artifactId>
             <scope>runtime</scope>
         </dependency>
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-registry-prometheus</artifactId>
+            <scope>runtime</scope>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-actuator</artifactId>
diff --git a/src/main/java/org/gridsuite/study/server/StudyController.java b/src/main/java/org/gridsuite/study/server/StudyController.java
@@ -1106,10 +1106,9 @@ public ResponseEntity<Void> updateNetworkModification(@Parameter(description = "
     public ResponseEntity<Void> deleteNetworkModifications(@Parameter(description = "Study UUID") @PathVariable("studyUuid") UUID studyUuid,
                                                            @Parameter(description = "Node UUID") @PathVariable("nodeUuid") UUID nodeUuid,
                                                            @Parameter(description = "Network modification UUIDs") @RequestParam(name = "uuids", required = false) List<UUID> networkModificationUuids,
-                                                           @Parameter(description = "Delete only stashed modifications") @RequestParam(name = "onlyStashed", required = false, defaultValue = "false") Boolean onlyStashed,
                                                            @RequestHeader(HEADER_USER_ID) String userId) {
         studyService.assertCanModifyNode(studyUuid, nodeUuid);
-        studyService.deleteNetworkModifications(studyUuid, nodeUuid, networkModificationUuids, onlyStashed, userId);
+        studyService.deleteNetworkModifications(studyUuid, nodeUuid, networkModificationUuids, userId);
 
         return ResponseEntity.ok().build();
     }
diff --git a/src/main/java/org/gridsuite/study/server/SupervisionController.java b/src/main/java/org/gridsuite/study/server/SupervisionController.java
@@ -11,8 +11,10 @@
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
 import io.swagger.v3.oas.annotations.tags.Tag;
+import org.gridsuite.study.server.service.StudyService;
 import org.gridsuite.study.server.service.SupervisionService;
 
+import java.util.List;
 import java.util.UUID;
 
 import org.gridsuite.study.server.dto.ComputationType;
@@ -42,8 +44,11 @@ public class SupervisionController {
 
     private final SupervisionService supervisionService;
 
-    public SupervisionController(SupervisionService supervisionService) {
+    private final StudyService studyService;
+
+    public SupervisionController(SupervisionService supervisionService, StudyService studyService) {
         this.supervisionService = supervisionService;
+        this.studyService = studyService;
     }
 
     @DeleteMapping(value = "/computation/results")
@@ -96,6 +101,21 @@ public ResponseEntity<Long> deleteStudyIndexedEquipmentsAndTombstoned(@PathVaria
         return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body(supervisionService.deleteStudyIndexedEquipmentsAndTombstoned(studyUuid));
     }
 
+    @GetMapping(value = "/orphan_indexed_network_uuids")
+    @Operation(summary = "Get all orphan indexed equipments network uuids")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "The list of orphan indexed equipments network uuids")})
+    public ResponseEntity<List<UUID>> getOrphanIndexedEquipments() {
+        return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body(studyService.getAllOrphanIndexedEquipmentsNetworkUuids());
+    }
+
+    @DeleteMapping(value = "/studies/{networkUuid}/indexed-equipments-by-network-uuid")
+    @Operation(summary = "delete indexed equipments and tombstoned equipments for the given networkUuid")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "all indexed equipments and tombstoned equipments for the given networkUuid have been deleted")})
+    public ResponseEntity<Long> deleteNetworkUuidIndexedEquipmentsAndTombstoned(@PathVariable("networkUuid") UUID networkUuid) {
+        studyService.deleteEquipmentIndexes(networkUuid);
+        return ResponseEntity.ok().build();
+    }
+
     @DeleteMapping(value = "/studies/{studyUuid}/nodes/builds")
     @Operation(summary = "Invalidate node builds for the given study")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "all built nodes for the given study have been invalidated")})
diff --git a/src/main/java/org/gridsuite/study/server/dto/BasicEquipmentInfos.java b/src/main/java/org/gridsuite/study/server/dto/BasicEquipmentInfos.java
@@ -8,6 +8,7 @@
 
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.*;
+import lombok.experimental.FieldNameConstants;
 import lombok.experimental.SuperBuilder;
 import org.springframework.data.annotation.AccessType;
 import org.springframework.data.annotation.Id;
@@ -27,6 +28,7 @@
 @Setter
 @ToString
 @EqualsAndHashCode
+@FieldNameConstants
 @Schema(description = "Basic equipment infos")
 public class BasicEquipmentInfos {
     @Id
diff --git a/src/main/java/org/gridsuite/study/server/elasticsearch/EquipmentInfosService.java b/src/main/java/org/gridsuite/study/server/elasticsearch/EquipmentInfosService.java
@@ -7,17 +7,22 @@
 package org.gridsuite.study.server.elasticsearch;
 
 import co.elastic.clients.elasticsearch._types.FieldValue;
+import co.elastic.clients.elasticsearch._types.aggregations.*;
+import co.elastic.clients.elasticsearch._types.aggregations.Aggregation;
 import co.elastic.clients.elasticsearch._types.query_dsl.*;
 import com.powsybl.iidm.network.VariantManagerConstants;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.gridsuite.study.server.dto.BasicEquipmentInfos;
 import org.gridsuite.study.server.dto.EquipmentInfos;
 import org.gridsuite.study.server.dto.TombstonedEquipmentInfos;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.data.domain.PageRequest;
-import org.springframework.data.elasticsearch.client.elc.NativeQuery;
-import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder;
-import org.springframework.data.elasticsearch.client.elc.Queries;
+import org.springframework.data.elasticsearch.client.elc.*;
 import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
 import org.springframework.data.elasticsearch.core.SearchHit;
+import org.springframework.data.elasticsearch.core.SearchHits;
 import org.springframework.lang.NonNull;
 import org.springframework.stereotype.Service;
 
@@ -41,6 +46,8 @@ public enum FieldSelector {
 
     private static final int PAGE_MAX_SIZE = 400;
 
+    private static final int COMPOSITE_AGGREGATION_BATCH_SIZE = 1000;
+
     public static final Map<String, Integer> EQUIPMENT_TYPE_SCORES = Map.ofEntries(
             entry("SUBSTATION", 15),
             entry("VOLTAGE_LEVEL", 14),
@@ -72,6 +79,8 @@ public enum FieldSelector {
 
     private final ElasticsearchOperations elasticsearchOperations;
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(EquipmentInfosService.class);
+
     public EquipmentInfosService(EquipmentInfosRepository equipmentInfosRepository, TombstonedEquipmentInfosRepository tombstonedEquipmentInfosRepository, ElasticsearchOperations elasticsearchOperations) {
         this.equipmentInfosRepository = equipmentInfosRepository;
         this.tombstonedEquipmentInfosRepository = tombstonedEquipmentInfosRepository;
@@ -105,6 +114,101 @@ public long getEquipmentInfosCount() {
         return equipmentInfosRepository.count();
     }
 
+    private CompositeAggregation buildCompositeAggregation(String field, Map<String, FieldValue> afterKey) {
+        List<Map<String, CompositeAggregationSource>> sources = List.of(
+                Map.of(field, CompositeAggregationSource.of(s -> s.terms(t -> t.field(field + ".keyword")))
+                )
+        );
+
+        CompositeAggregation.Builder compositeAggregationBuilder = new CompositeAggregation.Builder()
+                .size(COMPOSITE_AGGREGATION_BATCH_SIZE)
+                .sources(sources);
+
+        if (afterKey != null) {
+            compositeAggregationBuilder.after(afterKey);
+        }
+
+        return compositeAggregationBuilder.build();
+    }
+
+    /**
+     * Constructs a NativeQuery with a composite aggregation.
+     *
+     * @param compositeName The name of the composite aggregation.
+     * @param compositeAggregation The composite aggregation configuration.
+     * @return A NativeQuery object configured with the specified composite aggregation.
+     */
+    private NativeQuery buildCompositeAggregationQuery(String compositeName, CompositeAggregation compositeAggregation) {
+        Aggregation aggregation = Aggregation.of(a -> a.composite(compositeAggregation));
+
+        return new NativeQueryBuilder()
+                .withAggregation(compositeName, aggregation)
+                .build();
+    }
+
+    /**
+     * This method is used to extract the results of a composite aggregation from Elasticsearch search hits.
+     *
+     * @param searchHits The search hits returned from an Elasticsearch query.
+     * @param compositeName The name of the composite aggregation.
+     * @return A Pair consisting of two elements:
+     *         The left element of the Pair is a list of maps, where each map represents a bucket's key. Each bucket is a result of the composite aggregation.
+     *         The right element of the Pair is the afterKey map, which is used for pagination in Elasticsearch.
+     *         If there are no more pages, the afterKey will be null.
+     * @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-composite-aggregation.html">Elasticsearch Composite Aggregation Documentation</a>
+     */
+    private Pair<List<Map<String, FieldValue>>, Map<String, FieldValue>> extractCompositeAggregationResults(SearchHits<EquipmentInfos> searchHits, String compositeName) {
+        ElasticsearchAggregations aggregations = (ElasticsearchAggregations) searchHits.getAggregations();
+
+        List<Map<String, FieldValue>> results = new ArrayList<>();
+        if (aggregations != null) {
+            Map<String, ElasticsearchAggregation> aggregationList = aggregations.aggregationsAsMap();
+            if (!aggregationList.isEmpty()) {
+                Aggregate aggregate = aggregationList.get(compositeName).aggregation().getAggregate();
+                if (aggregate.isComposite() && aggregate.composite() != null) {
+                    for (CompositeBucket bucket : aggregate.composite().buckets().array()) {
+                        Map<String, FieldValue> key = bucket.key();
+                        results.add(key);
+                    }
+                    return Pair.of(results, aggregate.composite().afterKey());
+                }
+            }
+        }
+        return Pair.of(results, null);
+    }
+
+    public List<UUID> getEquipmentInfosDistinctNetworkUuids() {
+        List<UUID> networkUuids = new ArrayList<>();
+        Map<String, FieldValue> afterKey = null;
+        String compositeName = "composite_agg";
+        String networkUuidField = BasicEquipmentInfos.Fields.networkUuid;
+
+        do {
+            CompositeAggregation compositeAggregation = buildCompositeAggregation(networkUuidField, afterKey);
+            NativeQuery query = buildCompositeAggregationQuery(compositeName, compositeAggregation);
+
+            SearchHits<EquipmentInfos> searchHits = elasticsearchOperations.search(query, EquipmentInfos.class);
+            Pair<List<Map<String, FieldValue>>, Map<String, FieldValue>> searchResults = extractCompositeAggregationResults(searchHits, compositeName);
+
+            searchResults.getLeft().stream()
+                    .map(result -> result.get(networkUuidField))
+                    .filter(Objects::nonNull)
+                    .map(FieldValue::stringValue)
+                    .map(UUID::fromString)
+                    .forEach(networkUuids::add);
+
+            afterKey = searchResults.getRight();
+        } while (afterKey != null && !afterKey.isEmpty());
+
+        return networkUuids;
+    }
+
+    public List<UUID> getOrphanEquipmentInfosNetworkUuids(List<UUID> networkUuidsInDatabase) {
+        List<UUID> networkUuids = getEquipmentInfosDistinctNetworkUuids();
+        networkUuids.removeAll(networkUuidsInDatabase);
+        return networkUuids;
+    }
+
     public long getTombstonedEquipmentInfosCount() {
         return tombstonedEquipmentInfosRepository.count();
     }
diff --git a/src/main/java/org/gridsuite/study/server/notification/NotificationService.java b/src/main/java/org/gridsuite/study/server/notification/NotificationService.java
@@ -78,9 +78,12 @@ public class NotificationService {
     public static final String UPDATE_TYPE_INDEXATION_STATUS = "indexation_status_updated";
 
     public static final String MODIFICATIONS_CREATING_IN_PROGRESS = "creatingInProgress";
+    public static final String MODIFICATIONS_STASHING_IN_PROGRESS = "stashingInProgress";
+    public static final String MODIFICATIONS_RESTORING_IN_PROGRESS = "restoringInProgress";
     public static final String MODIFICATIONS_DELETING_IN_PROGRESS = "deletingInProgress";
     public static final String MODIFICATIONS_UPDATING_IN_PROGRESS = "updatingInProgress";
     public static final String MODIFICATIONS_UPDATING_FINISHED = "UPDATE_FINISHED";
+    public static final String MODIFICATIONS_DELETING_FINISHED = "DELETE_FINISHED";
 
     public static final String EVENTS_CRUD_CREATING_IN_PROGRESS = "eventCreatingInProgress";
     public static final String EVENTS_CRUD_DELETING_IN_PROGRESS = "eventDeletingInProgress";
@@ -344,6 +347,17 @@ public void emitEndModificationEquipmentNotification(UUID studyUuid, UUID parent
         );
     }
 
+    @PostCompletion
+    public void emitEndDeletionEquipmentNotification(UUID studyUuid, UUID parentNodeUuid, Collection<UUID> childrenUuids) {
+        sendUpdateMessage(MessageBuilder.withPayload("")
+                .setHeader(HEADER_STUDY_UUID, studyUuid)
+                .setHeader(HEADER_PARENT_NODE, parentNodeUuid)
+                .setHeader(HEADER_NODES, childrenUuids)
+                .setHeader(HEADER_UPDATE_TYPE, MODIFICATIONS_DELETING_FINISHED)
+                .build()
+        );
+    }
+
     public void emitStartEventCrudNotification(UUID studyUuid, UUID parentNodeUuid, Collection<UUID> childrenUuids, String crudType) {
         sendUpdateMessage(MessageBuilder.withPayload("")
                 .setHeader(HEADER_STUDY_UUID, studyUuid)
diff --git a/src/main/java/org/gridsuite/study/server/service/NetworkModificationService.java b/src/main/java/org/gridsuite/study/server/service/NetworkModificationService.java
@@ -126,13 +126,12 @@ public void deleteModifications(UUID groupUUid) {
         }
     }
 
-    public void deleteModifications(UUID groupUuid, List<UUID> modificationsUuids, boolean onlyStashed) {
+    public void deleteModifications(UUID groupUuid, List<UUID> modificationsUuids) {
         Objects.requireNonNull(groupUuid);
         var path = UriComponentsBuilder
                 .fromUriString(getNetworkModificationServerURI(false) + NETWORK_MODIFICATIONS_PATH)
                 .queryParam(UUIDS, modificationsUuids)
                 .queryParam(GROUP_UUID, groupUuid)
-                .queryParam(QUERY_PARAM_ONLY_STASHED, onlyStashed)
                 .buildAndExpand()
                 .toUriString();
         try {
diff --git a/src/main/java/org/gridsuite/study/server/service/StudyService.java b/src/main/java/org/gridsuite/study/server/service/StudyService.java
@@ -237,6 +237,16 @@ public List<CreatedStudyBasicInfos> getStudies() {
                 .collect(Collectors.toList());
     }
 
+    public List<UUID> getStudiesNetworkUuids() {
+        return studyRepository.findAll().stream()
+                .map(StudyEntity::getNetworkUuid)
+                .toList();
+    }
+
+    public List<UUID> getAllOrphanIndexedEquipmentsNetworkUuids() {
+        return equipmentInfosService.getOrphanEquipmentInfosNetworkUuids(getStudiesNetworkUuids());
+    }
+
     public String getStudyCaseName(UUID studyUuid) {
         Objects.requireNonNull(studyUuid);
         StudyEntity study = studyRepository.findById(studyUuid).orElseThrow(() -> new StudyException(STUDY_NOT_FOUND));
@@ -1344,29 +1354,29 @@ public void changeModificationActiveState(@NonNull UUID studyUuid, @NonNull UUID
     }
 
     @Transactional
-    public void deleteNetworkModifications(UUID studyUuid, UUID nodeUuid, List<UUID> modificationsUuids, boolean onlyStashed, String userId) {
+    public void deleteNetworkModifications(UUID studyUuid, UUID nodeUuid, List<UUID> modificationsUuids, String userId) {
         List<UUID> childrenUuids = networkModificationTreeService.getChildren(nodeUuid);
         notificationService.emitStartModificationEquipmentNotification(studyUuid, nodeUuid, childrenUuids, NotificationService.MODIFICATIONS_DELETING_IN_PROGRESS);
         try {
             if (!networkModificationTreeService.getStudyUuidForNodeId(nodeUuid).equals(studyUuid)) {
                 throw new StudyException(NOT_ALLOWED);
             }
             UUID groupId = networkModificationTreeService.getModificationGroupUuid(nodeUuid);
-            networkModificationService.deleteModifications(groupId, modificationsUuids, onlyStashed);
+            networkModificationService.deleteModifications(groupId, modificationsUuids);
             if (modificationsUuids != null) {
                 networkModificationTreeService.removeModificationsToExclude(nodeUuid, modificationsUuids);
             }
             updateStatuses(studyUuid, nodeUuid, false, false, false);
         } finally {
-            notificationService.emitEndModificationEquipmentNotification(studyUuid, nodeUuid, childrenUuids);
+            notificationService.emitEndDeletionEquipmentNotification(studyUuid, nodeUuid, childrenUuids);
         }
         notificationService.emitElementUpdated(studyUuid, userId);
     }
 
     @Transactional
     public void stashNetworkModifications(UUID studyUuid, UUID nodeUuid, List<UUID> modificationsUuids, String userId) {
         List<UUID> childrenUuids = networkModificationTreeService.getChildren(nodeUuid);
-        notificationService.emitStartModificationEquipmentNotification(studyUuid, nodeUuid, childrenUuids, NotificationService.MODIFICATIONS_DELETING_IN_PROGRESS);
+        notificationService.emitStartModificationEquipmentNotification(studyUuid, nodeUuid, childrenUuids, NotificationService.MODIFICATIONS_STASHING_IN_PROGRESS);
         try {
             if (!networkModificationTreeService.getStudyUuidForNodeId(nodeUuid).equals(studyUuid)) {
                 throw new StudyException(NOT_ALLOWED);
@@ -1384,7 +1394,7 @@ public void stashNetworkModifications(UUID studyUuid, UUID nodeUuid, List<UUID>
     @Transactional
     public void restoreNetworkModifications(UUID studyUuid, UUID nodeUuid, List<UUID> modificationsUuids, String userId) {
         List<UUID> childrenUuids = networkModificationTreeService.getChildren(nodeUuid);
-        notificationService.emitStartModificationEquipmentNotification(studyUuid, nodeUuid, childrenUuids, NotificationService.MODIFICATIONS_DELETING_IN_PROGRESS);
+        notificationService.emitStartModificationEquipmentNotification(studyUuid, nodeUuid, childrenUuids, NotificationService.MODIFICATIONS_RESTORING_IN_PROGRESS);
         try {
             if (!networkModificationTreeService.getStudyUuidForNodeId(nodeUuid).equals(studyUuid)) {
                 throw new StudyException(NOT_ALLOWED);
diff --git a/src/test/java/org/gridsuite/study/server/EquipmentInfosServiceTests.java b/src/test/java/org/gridsuite/study/server/EquipmentInfosServiceTests.java
diff --git a/src/test/java/org/gridsuite/study/server/NetworkModificationTest.java b/src/test/java/org/gridsuite/study/server/NetworkModificationTest.java
diff --git a/src/test/java/org/gridsuite/study/server/StudyTest.java b/src/test/java/org/gridsuite/study/server/StudyTest.java

Original file line number	Diff line number	Diff line change
`@@ -126,13 +126,12 @@ public void deleteModifications(UUID groupUUid) {`
`126`	`126`	`}`
`127`	`127`	`}`
`128`	`128`
`129`		`- public void deleteModifications(UUID groupUuid, List<UUID> modificationsUuids, boolean onlyStashed) {`
	`129`	`+ public void deleteModifications(UUID groupUuid, List<UUID> modificationsUuids) {`
`130`	`130`	`Objects.requireNonNull(groupUuid);`
`131`	`131`	`var path = UriComponentsBuilder`
`132`	`132`	`.fromUriString(getNetworkModificationServerURI(false) + NETWORK_MODIFICATIONS_PATH)`
`133`	`133`	`.queryParam(UUIDS, modificationsUuids)`
`134`	`134`	`.queryParam(GROUP_UUID, groupUuid)`
`135`		`- .queryParam(QUERY_PARAM_ONLY_STASHED, onlyStashed)`
`136`	`135`	`.buildAndExpand()`
`137`	`136`	`.toUriString();`
`138`	`137`	`try {`