DMP-4538 ARM RPO - Endpoint changes (#2413)

Co-authored-by: hmcts-jenkins-cnp <60659747+hmcts-jenkins-cnp[bot]@users.noreply.github.com>

charts/darts-api/Chart.yaml

@@ -4,7 +4,7 @@ appVersion: "1.0"
 description: A Helm chart for darts-api App
 name: darts-api
 home: https://github.com/hmcts/darts-api
-version: 0.0.95
+version: 0.0.96
 maintainers:
   - name: HMCTS darts team
 dependencies:

charts/darts-api/values.yaml

@@ -115,6 +115,7 @@ java:
     ACTIVE_DIRECTORY_B2C_AUTH_URI: https://hmctsstgextid.b2clogin.com/hmctsstgextid.onmicrosoft.com
     ARM_URL: http://darts-stub-services.{{ .Values.global.environment }}.platform.hmcts.net
     FEIGN_LOG_LEVEL: none
+    ARM_RPO_THREAD_SLEEP_DURATION: 5s
 
 function:
   scaleType: Job
@@ -228,6 +229,7 @@ function:
     ARM_URL: http://darts-stub-services.{{ .Values.global.environment }}.platform.hmcts.net
     FEIGN_LOG_LEVEL: none
     IS_MOCK_ARM_RPO_DOWNLOAD_CSV: false
+    ARM_RPO_THREAD_SLEEP_DURATION: 5s
 
   secrets:
     DARTS_API_DB_CONNECTION_STRING:

src/integrationTest/java/uk/gov/hmcts/darts/arm/client/ArmRpoClientIntTest.java

@@ -50,15 +50,10 @@ class ArmRpoClientIntTest extends IntegrationBaseWithWiremock {
     private static final String EXPECTED_RESPONSE_DIRECTORY = BASE_JSON_DIRECTORY + "expectedResponse/";
 
     private static final String URL_PREFIX = "/api/v1/";
-
-
-    private static final String GET_RECORD_MANAGEMENT_MATTER_PATH = "/api/v1/getRecordManagementMatter";
-    private static final String GET_STORAGE_ACCOUNTS_PATH = "/api/v1/getStorageAccounts";
-
+
     @Autowired
     private ArmRpoClient armRpoClient;
 
-
     private static Stream<Arguments> genericArmRpoClientTestArguments() {
         return Stream.of(
             Arguments.of("getRecordManagementMatter", (BiFunction<ArmRpoClient, String, ClientCallable>) (armRpoClient, bearerAuth) -> {

src/integrationTest/java/uk/gov/hmcts/darts/arm/rpo/ArmRpoApiGetExtendedSearchesByMatterIntTest.java

@@ -40,6 +40,7 @@ void getExtendedSearchesByMatterSuccess() {
         ExtendedSearchesByMatterResponse.Search search = new ExtendedSearchesByMatterResponse.Search();
         search.setTotalCount(4);
         search.setName(PRODUCTION_NAME);
+        search.setIsSaved(true);
         ExtendedSearchesByMatterResponse.SearchDetail searchDetail = new ExtendedSearchesByMatterResponse.SearchDetail();
         searchDetail.setSearch(search);
         extendedSearchesByMatterResponse.setSearches(List.of(searchDetail));

src/integrationTest/java/uk/gov/hmcts/darts/arm/service/ArmRpoPollServiceIntTest.java

@@ -138,6 +138,56 @@ void pollArmRpo_shouldPollSuccessfullyWithSaveBackgroundCompleted() throws IOExc
 
     }
 
+    @Test
+    void pollArmRpo_shouldPollSuccessfullyWithGetExtendedSearchesByMatterInProgress() throws IOException {
+        // given
+        armRpoExecutionDetailEntity.setArmRpoStatus(ArmRpoHelper.inProgressRpoStatus());
+        armRpoExecutionDetailEntity.setArmRpoState(ArmRpoHelper.getExtendedSearchesByMatterRpoState());
+        armRpoExecutionDetailEntity.setMatterId("MatterId");
+        armRpoExecutionDetailEntity.setSearchId("SearchId");
+        armRpoExecutionDetailEntity.setStorageAccountId("StorageAccountId");
+        armRpoExecutionDetailEntity.setProductionId(PRODUCTION_ID);
+        armRpoExecutionDetailEntity = dartsPersistence.save(armRpoExecutionDetailEntity);
+
+        when(armApiService.getArmBearerToken()).thenReturn(BEARER_TOKEN);
+        when(armRpoClient.getExtendedSearchesByMatter(any(), any()))
+            .thenReturn(getExtendedSearchesByMatterResponse());
+        when(armRpoClient.getMasterIndexFieldByRecordClassSchema(any(), any()))
+            .thenReturn(getMasterIndexFieldByRecordClassSchemaResponse("propertyName", "ingestionDate"));
+        when(armRpoClient.createExportBasedOnSearchResultsTable(anyString(), any()))
+            .thenReturn(getCreateExportBasedOnSearchResultsTableResponse());
+        when(armRpoClient.getExtendedProductionsByMatter(anyString(), any()))
+            .thenReturn(getExtendedProductionsByMatterResponse());
+        when(armRpoClient.getProductionOutputFiles(any(), any()))
+            .thenReturn(getProductionOutputFilesResponse(PRODUCTION_ID));
+
+        when(armRpoDownloadProduction.downloadProduction(any(), any(), any()))
+            .thenReturn(getFeignResponse(200));
+
+        when(armRpoClient.removeProduction(any(), any()))
+            .thenReturn(getRemoveProductionResponse());
+
+        // when
+        armRpoPollService.pollArmRpo(false);
+
+        // then
+        var updatedArmRpoExecutionDetailEntity = dartsPersistence.getArmRpoExecutionDetailRepository().findById(armRpoExecutionDetailEntity.getId());
+        assertNotNull(updatedArmRpoExecutionDetailEntity);
+        assertEquals(ArmRpoHelper.removeProductionRpoState().getId(), updatedArmRpoExecutionDetailEntity.get().getArmRpoState().getId());
+        assertEquals(ArmRpoHelper.completedRpoStatus().getId(), updatedArmRpoExecutionDetailEntity.get().getArmRpoStatus().getId());
+
+        verify(armRpoClient).getExtendedSearchesByMatter(any(), any());
+        verify(armRpoClient).getMasterIndexFieldByRecordClassSchema(any(), any());
+        verify(armRpoClient).createExportBasedOnSearchResultsTable(anyString(), any());
+        verify(armRpoClient).getExtendedProductionsByMatter(anyString(), any());
+        verify(armRpoClient).getProductionOutputFiles(any(), any());
+        verify(armRpoDownloadProduction).downloadProduction(any(), any(), any());
+        verify(armRpoClient).removeProduction(any(), any());
+
+        verifyNoMoreInteractions(armRpoClient);
+
+    }
+
     @Test
     void pollArmRpo_shouldPollSuccessfullyWithSaveBackgroundCompletedCreateExportInProgress() throws IOException {
         // given
@@ -341,6 +391,7 @@ private ExtendedSearchesByMatterResponse getExtendedSearchesByMatterResponse() {
         ExtendedSearchesByMatterResponse.Search search = new ExtendedSearchesByMatterResponse.Search();
         search.setTotalCount(4);
         search.setName("DARTS_RPO_2024-08-13");
+        search.setIsSaved(true);
         ExtendedSearchesByMatterResponse.SearchDetail searchDetail = new ExtendedSearchesByMatterResponse.SearchDetail();
         searchDetail.setSearch(search);
         response.setSearches(List.of(searchDetail));

src/integrationTest/resources/tests/arm/client/ArmRpoClientIntTest/expectedResponse/getExtendedSearchesByMatter.json

@@ -8,7 +8,8 @@
     {
       "search": {
         "name": "DARTS_RPO_2024-08-13",
-        "totalCount": 5
+        "totalCount": 5,
+        "isSaved": true
       }
     }
   ]

src/main/java/uk/gov/hmcts/darts/arm/client/model/rpo/ExtendedSearchesByMatterResponse.java

@@ -31,6 +31,7 @@ public static class Search {
 
         private String name;
         private Integer totalCount;
+        private Boolean isSaved;
 
     }
 

src/main/java/uk/gov/hmcts/darts/arm/exception/ArmRpoGetExtendedSearchesByMatterIdException.java

@@ -0,0 +1,9 @@
+package uk.gov.hmcts.darts.arm.exception;
+
+import uk.gov.hmcts.darts.common.exception.DartsException;
+
+public class ArmRpoGetExtendedSearchesByMatterIdException extends DartsException {
+    public ArmRpoGetExtendedSearchesByMatterIdException(String message) {
+        super(message);
+    }
+}

src/main/java/uk/gov/hmcts/darts/arm/rpo/impl/ArmRpoApiImpl.java

@@ -34,6 +34,7 @@
 import uk.gov.hmcts.darts.arm.component.impl.GetExtendedSearchesByMatterRequestGenerator;
 import uk.gov.hmcts.darts.arm.config.ArmApiConfigurationProperties;
 import uk.gov.hmcts.darts.arm.exception.ArmRpoException;
+import uk.gov.hmcts.darts.arm.exception.ArmRpoGetExtendedSearchesByMatterIdException;
 import uk.gov.hmcts.darts.arm.helper.ArmRpoHelper;
 import uk.gov.hmcts.darts.arm.model.rpo.MasterIndexFieldByRecordClassSchema;
 import uk.gov.hmcts.darts.arm.rpo.ArmRpoApi;
@@ -53,6 +54,7 @@
 import java.util.List;
 import java.util.Objects;
 
+import static java.lang.Boolean.FALSE;
 import static java.util.Objects.isNull;
 
 @Service
@@ -78,6 +80,7 @@ public class ArmRpoApiImpl implements ArmRpoApi {
 
     @Override
     public void getRecordManagementMatter(String bearerToken, Integer executionId, UserAccountEntity userAccount) {
+        log.debug("getRecordManagementMatter called with executionId: {}", executionId);
         var armRpoExecutionDetailEntity = armRpoService.getArmRpoExecutionDetailEntity(executionId);
         armRpoService.updateArmRpoStateAndStatus(armRpoExecutionDetailEntity, ArmRpoHelper.getRecordManagementMatterRpoState(),
                                                  ArmRpoHelper.inProgressRpoStatus(), userAccount);
@@ -104,6 +107,7 @@ public void getRecordManagementMatter(String bearerToken, Integer executionId, U
 
     @Override
     public void getIndexesByMatterId(String bearerToken, Integer executionId, String matterId, UserAccountEntity userAccount) {
+        log.debug("getIndexesByMatterId called with executionId: {}", executionId);
         var armRpoExecutionDetailEntity = armRpoService.getArmRpoExecutionDetailEntity(executionId);
         armRpoService.updateArmRpoStateAndStatus(armRpoExecutionDetailEntity, ArmRpoHelper.getIndexesByMatterIdRpoState(),
                                                  ArmRpoHelper.inProgressRpoStatus(), userAccount);
@@ -142,6 +146,7 @@ private IndexesByMatterIdRequest createIndexesByMatterIdRequest(String matterId)
 
     @Override
     public void getStorageAccounts(String bearerToken, Integer executionId, UserAccountEntity userAccount) {
+        log.debug("getStorageAccounts called with executionId: {}", executionId);
         var armRpoExecutionDetailEntity = armRpoService.getArmRpoExecutionDetailEntity(executionId);
         armRpoService.updateArmRpoStateAndStatus(armRpoExecutionDetailEntity, ArmRpoHelper.getStorageAccountsRpoState(),
                                                  ArmRpoHelper.inProgressRpoStatus(), userAccount);
@@ -187,6 +192,7 @@ public void getStorageAccounts(String bearerToken, Integer executionId, UserAcco
 
     @Override
     public void getProfileEntitlements(String bearerToken, Integer executionId, UserAccountEntity userAccount) {
+        log.debug("getProfileEntitlements called with executionId: {}", executionId);
         final ArmRpoExecutionDetailEntity executionDetail = armRpoService.getArmRpoExecutionDetailEntity(executionId);
         armRpoService.updateArmRpoStateAndStatus(executionDetail,
                                                  ArmRpoHelper.getProfileEntitlementsRpoState(),
@@ -236,6 +242,7 @@ public List<MasterIndexFieldByRecordClassSchema> getMasterIndexFieldByRecordClas
                                                                                             Integer executionId,
                                                                                             ArmRpoStateEntity rpoStateEntity,
                                                                                             UserAccountEntity userAccount) {
+        log.debug("getMasterIndexFieldByRecordClassSchema called with executionId: {}", executionId);
         var armRpoExecutionDetailEntity = armRpoService.getArmRpoExecutionDetailEntity(executionId);
         armRpoService.updateArmRpoStateAndStatus(armRpoExecutionDetailEntity, rpoStateEntity,
                                                  ArmRpoHelper.inProgressRpoStatus(), userAccount);
@@ -288,6 +295,7 @@ public List<MasterIndexFieldByRecordClassSchema> getMasterIndexFieldByRecordClas
     @Override
     public String addAsyncSearch(String bearerToken, Integer executionId, UserAccountEntity userAccount) {
 
+        log.debug("addAsyncSearch called with executionId: {}", executionId);
         final ArmRpoExecutionDetailEntity executionDetail = armRpoService.getArmRpoExecutionDetailEntity(executionId);
         armRpoService.updateArmRpoStateAndStatus(executionDetail,
                                                  ArmRpoHelper.addAsyncSearchRpoState(),
@@ -343,6 +351,7 @@ public String addAsyncSearch(String bearerToken, Integer executionId, UserAccoun
 
     @Override
     public void saveBackgroundSearch(String bearerToken, Integer executionId, String searchName, UserAccountEntity userAccount) {
+        log.debug("saveBackgroundSearch called with executionId: {}", executionId);
         var armRpoExecutionDetailEntity = armRpoService.getArmRpoExecutionDetailEntity(executionId);
         armRpoService.updateArmRpoStateAndStatus(armRpoExecutionDetailEntity, ArmRpoHelper.saveBackgroundSearchRpoState(),
                                                  ArmRpoHelper.inProgressRpoStatus(), userAccount);
@@ -394,11 +403,16 @@ public String getExtendedSearchesByMatter(String bearerToken, Integer executionI
             || isNull(extendedSearchesByMatterResponse.getSearches().getFirst())
             || isNull(extendedSearchesByMatterResponse.getSearches().getFirst().getSearch())
             || isNull(extendedSearchesByMatterResponse.getSearches().getFirst().getSearch().getTotalCount())
-            || StringUtils.isBlank(extendedSearchesByMatterResponse.getSearches().getFirst().getSearch().getName())) {
+            || StringUtils.isBlank(extendedSearchesByMatterResponse.getSearches().getFirst().getSearch().getName())
+            || isNull(extendedSearchesByMatterResponse.getSearches().getFirst().getSearch().getIsSaved())) {
             throw handleFailureAndCreateException(errorMessage.append("Search data is missing").toString(),
                                                   armRpoExecutionDetailEntity, userAccount);
         }
 
+        if (FALSE.equals(extendedSearchesByMatterResponse.getSearches().getFirst().getSearch().getIsSaved())) {
+            log.warn("The extendedSearchesByMatterResponse is not saved - with executionId: {}", executionId);
+            throw new ArmRpoGetExtendedSearchesByMatterIdException("The extendedSearchesByMatterResponse is not saved");
+        }
         armRpoExecutionDetailEntity.setSearchItemCount(extendedSearchesByMatterResponse.getSearches().getFirst().getSearch().getTotalCount());
         armRpoService.updateArmRpoStatus(armRpoExecutionDetailEntity, ArmRpoHelper.completedRpoStatus(), userAccount);
         return extendedSearchesByMatterResponse.getSearches().getFirst().getSearch().getName();

src/main/java/uk/gov/hmcts/darts/arm/service/TriggerArmRpoSearchService.java

@@ -1,5 +1,7 @@
 package uk.gov.hmcts.darts.arm.service;
 
+import java.time.Duration;
+
 public interface TriggerArmRpoSearchService {
-    void triggerArmRpoSearch();
+    void triggerArmRpoSearch(Duration threadSleepDuration);
 }

src/main/java/uk/gov/hmcts/darts/arm/service/impl/ArmRpoPollServiceImpl.java

@@ -5,6 +5,7 @@
 import org.apache.commons.collections.CollectionUtils;
 import org.springframework.stereotype.Service;
 import uk.gov.hmcts.darts.arm.config.ArmDataManagementConfiguration;
+import uk.gov.hmcts.darts.arm.exception.ArmRpoGetExtendedSearchesByMatterIdException;
 import uk.gov.hmcts.darts.arm.helper.ArmRpoHelper;
 import uk.gov.hmcts.darts.arm.model.rpo.MasterIndexFieldByRecordClassSchema;
 import uk.gov.hmcts.darts.arm.rpo.ArmRpoApi;
@@ -39,10 +40,9 @@ public class ArmRpoPollServiceImpl implements ArmRpoPollService {
     private final ArmDataManagementConfiguration armDataManagementConfiguration;
     private final LogApi logApi;
 
-    private List<File> tempProductionFiles = new ArrayList<>();
-
-    private List<Integer> allowableFailedStates = new ArrayList<>();
+    private List<File> tempProductionFiles;
 
+    private List<Integer> allowableFailedStates;
 
     @Override
     public void pollArmRpo(boolean isManualRun) {
@@ -69,6 +69,7 @@ public void pollArmRpo(boolean isManualRun) {
 
             // step to call ARM RPO API to get the extended searches by matter
             String productionName = armRpoApi.getExtendedSearchesByMatter(bearerToken, executionId, userAccount);
+
             // step to call ARM RPO API to get the master index field by record class schema
             List<MasterIndexFieldByRecordClassSchema> headerColumns = armRpoApi.getMasterIndexFieldByRecordClassSchema(
                 bearerToken, executionId,
@@ -95,10 +96,13 @@ public void pollArmRpo(boolean isManualRun) {
                 } else {
                     log.warn("No production export files found");
                 }
+                logApi.armRpoPollingSuccessful(executionId);
             } else {
-                log.warn("Create export of production files is still in progress");
+                log.warn("ARM RPO Polling is still in-progress as the createExportBasedOnSearchResultsTable is still not completed");
             }
-            logApi.armRpoPollingSuccessful(executionId);
+            log.info("Polling ARM RPO service completed");
+        } catch (ArmRpoGetExtendedSearchesByMatterIdException e) {
+            log.warn("ARM RPO Polling getExtendedSearchesByMatterId is still in-progress", e);
         } catch (Exception e) {
             log.error("Error while polling ARM RPO", e);
             logApi.armRpoPollingFailed(executionId);
@@ -161,9 +165,12 @@ private ArmRpoExecutionDetailEntity getArmRpoExecutionDetailEntity(boolean isMan
         if (isNull(armRpoExecutionDetailEntity)) {
             return null;
         }
+
         // If the previous state is saveBackgroundSearch and status is completed
+        // or the previous state is getExtendedSearchesByMatterId and status is in progress
         // or the previous state is createExportBasedOnSearchResultsTable and status is in progress, return the entity
         if (saveBackgroundSearchCompleted(armRpoExecutionDetailEntity)
+            || getExtendedSearchesByMatterIdInProgress(armRpoExecutionDetailEntity)
             || createExportBasedOnSearchResultsTableInProgress(armRpoExecutionDetailEntity)) {
             return armRpoExecutionDetailEntity;
         }
@@ -184,6 +191,12 @@ private boolean pollServiceNotInProgress(ArmRpoExecutionDetailEntity armRpoExecu
         return false;
     }
 
+    private boolean getExtendedSearchesByMatterIdInProgress(ArmRpoExecutionDetailEntity armRpoExecutionDetailEntity) {
+        return nonNull(armRpoExecutionDetailEntity.getArmRpoState())
+            && ArmRpoHelper.getExtendedSearchesByMatterRpoState().getId().equals(armRpoExecutionDetailEntity.getArmRpoState().getId())
+            && ArmRpoHelper.inProgressRpoStatus().getId().equals(armRpoExecutionDetailEntity.getArmRpoStatus().getId());
+    }
+
     private boolean createExportBasedOnSearchResultsTableInProgress(ArmRpoExecutionDetailEntity armRpoExecutionDetailEntity) {
         return nonNull(armRpoExecutionDetailEntity.getArmRpoState())
             && ArmRpoHelper.createExportBasedOnSearchResultsTableRpoState().getId().equals(armRpoExecutionDetailEntity.getArmRpoState().getId())

src/main/java/uk/gov/hmcts/darts/arm/service/impl/TriggerArmRpoSearchServiceImpl.java

@@ -11,6 +11,8 @@
 import uk.gov.hmcts.darts.authorisation.component.UserIdentity;
 import uk.gov.hmcts.darts.log.api.LogApi;
 
+import java.time.Duration;
+
 @Slf4j
 @Service
 @RequiredArgsConstructor
@@ -29,10 +31,12 @@ public class TriggerArmRpoSearchServiceImpl implements TriggerArmRpoSearchServic
      * If any of the underlying ArmRpoApi methods fail, ArmRpoApi will explicitly set the arm_rpo_execution_detail status to FAILED for that
      * particular stage. So if you're thinking about adding any transactionality at the level of TriggerArmRpoSearchServiceImpl please be careful to avoid
      * unwanted arm_rpo_execution_detail rollbacks.
+     *
+     * @param threadSleepDuration the duration to sleep the thread between API call
      */
     @Override
-    public void triggerArmRpoSearch() {
-        log.info("Triggering ARM RPO search flow...");
+    public void triggerArmRpoSearch(Duration threadSleepDuration) {
+        log.info("Triggering ARM RPO search flow with sleep duration {}", threadSleepDuration);
         Integer executionId = null;
         try {
             var userAccountEntity = userIdentity.getUserAccount();
@@ -70,6 +74,7 @@ public void triggerArmRpoSearch() {
             String searchName = armRpoApi.addAsyncSearch(armBearerToken,
                                                          executionId,
                                                          userAccountEntity);
+            sleep(threadSleepDuration);
 
             armRpoApi.saveBackgroundSearch(armBearerToken,
                                            executionId,
@@ -84,4 +89,14 @@ public void triggerArmRpoSearch() {
         }
     }
 
+    // Added method to fix sonar complaint
+    void sleep(Duration threadSleepDuration) {
+        try {
+            Thread.sleep(threadSleepDuration);
+        } catch (InterruptedException e) {
+            log.error("Trigger ARM RPO search thread sleep interrupted", e);
+            Thread.currentThread().interrupt();
+        }
+    }
+
 }

src/main/java/uk/gov/hmcts/darts/task/config/ProcessE2EArmRpoPendingAutomatedTaskConfig.java

@@ -5,11 +5,15 @@
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.context.annotation.Configuration;
 
+import java.time.Duration;
+
 @ConfigurationProperties("darts.automated.task.process-e2e-arm-rpo-pending")
 @Getter
 @Setter
 @Configuration
 public class ProcessE2EArmRpoPendingAutomatedTaskConfig extends AbstractAutomatedTaskConfig {
 
     private boolean processE2eArmRpo;
+
+    private Duration threadSleepDuration;
 }