DMP-4550: Performance Testing - Automated Tasks - UnstructuredToArmDataStore - 28K Batch Size - SELECT darts taking 12.5 minutes

src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java

@@ -412,47 +412,46 @@ List<ExternalObjectDirectoryEntity> findEodsForTransferExcludingMedia(ObjectReco
     default List<Integer> findEodsNotInOtherStorage(ObjectRecordStatusEntity status, ExternalLocationTypeEntity type,
                                                     ExternalLocationTypeEntity notExistsLocation, Integer limitRecords) {
         Set<Integer> results = new HashSet<>();//Ensures no duplicates
-        results.addAll(findEodsNotInOtherStorageOnlyMedia(status, type, notExistsLocation, limitRecords));
+        results.addAll(findEodsNotInOtherStorageOnlyMedia(status.getId(), type.getId(), notExistsLocation.getId(), limitRecords));
         if (results.size() < limitRecords) {
-            results.addAll(findEodsNotInOtherStorageExcludingMedia(status, type, notExistsLocation, limitRecords - results.size()));
+            results.addAll(findEodsNotInOtherStorageExcludingMedia(status.getId(), type.getId(), notExistsLocation.getId(), limitRecords - results.size()));
         }
         return new ArrayList<>(results);
     }
 
     @Query(
-        """
-            SELECT eod.id FROM ExternalObjectDirectoryEntity eod
-            WHERE eod.status = :status
-            AND eod.externalLocationType = :type
-            AND eod.media is not null          
-            AND NOT EXISTS (select 1 from ExternalObjectDirectoryEntity eod2
-            WHERE eod2.externalLocationType = :notExistsLocation
-            AND eod.media = eod2.media)
-            ORDER BY eod.id
-            LIMIT :limitRecords
-            """
+        value = """
+            select eode1_0.eod_id from darts.external_object_directory eode1_0
+            where eode1_0.ors_id=:status
+            and eode1_0.elt_id=:type
+            and eode1_0.med_id is not null 
+            and not exists(select 1 from darts.external_object_directory eode2_0 
+                where eode2_0.elt_id=:notExistsLocation 
+                and eode1_0.med_id=eode2_0.med_id)
+            fetch first :limitRecords rows only                 
+            """,
+        nativeQuery = true
     )
-    List<Integer> findEodsNotInOtherStorageOnlyMedia(ObjectRecordStatusEntity status, ExternalLocationTypeEntity type,
-                                                     ExternalLocationTypeEntity notExistsLocation, Integer limitRecords);
-
+    List<Integer> findEodsNotInOtherStorageOnlyMedia(Integer status, Integer type,
+                                                     Integer notExistsLocation, Integer limitRecords);
 
     @Query(
-        """
-            SELECT eod.id FROM ExternalObjectDirectoryEntity eod
-            WHERE eod.status = :status
-            AND eod.externalLocationType = :type
-            AND eod.media is null          
-            AND NOT EXISTS (select 1 from ExternalObjectDirectoryEntity eod2
-            where eod2.externalLocationType = :notExistsLocation
-            and ((eod.transcriptionDocumentEntity is not null and eod.transcriptionDocumentEntity = eod2.transcriptionDocumentEntity)
-              OR (eod.annotationDocumentEntity is not null and eod.annotationDocumentEntity = eod2.annotationDocumentEntity)
-              OR (eod.caseDocument is not null and eod.caseDocument = eod2.caseDocument )))
-            order by eod.id
-            LIMIT :limitRecords
-            """
+        value = """
+            select eode1_0.eod_id from darts.external_object_directory eode1_0
+            where eode1_0.ors_id=:status
+            and eode1_0.elt_id=:type
+            and eode1_0.med_id is null 
+            and not exists(select 1 from darts.external_object_directory eode2_0 
+                where eode2_0.elt_id=:notExistsLocation 
+                and ((eode1_0.trd_id is not null and eode1_0.trd_id = eode2_0.trd_id)
+                    or (eode1_0.ado_id is not null and eode1_0.ado_id = eode2_0.ado_id)
+                    or (eode1_0.cad_id is not null and eode1_0.cad_id = eode2_0.cad_id)))
+            fetch first :limitRecords rows only                 
+            """,
+        nativeQuery = true
     )
-    List<Integer> findEodsNotInOtherStorageExcludingMedia(ObjectRecordStatusEntity status, ExternalLocationTypeEntity type,
-                                                          ExternalLocationTypeEntity notExistsLocation, Integer limitRecords);
+    List<Integer> findEodsNotInOtherStorageExcludingMedia(Integer status, Integer type,
+                                                          Integer notExistsLocation, Integer limitRecords);
 
     @Query(
         """

src/test/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepositoryTest.java

@@ -15,6 +15,7 @@
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 class ExternalObjectDirectoryRepositoryTest {
 
@@ -81,55 +82,72 @@ void findEodsForTransfer_mediaItemsAreEqualToLimit_shouldReturnOnlyMedia() {
 
     @Test
     void findEodsNotInOtherStorage_mediaItemsArelessThanLimit_shouldReturnBothMediaAndNonMedia() {
+        final int mediaEod1Id = 1;
+        final int mediaEod2Id = 2;
+        final int nonMediaEod1Id = 3;
+        final int statusId = 4;
+        final int typeId = 5;
+        final int notExistsTypeId = 6;
+        final int limitRecords = 5;
+        final int numberOfMediaEods = 2;
+
         ExternalObjectDirectoryRepository externalObjectDirectoryRepository = spy(ExternalObjectDirectoryRepository.class);
-        int mediaEod1Id = 1;
-        int mediaEod2Id = 2;
-        int nonMediaEod1Id = 3;
         doReturn(List.of(mediaEod1Id, mediaEod2Id))
             .when(externalObjectDirectoryRepository)
             .findEodsNotInOtherStorageOnlyMedia(any(), any(), any(), any());
         doReturn(List.of(nonMediaEod1Id))
             .when(externalObjectDirectoryRepository)
             .findEodsNotInOtherStorageExcludingMedia(any(), any(), any(), any());
 
+
         ObjectRecordStatusEntity status = mock(ObjectRecordStatusEntity.class);
+        when(status.getId()).thenReturn(statusId);
         ExternalLocationTypeEntity type = mock(ExternalLocationTypeEntity.class);
+        when(type.getId()).thenReturn(typeId);
         ExternalLocationTypeEntity notExistsType = mock(ExternalLocationTypeEntity.class);
+        when(notExistsType.getId()).thenReturn(notExistsTypeId);
 
         List<Integer> eods = externalObjectDirectoryRepository.findEodsNotInOtherStorage(
-            status, type, notExistsType, 5);
+            status, type, notExistsType, limitRecords);
 
         assertThat(eods)
             .hasSize(3)
             .containsExactlyInAnyOrder(mediaEod1Id, mediaEod2Id, nonMediaEod1Id);
 
         verify(externalObjectDirectoryRepository)
-            .findEodsNotInOtherStorageOnlyMedia(status, type, notExistsType, 5);
+            .findEodsNotInOtherStorageOnlyMedia(statusId, typeId, notExistsTypeId, limitRecords);
         verify(externalObjectDirectoryRepository)
-            .findEodsNotInOtherStorageExcludingMedia(status, type, notExistsType, 3);
+            .findEodsNotInOtherStorageExcludingMedia(statusId, typeId, notExistsTypeId, limitRecords - numberOfMediaEods);
     }
 
     @Test
     void findEodsNotInOtherStorage_mediaItemsAreEqualToLimit_shouldReturnOnlyMedia() {
+        final int mediaEod1Id = 1;
+        final int mediaEod2Id = 2;
+        final int statusId = 4;
+        final int typeId = 5;
+        final int notExistsTypeId = 6;
+        final int limitRecords = 2;
         ExternalObjectDirectoryRepository externalObjectDirectoryRepository = spy(ExternalObjectDirectoryRepository.class);
-        int mediaEod1Id = 1;
-        int mediaEod2Id = 2;
         doReturn(List.of(mediaEod1Id, mediaEod2Id))
             .when(externalObjectDirectoryRepository)
             .findEodsNotInOtherStorageOnlyMedia(any(), any(), any(), any());
         ObjectRecordStatusEntity status = mock(ObjectRecordStatusEntity.class);
+        when(status.getId()).thenReturn(statusId);
         ExternalLocationTypeEntity type = mock(ExternalLocationTypeEntity.class);
+        when(type.getId()).thenReturn(typeId);
         ExternalLocationTypeEntity notExistsType = mock(ExternalLocationTypeEntity.class);
+        when(notExistsType.getId()).thenReturn(notExistsTypeId);
 
         List<Integer> eods = externalObjectDirectoryRepository.findEodsNotInOtherStorage(
-            status, type, notExistsType, 2);
+            status, type, notExistsType, limitRecords);
 
         assertThat(eods)
             .hasSize(2)
             .containsExactlyInAnyOrder(mediaEod1Id, mediaEod2Id);
 
         verify(externalObjectDirectoryRepository)
-            .findEodsNotInOtherStorageOnlyMedia(status, type, notExistsType, 2);
+            .findEodsNotInOtherStorageOnlyMedia(statusId, typeId, notExistsTypeId, limitRecords);
         verify(externalObjectDirectoryRepository, never())
             .findEodsNotInOtherStorageExcludingMedia(any(), any(), any(), any());
     }