From e08b8c47bd715920c863f4d8878f1d443ea2657f Mon Sep 17 00:00:00 2001 From: Ben Edwards <147524406+Ben-Edwards-cgi@users.noreply.github.com> Date: Fri, 3 Jan 2025 11:26:31 +0000 Subject: [PATCH] DMP-4467: Stage 1 - Performance Testing - Automated Tasks - UnstructuredToArmDataStore - Running a batch size of 50,000 you cannot see error until transaction completed or failed (#2371) --- .../helper/DataStoreToArmHelperIntTest.java | 27 +---- ...nstructuredToArmBatchProcessorIntTest.java | 108 +++++++----------- .../component/ArchiveRecordFileGenerator.java | 2 +- .../impl/ArchiveRecordFileGeneratorImpl.java | 49 +++----- .../arm/helper/DataStoreToArmHelper.java | 57 ++++++--- .../impl/DetsToArmBatchPushProcessorImpl.java | 66 ++++------- .../UnstructuredToArmBatchProcessorImpl.java | 42 ++----- .../impl/FileOperationServiceImpl.java | 1 - .../ArchiveRecordFileGeneratorImplTest.java | 39 ++----- .../arm/helper/DataStoreToArmHelperTest.java | 38 +++--- .../UnstructuredToArmBatchProcessorTest.java | 24 ---- .../DetsToArmBatchPushProcessorImplTest.java | 42 +------ 12 files changed, 165 insertions(+), 330 deletions(-) diff --git a/src/integrationTest/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelperIntTest.java b/src/integrationTest/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelperIntTest.java index a9927c5fd2..76c41d530c 100644 --- a/src/integrationTest/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelperIntTest.java +++ b/src/integrationTest/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelperIntTest.java @@ -24,13 +24,11 @@ import uk.gov.hmcts.darts.common.entity.ObjectStateRecordEntity; import uk.gov.hmcts.darts.common.entity.UserAccountEntity; import uk.gov.hmcts.darts.common.util.EodHelper; -import uk.gov.hmcts.darts.test.common.FileStore; import uk.gov.hmcts.darts.testutils.IntegrationBase; import uk.gov.hmcts.darts.testutils.stubs.ExternalObjectDirectoryStub; import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.time.ZoneOffset; @@ -39,6 +37,7 @@ import java.util.Optional; import java.util.UUID; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -248,25 +247,15 @@ void updateExternalObjectDirectoryFailedTransferAttempts() { assertEquals(2, externalObjectDirectory.getTransferAttempts()); } - @Test - void createEmptyArchiveRecordsFile() { - String manifestFilePrefix = "DETS"; - - File result = dataStoreToArmHelper.createEmptyArchiveRecordsFile(manifestFilePrefix); - - assertNotNull(result); - } - @Test void updateArmEodToArmIngestionStatus() { ArmBatchItem batchItem = new ArmBatchItem(); ArmBatchItems batchItems = new ArmBatchItems(); - File archiveRecordsFile = new File("testfile"); UserAccountEntity userAccount = dartsDatabase.getUserAccountStub().getIntegrationTestUserAccountEntity(); ExternalLocationTypeEntity eodSourceLocation = dartsDatabase.getExternalLocationTypeRepository().findById(DETS.getId()).orElseThrow(); dataStoreToArmHelper.updateArmEodToArmIngestionStatus(externalObjectDirectory, batchItem, batchItems, - archiveRecordsFile, userAccount, eodSourceLocation); + "testfile", userAccount, eodSourceLocation); assertEquals(ARM_INGESTION.getId(), externalObjectDirectory.getStatus().getId()); } @@ -275,12 +264,11 @@ void updateArmEodToArmIngestionStatus() { void createArmEodWithArmIngestionStatus() { ArmBatchItem batchItem = new ArmBatchItem(); ArmBatchItems batchItems = new ArmBatchItems(); - File archiveRecordsFile = new File("testfile"); UserAccountEntity userAccount = dartsDatabase.getUserAccountStub().getIntegrationTestUserAccountEntity(); ExternalObjectDirectoryEntity result = dataStoreToArmHelper.createArmEodWithArmIngestionStatus(externalObjectDirectory, batchItem, batchItems, - archiveRecordsFile, userAccount); + "testfile", userAccount); assertNotNull(result); } @@ -308,7 +296,7 @@ void shouldAddEntryToManifestFile() { } @Test - void writeManifestFile() throws IOException { + void generateManifestFileContents_typical() throws IOException { ArmBatchItems batchItems = new ArmBatchItems(); ArmBatchItem batchItem = new ArmBatchItem(); batchItem.setArmEod(externalObjectDirectory); @@ -316,12 +304,9 @@ void writeManifestFile() throws IOException { batchItem.setArchiveRecord(archiveRecord); batchItem.setRawFilePushSuccessful(true); batchItems.add(batchItem); - String fileLocation = tempDirectory.getAbsolutePath(); - File archiveFile = FileStore.getFileStore().create(Path.of(fileLocation), Path.of("archive-records.a360")); - - dataStoreToArmHelper.writeManifestFile(batchItems, archiveFile); + String result = dataStoreToArmHelper.generateManifestFileContents(batchItems, "archive-records.a360"); - assertTrue(archiveFile.exists()); + assertThat(result).isNotBlank(); } @Test diff --git a/src/integrationTest/java/uk/gov/hmcts/darts/task/service/UnstructuredToArmBatchProcessorIntTest.java b/src/integrationTest/java/uk/gov/hmcts/darts/task/service/UnstructuredToArmBatchProcessorIntTest.java index 82449046e5..6528e4cc44 100644 --- a/src/integrationTest/java/uk/gov/hmcts/darts/task/service/UnstructuredToArmBatchProcessorIntTest.java +++ b/src/integrationTest/java/uk/gov/hmcts/darts/task/service/UnstructuredToArmBatchProcessorIntTest.java @@ -1,10 +1,8 @@ package uk.gov.hmcts.darts.task.service; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import org.mockito.ArgumentCaptor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; @@ -13,6 +11,7 @@ import uk.gov.hmcts.darts.arm.component.ArchiveRecordFileGenerator; import uk.gov.hmcts.darts.arm.config.ArmDataManagementConfiguration; import uk.gov.hmcts.darts.arm.config.UnstructuredToArmProcessorConfiguration; +import uk.gov.hmcts.darts.arm.helper.DataStoreToArmHelper; import uk.gov.hmcts.darts.arm.mapper.MediaArchiveRecordMapper; import uk.gov.hmcts.darts.arm.service.ArchiveRecordService; import uk.gov.hmcts.darts.arm.service.ExternalObjectDirectoryService; @@ -32,24 +31,18 @@ import uk.gov.hmcts.darts.testutils.stubs.AuthorisationStub; import uk.gov.hmcts.darts.testutils.stubs.ExternalObjectDirectoryStub; -import java.io.BufferedReader; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; import static java.lang.String.format; -import static java.nio.file.Files.lines; -import static java.nio.file.Files.readString; import static java.time.temporal.ChronoUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.within; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.matches; import static org.mockito.Mockito.doReturn; @@ -74,7 +67,7 @@ @Slf4j class UnstructuredToArmBatchProcessorIntTest extends IntegrationBase { - ArgumentCaptor manifestFileNameCaptor = ArgumentCaptor.forClass(File.class); + ArgumentCaptor manifestFileNameCaptor = ArgumentCaptor.forClass(String.class); @SpyBean private ArmDataManagementApi armDataManagementApi; @@ -91,7 +84,7 @@ class UnstructuredToArmBatchProcessorIntTest extends IntegrationBase { private UserIdentity userIdentity; @Autowired private ArmDataManagementConfiguration armDataManagementConfiguration; - @Autowired + @SpyBean private FileOperationService fileOperationService; @SpyBean private ArchiveRecordService archiveRecordService; @@ -105,6 +98,8 @@ class UnstructuredToArmBatchProcessorIntTest extends IntegrationBase { private ExternalObjectDirectoryRepository eodRepository; @SpyBean private MediaArchiveRecordMapper mediaArchiveRecordMapper; + @SpyBean + private DataStoreToArmHelper dataStoreToArmHelper; @MockBean private UnstructuredToArmProcessorConfiguration unstructuredToArmProcessorConfiguration; @@ -117,9 +112,6 @@ class UnstructuredToArmBatchProcessorIntTest extends IntegrationBase { @Autowired private ExternalObjectDirectoryService eodService; - @TempDir - private File tempDirectory; - @Autowired private UnstructuredToArmBatchProcessor unstructuredToArmProcessor; @@ -286,9 +278,6 @@ void testBatchedQueryWhereSomeFailedToPush() throws IOException { String failedFilename3 = format("^%s.*", 9, eod3.getMedia().getId(), eod3.getTransferAttempts()); doThrow(dartsException).when(armDataManagementApi).copyBlobDataToArm(any(), matches(failedFilename3)); - String fileLocation = tempDirectory.getAbsolutePath(); - armDataManagementConfiguration.setTempBlobWorkspace(fileLocation); - //when unstructuredToArmProcessor.processUnstructuredToArm(5); @@ -321,19 +310,12 @@ void testBatchedQueryWhereSomeFailedToPush() throws IOException { ); assertThat(failedMediaList.size()).isEqualTo(2); - MediaEntity media = medias.stream() - .filter(mediaEntity -> mediaEntity.getId().equals(foundMediaList.get(0))).findFirst().orElseThrow(); - var successEod = eodRepository.findByMediaAndExternalLocationType(media, armLocation()).getFirst(); - + ArgumentCaptor manifestFileContentCaptor = ArgumentCaptor.forClass(String.class); + verify(dataStoreToArmHelper).convertStringToBinaryData(manifestFileContentCaptor.capture()); + String manifestFileContents = manifestFileContentCaptor.getValue(); - Files.walk(Path.of(fileLocation)).filter(Files::isRegularFile).collect(Collectors.toList()); - String[] types = {"a360"}; - File manifestFile = FileUtils.listFiles(new File(fileLocation), types, true).stream() - .filter(file -> file.getName().contains(successEod.getManifestFile())).findFirst().orElseThrow(); - - log.info("manifestFile {}", manifestFile.getAbsolutePath()); int expectedNumberOfRows = 6; - String manifestFileContents = verifyManifestFileContents(manifestFile, expectedNumberOfRows); + assertEquals(expectedNumberOfRows, manifestFileContents.lines().count()); log.info("actual response {}", manifestFileContents); List successfulMedias = medias.stream() @@ -385,20 +367,25 @@ void movePendingMediaDataFromUnstructuredToArmStorage() throws IOException { verify(armDataManagementApi, times(1)).copyBlobDataToArm(any(), eq(rawFile0Name)); verify(armDataManagementApi, times(1)).saveBlobDataToArm(matches("DARTS_.+\\.a360"), any()); - verify(archiveRecordFileGenerator).generateArchiveRecords(any(), manifestFileNameCaptor.capture()); - File manifestFile = manifestFileNameCaptor.getValue(); - Path manifestFilePath = manifestFile.toPath(); - assertThat(lines(manifestFilePath).count()).isEqualTo(4); - assertThat(readString(manifestFilePath)) + verify(archiveRecordFileGenerator).generateArchiveRecords(manifestFileNameCaptor.capture(), any()); + + + ArgumentCaptor manifestFileContentCaptor = ArgumentCaptor.forClass(String.class); + verify(dataStoreToArmHelper).convertStringToBinaryData(manifestFileContentCaptor.capture()); + String manifestFileContent = manifestFileContentCaptor.getValue(); + + assertThat(manifestFileContent.lines().count()).isEqualTo(4); + assertThat(manifestFileContent) .contains("\"operation\":\"create_record\"", "\"operation\":\"upload_new_file\"", "\"dz_file_name\":\"" + rawFile0Name, "\"dz_file_name\":\"" + rawFile1Name); - assertThat(armDropZoneEodsMedia0.get(0).getManifestFile()).isEqualTo(manifestFile.getName()); + String manifestFileName = manifestFileNameCaptor.getValue(); + assertThat(armDropZoneEodsMedia0.get(0).getManifestFile()).isEqualTo(manifestFileName); assertThat(armDropZoneEodsMedia0.get(0).getLastModifiedBy().getId()).isEqualTo(testUser.getId()); assertThat(armDropZoneEodsMedia0.get(0).getLastModifiedDateTime()).isCloseToUtcNow(within(1, SECONDS)); - assertThat(armDropZoneEodsMedia1.get(0).getManifestFile()).isEqualTo(manifestFile.getName()); + assertThat(armDropZoneEodsMedia1.get(0).getManifestFile()).isEqualTo(manifestFileName); } @Test @@ -432,20 +419,23 @@ void movePreviousArmFailedFromUnstructuredToArmStorage() throws IOException { var armDropzoneEodsMedia3 = eodRepository.findByMediaStatusAndType(medias.get(3), armDropZoneStatus(), armLocation()); assertThat(armDropzoneEodsMedia3).hasSize(0); - verify(archiveRecordFileGenerator).generateArchiveRecords(any(), manifestFileNameCaptor.capture()); - File manifestFile = manifestFileNameCaptor.getValue(); - assertThat(armDropzoneEodsMedia0.get(0).getManifestFile()).isEqualTo(manifestFile.getName()); + + verify(archiveRecordFileGenerator).generateArchiveRecords(manifestFileNameCaptor.capture(), any()); + String manifestFileName = manifestFileNameCaptor.getValue(); + assertThat(armDropzoneEodsMedia0.get(0).getManifestFile()).isEqualTo(manifestFileName); assertThat(armDropzoneEodsMedia0.get(0).getLastModifiedBy().getId()).isEqualTo(testUser.getId()); assertThat(armDropzoneEodsMedia0.get(0).getLastModifiedDateTime()).isCloseToUtcNow(within(1, SECONDS)); - assertThat(armDropzoneEodsMedia1.get(0).getManifestFile()).isEqualTo(manifestFile.getName()); + assertThat(armDropzoneEodsMedia1.get(0).getManifestFile()).isEqualTo(manifestFileName); - Path generatedManifestFilePath = manifestFile.toPath(); - assertThat(lines(generatedManifestFilePath).count()).isEqualTo(4); - assertThat(readString(generatedManifestFilePath)).contains( + ArgumentCaptor manifestFileContentCaptor = ArgumentCaptor.forClass(String.class); + verify(dataStoreToArmHelper).convertStringToBinaryData(manifestFileContentCaptor.capture()); + String manifestFileContent = manifestFileContentCaptor.getValue(); + assertThat(manifestFileContent.lines().count()).isEqualTo(4); + assertThat(manifestFileContent).contains( format("_%d_", medias.get(0).getId()), format("_%d_", medias.get(1).getId()) ); - assertThat(readString(generatedManifestFilePath)).doesNotContain(format("_%d_", medias.get(2).getId())); + assertThat(manifestFileContent).doesNotContain(format("_%d_", medias.get(2).getId())); } @Test @@ -468,11 +458,15 @@ void movePreviousArmFailedWithNoCorrespondingUnstructuredFailsAndProcessingConti assertThat(failedArmEods.get(0).getTransferAttempts()).isEqualTo(2); assertThat(eodRepository.findByMediaStatusAndType(medias.get(1), armDropZoneStatus(), armLocation())).hasSize(1); - verify(archiveRecordFileGenerator).generateArchiveRecords(any(), manifestFileNameCaptor.capture()); - Path generatedManifestFilePath = manifestFileNameCaptor.getValue().toPath(); - assertThat(lines(generatedManifestFilePath).count()).isEqualTo(2); - assertThat(readString(generatedManifestFilePath)).contains(format("_%d_", medias.get(1).getId())); - assertThat(readString(generatedManifestFilePath)).doesNotContain(format("_%d_", medias.get(0).getId())); + verify(archiveRecordFileGenerator).generateArchiveRecords(manifestFileNameCaptor.capture(), any()); + + ArgumentCaptor manifestFileContentCaptor = ArgumentCaptor.forClass(String.class); + verify(dataStoreToArmHelper).convertStringToBinaryData(manifestFileContentCaptor.capture()); + String manifestFileContent = manifestFileContentCaptor.getValue(); + + assertThat(manifestFileContent.lines().count()).isEqualTo(2); + assertThat(manifestFileContent).contains(format("_%d_", medias.get(1).getId())); + assertThat(manifestFileContent).doesNotContain(format("_%d_", medias.get(0).getId())); } @Test @@ -545,7 +539,7 @@ void writingManifestFileFails() { externalObjectDirectoryStub.createAndSaveEod(medias.get(1), STORED, UNSTRUCTURED); externalObjectDirectoryStub.createAndSaveEod(medias.get(1), ARM_MANIFEST_FAILED, ARM); - doThrow(RuntimeException.class).when(archiveRecordFileGenerator).generateArchiveRecords(any(), any()); + doThrow(RuntimeException.class).when(archiveRecordFileGenerator).generateArchiveRecords(anyString(), any()); //when unstructuredToArmProcessor.processUnstructuredToArm(5); @@ -590,20 +584,4 @@ void pushingManifestFileFails() { assertThat(failedEod.getTransferAttempts()).isEqualTo(2); assertThat(failedEod.getManifestFile()).isEqualTo("existingManifestFile"); } - - @SuppressWarnings("PMD.AssignmentInOperand") - private static String verifyManifestFileContents(File manifestFile, int expectedNumberOfRows) throws IOException { - StringBuilder fileContents = new StringBuilder(); - int rows = 0; - try (BufferedReader reader = Files.newBufferedReader(manifestFile.toPath())) { - String line; - - while ((line = reader.readLine()) != null) { - ++rows; - fileContents.append(line); - } - } - assertEquals(expectedNumberOfRows, rows); - return fileContents.toString(); - } } \ No newline at end of file diff --git a/src/main/java/uk/gov/hmcts/darts/arm/component/ArchiveRecordFileGenerator.java b/src/main/java/uk/gov/hmcts/darts/arm/component/ArchiveRecordFileGenerator.java index 82f02667a1..25cb10f98e 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/component/ArchiveRecordFileGenerator.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/component/ArchiveRecordFileGenerator.java @@ -9,5 +9,5 @@ public interface ArchiveRecordFileGenerator { boolean generateArchiveRecord(ArchiveRecord archiveRecord, File archiveRecordFile, ArchiveRecordType archiveRecordType); - void generateArchiveRecords(List archiveRecords, File archiveRecordsFile); + String generateArchiveRecords(String archiveFileName, List archiveRecords); } diff --git a/src/main/java/uk/gov/hmcts/darts/arm/component/impl/ArchiveRecordFileGeneratorImpl.java b/src/main/java/uk/gov/hmcts/darts/arm/component/impl/ArchiveRecordFileGeneratorImpl.java index 4b729bb242..dcb2db3387 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/component/impl/ArchiveRecordFileGeneratorImpl.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/component/impl/ArchiveRecordFileGeneratorImpl.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; import org.springframework.stereotype.Component; import uk.gov.hmcts.darts.arm.component.ArchiveRecordFileGenerator; import uk.gov.hmcts.darts.arm.enums.ArchiveRecordType; @@ -17,7 +16,6 @@ import java.nio.file.Files; import java.util.List; -import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.isNull; @Component @@ -50,41 +48,28 @@ public boolean generateArchiveRecord(ArchiveRecord archiveRecord, File archiveRe } @Override - public void generateArchiveRecords(List archiveRecords, File archiveRecordsFile) { + public String generateArchiveRecords(String archiveFileName, List archiveRecords) { + StringBuilder archiveRecordsStringBuilder = new StringBuilder(); + String archiveRecordsString = null; if (!archiveRecords.isEmpty()) { - try (BufferedWriter fileWriter = Files.newBufferedWriter(archiveRecordsFile.toPath()); PrintWriter printWriter = new PrintWriter(fileWriter)) { - for (var archiveRecord : archiveRecords) { - try { - String archiveRecordOperation = objectMapper.writeValueAsString(archiveRecord.getArchiveRecordOperation()); - String uploadNewFileRecord = objectMapper.writeValueAsString(archiveRecord.getUploadNewFileRecord()); - log.debug("About to write {}{} to file {} for EOD {}", archiveRecordOperation, uploadNewFileRecord, - archiveRecordsFile.getAbsolutePath(), archiveRecord.getArchiveRecordOperation().getRelationId()); - printWriter.println(archiveRecordOperation); - printWriter.println(uploadNewFileRecord); - } catch (Exception e) { - log.error("Unable to write archive record for EOD {} to ARM file {}", - archiveRecord.getArchiveRecordOperation().getRelationId(), - archiveRecordsFile.getAbsoluteFile()); - throw new DartsException(e); - } + for (var archiveRecord : archiveRecords) { + try { + String archiveRecordOperation = objectMapper.writeValueAsString(archiveRecord.getArchiveRecordOperation()); + String uploadNewFileRecord = objectMapper.writeValueAsString(archiveRecord.getUploadNewFileRecord()); + archiveRecordsStringBuilder.append(archiveRecordOperation).append(System.lineSeparator()); + archiveRecordsStringBuilder.append(uploadNewFileRecord).append(System.lineSeparator()); + } catch (Exception e) { + log.error("Unable to write archive record for EOD {}", + archiveRecord.getArchiveRecordOperation().getRelationId()); + throw new DartsException(e); } - } catch (IOException e) { - log.error("Unable to write ARM manifest file {}", archiveRecordsFile.getAbsoluteFile(), e); - throw new DartsException(e); } - logManifestFile(archiveRecords, archiveRecordsFile); - } - } - - private void logManifestFile(List archiveRecords, File archiveRecordsFile) { - try { - String contents = FileUtils.readFileToString(archiveRecordsFile.getAbsoluteFile(), UTF_8); + archiveRecordsString = archiveRecordsStringBuilder.toString(); log.info("Contents of manifest file {} for EOD {}\n{}", - archiveRecordsFile.getAbsoluteFile(), + archiveFileName, archiveRecords.get(0).getArchiveRecordOperation().getRelationId(), - contents); - } catch (Exception e) { - log.error("Unable to read ARM manifest file {}", archiveRecordsFile.getAbsoluteFile(), e); + archiveRecordsString); } + return archiveRecordsString; } } diff --git a/src/main/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelper.java b/src/main/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelper.java index 89e4022c6a..eb68d82c32 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelper.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelper.java @@ -1,7 +1,8 @@ package uk.gov.hmcts.darts.arm.helper; +import com.azure.core.util.BinaryData; +import com.azure.storage.blob.models.BlobStorageException; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ObjectUtils; import org.springframework.data.domain.Pageable; @@ -23,8 +24,6 @@ import uk.gov.hmcts.darts.common.util.EodHelper; import uk.gov.hmcts.darts.log.api.LogApi; -import java.io.File; -import java.nio.file.Path; import java.text.MessageFormat; import java.time.Duration; import java.time.Instant; @@ -45,6 +44,8 @@ @Slf4j @RequiredArgsConstructor public class DataStoreToArmHelper { + + private static final int BLOB_ALREADY_EXISTS_STATUS_CODE = 409; private final ObjectRecordStatusRepository objectRecordStatusRepository; private final ExternalObjectDirectoryRepository externalObjectDirectoryRepository; private final ArmDataManagementConfiguration armDataManagementConfiguration; @@ -56,7 +57,7 @@ public class DataStoreToArmHelper { public List getEodEntitiesToSendToArm(ExternalLocationTypeEntity sourceLocation, - ExternalLocationTypeEntity armLocation, int maxResultSize) { + ExternalLocationTypeEntity armLocation, int maxResultSize) { ObjectRecordStatusEntity armRawStatusFailed = objectRecordStatusRepository.getReferenceById(ARM_RAW_DATA_FAILED.getId()); ObjectRecordStatusEntity armManifestFailed = objectRecordStatusRepository.getReferenceById(ARM_MANIFEST_FAILED.getId()); @@ -218,28 +219,24 @@ public void updateExternalObjectDirectoryFailedTransferAttempts(ExternalObjectDi externalObjectDirectoryRepository.saveAndFlush(externalObjectDirectoryEntity); } - @SneakyThrows - public File createEmptyArchiveRecordsFile(String manifestFilePrefix) { + public String getArchiveRecordsFileName(String manifestFilePrefix) { String fileNameFormat = "%s_%s.%s"; - String fileName = String.format(fileNameFormat, - manifestFilePrefix, - UUID.randomUUID(), - armDataManagementConfiguration.getFileExtension() + return String.format(fileNameFormat, + manifestFilePrefix, + UUID.randomUUID(), + armDataManagementConfiguration.getFileExtension() ); - Path filePath = fileOperationService.createFile(fileName, armDataManagementConfiguration.getTempBlobWorkspace(), true); - log.info("Created empty archive records file {}", filePath.getFileName()); - return filePath.toFile(); } public void updateArmEodToArmIngestionStatus(ExternalObjectDirectoryEntity armEod, ArmBatchItem batchItem, ArmBatchItems batchItems, - File archiveRecordsFile, UserAccountEntity userAccount, + String archiveRecordsFileName, UserAccountEntity userAccount, ExternalLocationTypeEntity eodSourceLocation) { var matchingEntity = getExternalObjectDirectoryEntity(armEod, eodSourceLocation, EodHelper.storedStatus()); if (matchingEntity.isPresent()) { batchItem.setSourceEod(matchingEntity.get()); batchItems.add(batchItem); - armEod.setManifestFile(archiveRecordsFile.getName()); + armEod.setManifestFile(archiveRecordsFileName); incrementTransferAttempts(armEod); updateExternalObjectDirectoryStatus(armEod, EodHelper.armIngestionStatus(), userAccount); } else { @@ -252,10 +249,10 @@ public void updateArmEodToArmIngestionStatus(ExternalObjectDirectoryEntity armEo public ExternalObjectDirectoryEntity createArmEodWithArmIngestionStatus(ExternalObjectDirectoryEntity currentEod, ArmBatchItem batchItem, ArmBatchItems batchItems, - File archiveRecordsFile, + String archiveRecordsFileName, UserAccountEntity userAccount) { ExternalObjectDirectoryEntity armEod = createArmExternalObjectDirectoryEntity(currentEod, EodHelper.armIngestionStatus(), userAccount); - armEod.setManifestFile(archiveRecordsFile.getName()); + armEod.setManifestFile(archiveRecordsFileName); var savedArmEod = externalObjectDirectoryRepository.saveAndFlush(armEod); batchItem.setArmEod(savedArmEod); @@ -274,8 +271,8 @@ public boolean shouldAddEntryToManifestFile(ArmBatchItem batchItem) { return equalsAnyStatus(batchItem.getPreviousStatus(), EodHelper.failedArmManifestFileStatus(), EodHelper.armResponseManifestFailedStatus()); } - public void writeManifestFile(ArmBatchItems batchItems, File archiveRecordsFile) { - archiveRecordFileGenerator.generateArchiveRecords(batchItems.getArchiveRecords(), archiveRecordsFile); + public String generateManifestFileContents(ArmBatchItems batchItems, String archiveRecordsFileName) { + return archiveRecordFileGenerator.generateArchiveRecords(archiveRecordsFileName, batchItems.getArchiveRecords()); } public void recoverByUpdatingEodToFailedArmStatus(ArmBatchItem batchItem, UserAccountEntity userAccount) { @@ -293,4 +290,26 @@ public void recoverByUpdatingEodToFailedArmStatus(ArmBatchItem batchItem, UserAc public Long getFileSize(int externalObjectDirectoryId) { return externalObjectDirectoryRepository.findFileSize(externalObjectDirectoryId); } + + + public void copyMetadataToArm(String manifestFileContents, String archiveRecordsFileName) { + try { + BinaryData metadataFileBinary = convertStringToBinaryData(Optional.ofNullable(manifestFileContents).orElse("")); + armDataManagementApi.saveBlobDataToArm(archiveRecordsFileName, metadataFileBinary); + } catch (BlobStorageException e) { + if (e.getStatusCode() == BLOB_ALREADY_EXISTS_STATUS_CODE) { + log.info("Metadata BLOB already exists", e); + } else { + log.error("Failed to move BLOB metadata for file {}", archiveRecordsFileName, e); + throw e; + } + } catch (Exception e) { + log.error("Unable to move BLOB metadata for file {}", archiveRecordsFileName, e); + throw e; + } + } + + public BinaryData convertStringToBinaryData(String manifestFileContents) { + return BinaryData.fromString(manifestFileContents); + } } diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/DetsToArmBatchPushProcessorImpl.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/DetsToArmBatchPushProcessorImpl.java index 972b96ac8a..76ef569167 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/DetsToArmBatchPushProcessorImpl.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/DetsToArmBatchPushProcessorImpl.java @@ -1,7 +1,5 @@ package uk.gov.hmcts.darts.arm.service.impl; -import com.azure.core.util.BinaryData; -import com.azure.storage.blob.models.BlobStorageException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.ListUtils; @@ -31,7 +29,6 @@ import uk.gov.hmcts.darts.common.util.EodHelper; import uk.gov.hmcts.darts.log.api.LogApi; -import java.io.File; import java.time.Duration; import java.time.Instant; import java.time.OffsetDateTime; @@ -61,7 +58,6 @@ public class DetsToArmBatchPushProcessorImpl implements DetsToArmBatchPushProces private final CurrentTimeHelper currentTimeHelper; private final ExternalObjectDirectoryService externalObjectDirectoryService; - private static final int BLOB_ALREADY_EXISTS_STATUS_CODE = 409; public void processDetsToArm(int taskBatchSize) { log.info("Started running DETS ARM Batch Push processing at: {}", OffsetDateTime.now()); @@ -69,14 +65,14 @@ public void processDetsToArm(int taskBatchSize) { // Because the query is long-running, get all the EODs that need to be processed in one go List eodsForTransfer = getDetsEodEntitiesToSendToArm(eodSourceLocation, - EodHelper.armLocation(), - taskBatchSize); + EodHelper.armLocation(), + taskBatchSize); log.info("Found {} DETS pending entities to process from source '{}'", eodsForTransfer.size(), eodSourceLocation.getDescription()); if (!eodsForTransfer.isEmpty()) { //ARM has a max batch size for manifest items, so lets loop through the big list creating lots of individual batches for ARM to process separately List> batchesForArm = ListUtils.partition(eodsForTransfer, - detsToArmProcessorConfiguration.getMaxArmManifestItems()); + detsToArmProcessorConfiguration.getMaxArmManifestItems()); int batchCounter = 1; UserAccountEntity userAccount = userIdentity.getUserAccount(); for (List eodsForBatch : batchesForArm) { @@ -89,7 +85,7 @@ public void processDetsToArm(int taskBatchSize) { } private List getDetsEodEntitiesToSendToArm(ExternalLocationTypeEntity sourceLocation, - ExternalLocationTypeEntity armLocation, int maxResultSize) { + ExternalLocationTypeEntity armLocation, int maxResultSize) { ObjectRecordStatusEntity armRawStatusFailed = EodHelper.failedArmRawDataStatus(); ObjectRecordStatusEntity armManifestFailed = EodHelper.failedArmManifestFileStatus(); @@ -117,7 +113,8 @@ private List getDetsEodEntitiesToSendToArm(ExternalLocationTypeEntity s } private void createAndSendBatchFile(List eodsForBatch, UserAccountEntity userAccount) { - File archiveRecordsFile = dataStoreToArmHelper.createEmptyArchiveRecordsFile(getManifestFilePrefix()); + String archiveRecordsFileName = dataStoreToArmHelper.getArchiveRecordsFileName(detsToArmProcessorConfiguration.getManifestFilePrefix()); + var batchItems = new ArmBatchItems(); for (var currentEod : eodsForBatch) { @@ -130,7 +127,7 @@ private void createAndSendBatchFile(List eodsForB armEod = currentEod; batchItem.setArmEod(armEod); dataStoreToArmHelper.updateArmEodToArmIngestionStatus( - currentEod, batchItem, batchItems, archiveRecordsFile, userAccount, EodHelper.detsLocation()); + currentEod, batchItem, batchItems, archiveRecordsFileName, userAccount, EodHelper.detsLocation()); objectStateRecord = getObjectStateRecordEntity(armEod); // Reset the failed status from previous attempt objectStateRecord.setObjectStatus(null); @@ -138,7 +135,7 @@ private void createAndSendBatchFile(List eodsForB } else { armEod = dataStoreToArmHelper.createArmEodWithArmIngestionStatus( - currentEod, batchItem, batchItems, archiveRecordsFile, userAccount); + currentEod, batchItem, batchItems, archiveRecordsFileName, userAccount); objectStateRecord = getObjectStateRecordEntity(currentEod); // Save the new ARM EOD to the object state record objectStateRecord.setArmEodId(String.valueOf(armEod.getId())); @@ -161,7 +158,7 @@ private void createAndSendBatchFile(List eodsForB } } - if (!writeManifestAndCopyToArm(userAccount, batchItems, archiveRecordsFile)) { + if (!writeManifestAndCopyToArm(userAccount, batchItems, archiveRecordsFileName)) { return; } @@ -172,34 +169,35 @@ private void createAndSendBatchFile(List eodsForB } - private boolean writeManifestAndCopyToArm(UserAccountEntity userAccount, ArmBatchItems batchItems, File archiveRecordsFile) { + private boolean writeManifestAndCopyToArm(UserAccountEntity userAccount, ArmBatchItems batchItems, String archiveRecordsFileName) { try { if (!batchItems.getSuccessful().isEmpty()) { - dataStoreToArmHelper.writeManifestFile(batchItems, archiveRecordsFile); - updateObjectStateRecordManifestCreated(batchItems, archiveRecordsFile); - copyMetadataToArm(archiveRecordsFile); - updateObjectStateRecordManifestSuccessOrFailure(batchItems, archiveRecordsFile, true); + String archiveRecordsContents = dataStoreToArmHelper.generateManifestFileContents(batchItems, archiveRecordsFileName); + + updateObjectStateRecordManifestCreated(batchItems, archiveRecordsFileName); + dataStoreToArmHelper.copyMetadataToArm(archiveRecordsContents, archiveRecordsFileName); + updateObjectStateRecordManifestSuccessOrFailure(batchItems, true); } else { log.warn("No EODs were able to be processed, skipping manifest file creation"); } } catch (Exception e) { - String errorMessage = String.format("Error during generation of DETS batch manifest file %s", archiveRecordsFile.getName()); + String errorMessage = String.format("Error during generation of DETS batch manifest file %s", archiveRecordsFileName); log.error(errorMessage, e); batchItems.getSuccessful().forEach(batchItem -> dataStoreToArmHelper.recoverByUpdatingEodToFailedArmStatus(batchItem, userAccount)); final String errorMessageWithStackTrace = errorMessage + " - " + ExceptionUtils.getStackTrace(e); batchItems.getFailed().forEach(batchItem -> updateObjectStateRecordStatus(batchItem.getArmEod(), errorMessageWithStackTrace)); - updateObjectStateRecordManifestSuccessOrFailure(batchItems, archiveRecordsFile, false); + updateObjectStateRecordManifestSuccessOrFailure(batchItems, false); return false; } return true; } - private void updateObjectStateRecordManifestCreated(ArmBatchItems batchItems, File archiveRecordsFile) { + private void updateObjectStateRecordManifestCreated(ArmBatchItems batchItems, String archiveRecordsFileName) { for (var batchItem : batchItems.getSuccessful()) { ObjectStateRecordEntity objectStateRecord = getObjectStateRecordEntity(batchItem.getArmEod()); objectStateRecord.setFlagFileMfstCreated(true); objectStateRecord.setDateFileMfstCreated(currentTimeHelper.currentOffsetDateTime()); - objectStateRecord.setIdManifestFile(archiveRecordsFile.getName()); + objectStateRecord.setIdManifestFile(archiveRecordsFileName); objectStateRecordRepository.save(objectStateRecord); } @@ -209,11 +207,7 @@ private void updateObjectStateRecordManifestCreated(ArmBatchItems batchItems, Fi } } - private String getManifestFilePrefix() { - return detsToArmProcessorConfiguration.getManifestFilePrefix(); - } - - private void updateObjectStateRecordManifestSuccessOrFailure(ArmBatchItems batchItems, File archiveRecordsFile, boolean isSuccessfulTransfer) { + private void updateObjectStateRecordManifestSuccessOrFailure(ArmBatchItems batchItems, boolean isSuccessfulTransfer) { for (var batchItem : batchItems.getSuccessful()) { ObjectStateRecordEntity objectStateRecord = getObjectStateRecordEntity(batchItem.getArmEod()); objectStateRecord.setFlagMfstTransfToArml(isSuccessfulTransfer); @@ -322,24 +316,6 @@ private void setRawDataPushed(ExternalObjectDirectoryEntity detsExternalObjectDi objectStateRecord.setMd5FileTransfArml(detsExternalObjectDirectory.getChecksum()); setFileSize(detsExternalObjectDirectory, objectStateRecord); objectStateRecordRepository.save(objectStateRecord); - - } - - private void copyMetadataToArm(File manifestFile) { - try { - BinaryData metadataFileBinary = fileOperationService.convertFileToBinaryData(manifestFile.getAbsolutePath()); - armDataManagementApi.saveBlobDataToArm(manifestFile.getName(), metadataFileBinary); - } catch (BlobStorageException e) { - if (e.getStatusCode() == BLOB_ALREADY_EXISTS_STATUS_CODE) { - log.info("Metadata BLOB already exists", e); - } else { - log.error("Failed to move BLOB metadata for file {}", manifestFile.getAbsolutePath(), e); - throw e; - } - } catch (Exception e) { - log.error("Unable to move BLOB metadata for file {}", manifestFile.getAbsolutePath(), e); - throw e; - } } private void setFileSize(ExternalObjectDirectoryEntity detsExternalObjectDirectory, ObjectStateRecordEntity objectStateRecord) { @@ -348,6 +324,4 @@ private void setFileSize(ExternalObjectDirectoryEntity detsExternalObjectDirecto objectStateRecord.setFileSizeBytesArml(fileSize); } } - - } \ No newline at end of file diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/UnstructuredToArmBatchProcessorImpl.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/UnstructuredToArmBatchProcessorImpl.java index 6a3efcf51e..549b13dcc1 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/UnstructuredToArmBatchProcessorImpl.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/UnstructuredToArmBatchProcessorImpl.java @@ -1,7 +1,5 @@ package uk.gov.hmcts.darts.arm.service.impl; -import com.azure.core.util.BinaryData; -import com.azure.storage.blob.models.BlobStorageException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.ListUtils; @@ -25,7 +23,6 @@ import uk.gov.hmcts.darts.log.api.LogApi; import uk.gov.hmcts.darts.util.AsyncUtil; -import java.io.File; import java.text.MessageFormat; import java.time.OffsetDateTime; import java.util.List; @@ -105,11 +102,9 @@ public void processUnstructuredToArm(int taskBatchSize) { } private void createAndSendBatchFile(List eodsForBatch, UserAccountEntity userAccount) { - - File archiveRecordsFile = unstructuredToArmHelper.createEmptyArchiveRecordsFile(armDataManagementConfiguration.getManifestFilePrefix()); + String archiveRecordsFileName = unstructuredToArmHelper.getArchiveRecordsFileName(armDataManagementConfiguration.getManifestFilePrefix()); var batchItems = new ArmBatchItems(); - for (var currentEod : eodsForBatch) { var batchItem = new ArmBatchItem(); try { @@ -118,9 +113,9 @@ private void createAndSendBatchFile(List eodsForB //retry existing attempt that has previously failed. armEod = currentEod; batchItem.setArmEod(armEod); - updateArmEodToArmIngestionStatus(currentEod, batchItem, batchItems, archiveRecordsFile, userAccount); + updateArmEodToArmIngestionStatus(currentEod, batchItem, batchItems, archiveRecordsFileName, userAccount); } else { - armEod = unstructuredToArmHelper.createArmEodWithArmIngestionStatus(currentEod, batchItem, batchItems, archiveRecordsFile, userAccount); + armEod = unstructuredToArmHelper.createArmEodWithArmIngestionStatus(currentEod, batchItem, batchItems, archiveRecordsFileName, userAccount); } String rawFilename = unstructuredToArmHelper.generateRawFilename(armEod); @@ -138,11 +133,11 @@ private void createAndSendBatchFile(List eodsForB try { if (!batchItems.getSuccessful().isEmpty()) { - writeManifestFile(batchItems, archiveRecordsFile); - copyMetadataToArm(archiveRecordsFile); + String manifestFileContents = unstructuredToArmHelper.generateManifestFileContents(batchItems, archiveRecordsFileName); + unstructuredToArmHelper.copyMetadataToArm(manifestFileContents, archiveRecordsFileName); } } catch (Exception e) { - log.error("Error during generation of batch manifest file {}", archiveRecordsFile.getName(), e); + log.error("Error during generation of batch manifest file {}", archiveRecordsFileName, e); batchItems.getSuccessful().forEach(batchItem -> recoverByUpdatingEodToFailedArmStatus(batchItem, userAccount)); return; } @@ -154,12 +149,12 @@ private void createAndSendBatchFile(List eodsForB } private void updateArmEodToArmIngestionStatus(ExternalObjectDirectoryEntity armEod, ArmBatchItem batchItem, ArmBatchItems batchItems, - File archiveRecordsFile, UserAccountEntity userAccount) { + String archiveRecordsFileName, UserAccountEntity userAccount) { var matchingEntity = unstructuredToArmHelper.getExternalObjectDirectoryEntity(armEod, EodHelper.unstructuredLocation(), EodHelper.storedStatus()); if (matchingEntity.isPresent()) { batchItem.setSourceEod(matchingEntity.get()); batchItems.add(batchItem); - armEod.setManifestFile(archiveRecordsFile.getName()); + armEod.setManifestFile(archiveRecordsFileName); unstructuredToArmHelper.incrementTransferAttempts(armEod); unstructuredToArmHelper.updateExternalObjectDirectoryStatus(armEod, EodHelper.armIngestionStatus(), userAccount); } else { @@ -198,27 +193,6 @@ private boolean shouldAddEntryToManifestFile(ArmBatchItem batchItem) { return equalsAnyStatus(batchItem.getPreviousStatus(), EodHelper.failedArmManifestFileStatus(), EodHelper.armResponseManifestFailedStatus()); } - private void writeManifestFile(ArmBatchItems batchItems, File archiveRecordsFile) { - archiveRecordFileGenerator.generateArchiveRecords(batchItems.getArchiveRecords(), archiveRecordsFile); - } - - protected void copyMetadataToArm(File manifestFile) { - try { - BinaryData metadataFileBinary = fileOperationService.convertFileToBinaryData(manifestFile.getAbsolutePath()); - armDataManagementApi.saveBlobDataToArm(manifestFile.getName(), metadataFileBinary); - } catch (BlobStorageException e) { - if (e.getStatusCode() == BLOB_ALREADY_EXISTS_STATUS_CODE) { - log.info("Metadata BLOB already exists", e); - } else { - log.error("Failed to move BLOB metadata for file {}", manifestFile.getAbsolutePath(), e); - throw e; - } - } catch (Exception e) { - log.error("Unable to move BLOB metadata for file {}", manifestFile.getAbsolutePath(), e); - throw e; - } - } - @SuppressWarnings({"PMD.ConfusingTernary"}) private void recoverByUpdatingEodToFailedArmStatus(ArmBatchItem batchItem, UserAccountEntity userAccount) { if (batchItem.getArmEod() != null) { diff --git a/src/main/java/uk/gov/hmcts/darts/common/service/impl/FileOperationServiceImpl.java b/src/main/java/uk/gov/hmcts/darts/common/service/impl/FileOperationServiceImpl.java index de5e021c13..22a72129a5 100644 --- a/src/main/java/uk/gov/hmcts/darts/common/service/impl/FileOperationServiceImpl.java +++ b/src/main/java/uk/gov/hmcts/darts/common/service/impl/FileOperationServiceImpl.java @@ -87,5 +87,4 @@ public Path saveBinaryDataToSpecifiedWorkspace(BinaryData binaryData, String fil public BinaryData convertFileToBinaryData(String fileName) { return BinaryData.fromFile(Path.of(fileName)); } - } diff --git a/src/test/java/uk/gov/hmcts/darts/arm/component/impl/ArchiveRecordFileGeneratorImplTest.java b/src/test/java/uk/gov/hmcts/darts/arm/component/impl/ArchiveRecordFileGeneratorImplTest.java index 6e7350bad3..25dc551664 100644 --- a/src/test/java/uk/gov/hmcts/darts/arm/component/impl/ArchiveRecordFileGeneratorImplTest.java +++ b/src/test/java/uk/gov/hmcts/darts/arm/component/impl/ArchiveRecordFileGeneratorImplTest.java @@ -47,7 +47,6 @@ import static org.skyscreamer.jsonassert.JSONAssert.assertEquals; import static uk.gov.hmcts.darts.arm.util.ArchiveConstants.ArchiveRecordOperationValues.UPLOAD_NEW_FILE; import static uk.gov.hmcts.darts.test.common.TestUtils.getContentsFromFile; -import static uk.gov.hmcts.darts.test.common.TestUtils.getFile; @SuppressWarnings("PMD.AssignmentInOperand") @ExtendWith(MockitoExtension.class) @@ -155,30 +154,24 @@ void generateArchiveRecordWithNullArchiveRecord() throws Exception { } @Test - void generateArchiveRecords() throws Exception { - String fileLocation = tempDirectory.getAbsolutePath(); + void generateArchiveRecords_typical() throws Exception { String relationId = "1234"; - File archiveFile = FileStore.getFileStore().create(Path.of(fileLocation), Path.of("archive-records.a360")); var archiveRecord = createMediaArchiveRecord(relationId); List archiveRecords = List.of(archiveRecord, archiveRecord); - archiveRecordFileGenerator.generateArchiveRecords(archiveRecords, archiveFile); + String result = archiveRecordFileGenerator.generateArchiveRecords("archive-records.a360", archiveRecords); - File expectedResponse = getFile("Tests/arm/component/expectedResponseMultipleArchiveRecords.a360"); - assertThat(archiveFile).hasSameTextualContentAs(expectedResponse); + String expectedResponse = getContentsFromFile("Tests/arm/component/expectedResponseMultipleArchiveRecords.a360"); + assertThat(result).isEqualTo(expectedResponse); } @Test - void generateArchiveRecordsEmptyArchiveRecords() { - String fileLocation = tempDirectory.getAbsolutePath(); - File archiveFile = new File(fileLocation, "archive-records.a360"); - + void generateArchiveRecords_emptyArchiveRecords() { List archiveRecords = Collections.emptyList(); - archiveRecordFileGenerator.generateArchiveRecords(archiveRecords, archiveFile); - - assertThat(Files.notExists(archiveFile.toPath())).isTrue(); + String result = archiveRecordFileGenerator.generateArchiveRecords("archive-records.a360", archiveRecords); + assertThat(result).isBlank(); } @Test @@ -187,27 +180,11 @@ void generateArchiveRecordsArchiveRecordError() throws Exception { archiveRecordFileGenerator = new ArchiveRecordFileGeneratorImpl(objectMapper); when(objectMapper.writeValueAsString(any())).thenThrow(RuntimeException.class); - String fileLocation = tempDirectory.getAbsolutePath(); - File archiveFile = FileStore.getFileStore().create(Path.of(fileLocation), Path.of("archive-records.a360")); - - String relationId = "1234"; - var archiveRecord = createMediaArchiveRecord(relationId); - List archiveRecords = List.of(archiveRecord); - - assertThrows(DartsException.class, () -> archiveRecordFileGenerator.generateArchiveRecords(archiveRecords, archiveFile)); - } - - @Test - void generateArchiveRecordsFileError() { - - String fileLocation = tempDirectory.getAbsolutePath(); - File invalidFile = new File(fileLocation, "archive-recor/ds.a360"); - String relationId = "1234"; var archiveRecord = createMediaArchiveRecord(relationId); List archiveRecords = List.of(archiveRecord); - assertThrows(DartsException.class, () -> archiveRecordFileGenerator.generateArchiveRecords(archiveRecords, invalidFile)); + assertThrows(DartsException.class, () -> archiveRecordFileGenerator.generateArchiveRecords("archive-records.a360", archiveRecords)); } private static String getFileContents(File archiveFile) throws IOException { diff --git a/src/test/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelperTest.java b/src/test/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelperTest.java index affc183e97..7b692cf47c 100644 --- a/src/test/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelperTest.java +++ b/src/test/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelperTest.java @@ -12,6 +12,7 @@ import uk.gov.hmcts.darts.arm.api.ArmDataManagementApi; import uk.gov.hmcts.darts.arm.component.ArchiveRecordFileGenerator; import uk.gov.hmcts.darts.arm.config.ArmDataManagementConfiguration; +import uk.gov.hmcts.darts.arm.model.ArchiveRecord; import uk.gov.hmcts.darts.arm.model.batch.ArmBatchItems; import uk.gov.hmcts.darts.common.entity.ExternalLocationTypeEntity; import uk.gov.hmcts.darts.common.entity.ExternalObjectDirectoryEntity; @@ -33,14 +34,14 @@ import java.util.Optional; import java.util.UUID; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -199,32 +200,19 @@ void updateExternalObjectDirectoryFailedTransferAttempts() { verify(externalObjectDirectoryRepository).saveAndFlush(externalObjectDirectoryEntity); } - @Test - void createEmptyArchiveRecordsFile() throws IOException { - // given - String manifestFilePrefix = "DETS"; - when(fileOperationService.createFile(anyString(), anyString(), anyBoolean())).thenReturn(tempDirectory.toPath()); - when(armDataManagementConfiguration.getFileExtension()).thenReturn("a360"); - when(armDataManagementConfiguration.getTempBlobWorkspace()).thenReturn("/temp_workspace"); - - // when - File result = dataStoreToArmHelper.createEmptyArchiveRecordsFile(manifestFilePrefix); - - // then - assertNotNull(result); - } - @Test void writeManifestFile() { // given ArmBatchItems batchItems = mock(ArmBatchItems.class); - File archiveRecordsFile = mock(File.class); + ArchiveRecord archiveRecord = mock(ArchiveRecord.class); + List archiveRecords = List.of(archiveRecord); + when(batchItems.getArchiveRecords()).thenReturn(archiveRecords); // when - dataStoreToArmHelper.writeManifestFile(batchItems, archiveRecordsFile); + dataStoreToArmHelper.generateManifestFileContents(batchItems, "fileName"); // then - verify(archiveRecordFileGenerator).generateArchiveRecords(anyList(), any()); + verify(archiveRecordFileGenerator).generateArchiveRecords(eq("fileName"), eq(archiveRecords)); } @Test @@ -238,4 +226,14 @@ void getFileSize() { // then assertEquals(1000L, result); } + + + + @Test + void getArchiveRecordsFileName_typical() throws IOException { + when(armDataManagementConfiguration.getFileExtension()).thenReturn("a360"); + String name = dataStoreToArmHelper.getArchiveRecordsFileName("DARTS"); + + assertThat(name).matches("DARTS_.+\\.a360"); + } } \ No newline at end of file diff --git a/src/test/java/uk/gov/hmcts/darts/arm/service/UnstructuredToArmBatchProcessorTest.java b/src/test/java/uk/gov/hmcts/darts/arm/service/UnstructuredToArmBatchProcessorTest.java index 4a9ce4ca61..44d988678b 100644 --- a/src/test/java/uk/gov/hmcts/darts/arm/service/UnstructuredToArmBatchProcessorTest.java +++ b/src/test/java/uk/gov/hmcts/darts/arm/service/UnstructuredToArmBatchProcessorTest.java @@ -39,7 +39,6 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.matches; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -175,8 +174,6 @@ void testPaginatedBatchQuery() throws IOException { when(unstructuredToArmProcessorConfiguration.getThreads()).thenReturn(20); when(armDataManagementConfiguration.getMaxRetryAttempts()).thenReturn(3); - when(fileOperationService.createFile(any(), any(), anyBoolean())).thenReturn(manifestFilePath); - //when unstructuredToArmBatchProcessor.processUnstructuredToArm(5000); @@ -195,25 +192,4 @@ void testPaginatedBatchQuery() throws IOException { verifyNoMoreInteractions(logApi); } - - @Test - void testManifestFileName() throws IOException { - //given - when(armDataManagementConfiguration.getManifestFilePrefix()).thenReturn("DARTS"); - when(armDataManagementConfiguration.getFileExtension()).thenReturn("a360"); - when(armDataManagementConfiguration.getTempBlobWorkspace()).thenReturn("/temp_workspace"); - when(externalObjectDirectoryRepository.findEodsNotInOtherStorage(any(), any(), any(), any())).thenReturn(List.of(123)); - when(unstructuredToArmProcessorConfiguration.getMaxArmManifestItems()).thenReturn(1000); - when(unstructuredToArmProcessorConfiguration.getThreads()).thenReturn(20); - when(armDataManagementConfiguration.getMaxRetryAttempts()).thenReturn(3); - - //when - unstructuredToArmBatchProcessor.processUnstructuredToArm(5); - - //then - verify(fileOperationService).createFile(matches("DARTS_.+\\.a360"), eq("/temp_workspace"), eq(true)); - - verifyNoMoreInteractions(logApi); - } - } diff --git a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/DetsToArmBatchPushProcessorImplTest.java b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/DetsToArmBatchPushProcessorImplTest.java index cd90671bbb..0a998d41bf 100644 --- a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/DetsToArmBatchPushProcessorImplTest.java +++ b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/DetsToArmBatchPushProcessorImplTest.java @@ -44,14 +44,10 @@ import java.util.UUID; import static java.util.Collections.emptyList; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.matches; import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -73,7 +69,6 @@ class DetsToArmBatchPushProcessorImplTest { private ExternalObjectDirectoryEntity externalObjectDirectoryEntityDets; - private ExternalObjectDirectoryEntity externalObjectDirectoryEntityArm; @Mock private FileOperationService fileOperationService; @Mock @@ -140,11 +135,12 @@ void setUp() throws IOException { DETS_UUID); externalObjectDirectoryEntityDets.setOsrUuid(OSR_UUID); - externalObjectDirectoryEntityArm = new ExternalObjectDirectoryTestData().createExternalObjectDirectory( - mediaEntity1, - ExternalLocationTypeEnum.ARM, - ObjectRecordStatusEnum.ARM_INGESTION, - UUID.randomUUID()); + ExternalObjectDirectoryEntity externalObjectDirectoryEntityArm = new ExternalObjectDirectoryTestData() + .createExternalObjectDirectory( + mediaEntity1, + ExternalLocationTypeEnum.ARM, + ObjectRecordStatusEnum.ARM_INGESTION, + UUID.randomUUID()); externalObjectDirectoryEntityArm.setId(345); externalObjectDirectoryEntityArm.setStatus(EodHelper.armIngestionStatus()); objectStateRecordEntity = createMaxObjectStateRecordEntity(888L, @@ -206,32 +202,6 @@ void processDetsToArmSetObjectStatusNoMatchingDetsRecordErrorMessage() { } - @Test - void testManifestFileName() throws IOException { - //given - when(externalObjectDirectoryRepository.findEodsNotInOtherStorage(any(), any(), any(), any())).thenReturn(List.of(123)); - when(externalObjectDirectoryRepository.findAllById(List.of(123))).thenReturn(List.of(externalObjectDirectoryEntityDets)); - when(detsToArmProcessorConfiguration.getMaxArmManifestItems()).thenReturn(1000); - when(armDataManagementConfiguration.getMaxRetryAttempts()).thenReturn(3); - - when(externalObjectDirectoryRepository.findEodsNotInOtherStorage( - EodHelper.storedStatus(), - EodHelper.detsLocation(), - EodHelper.armLocation(), 5 - )).thenReturn(List.of(123)); - - when(externalObjectDirectoryRepository.saveAndFlush(any())).thenReturn(externalObjectDirectoryEntityArm); - externalObjectDirectoryEntityArm.setStatus(EodHelper.armIngestionStatus()); - - //when - detsToArmBatchPushProcessor.processDetsToArm(5); - - //then - verify(fileOperationService).createFile(matches("DETS_.+\\.a360"), eq(tempDirectory.getAbsolutePath()), eq(true)); - assertNotNull(objectStateRecordEntity.getObjectStatus()); - - } - public ObjectStateRecordEntity createMaxObjectStateRecordEntity(Long uuid, int detsEodId, int armEodId) { ObjectStateRecordEntity objectStateRecordEntity = new ObjectStateRecordEntity(); objectStateRecordEntity.setUuid(uuid);