DMP-3125 (#1495)

Co-authored-by: hmcts-jenkins-cnp <60659747+hmcts-jenkins-cnp[bot]@users.noreply.github.com>

Dockerfile

@@ -9,9 +9,14 @@ WORKDIR /usr/local/bin
 ADD lib/ffmpeg-release-amd64-static.tar.xz /usr/local/bin
 RUN cp -p ffmpeg*/ffmpeg /usr/bin
 
+ADD lib/azcopy_linux_amd64_10.24.0.tar.gz /usr/local/bin
+RUN cp -p azcopy*/azcopy /usr/bin
+RUN chmod 777 /usr/bin/azcopy
+
  # renovate: datasource=github-releases depName=microsoft/ApplicationInsights-Java
 FROM hmctspublic.azurecr.io/base/java:17-distroless
 COPY --from=build-env /usr/bin/ffmpeg /usr/bin
+COPY --from=build-env /usr/bin/azcopy /usr/bin
 
 COPY lib/applicationinsights.json /opt/app/
 COPY build/libs/darts-api.jar /opt/app/

Jenkinsfile_CNP

@@ -43,7 +43,9 @@ def secrets = [
                 secret('AzureAdB2CFuncTestROPCGlobalUsername', 'AZURE_AD_FUNCTIONAL_TEST_GLOBAL_USERNAME'),
                 secret('AzureAdB2CFuncTestROPCGlobalPassword', 'AZURE_AD_FUNCTIONAL_TEST_GLOBAL_PASSWORD'),
                 secret('ARMSasEndpoint', 'ARM_SAS_ENDPOINT'),
-                secret('DETSSasURLEndpoint', 'DETS_SAS_URL_ENDPOINT')
+                secret('DETSSasURLEndpoint', 'DETS_SAS_URL_ENDPOINT'),
+                secret('DartsInboundStorageSasUrl', 'DARTS_INBOUND_STORAGE_SAS_URL'),
+                secret('DartsUnstructuredStorageSasUrl', 'DARTS_UNSTRUCTURED_STORAGE_SAS_URL')
         ],
 ]
 

Jenkinsfile_nightly

@@ -44,8 +44,9 @@ def secrets = [
                 secret('AzureAdB2CFuncTestROPCGlobalUsername', 'AZURE_AD_FUNCTIONAL_TEST_GLOBAL_USERNAME'),
                 secret('AzureAdB2CFuncTestROPCGlobalPassword', 'AZURE_AD_FUNCTIONAL_TEST_GLOBAL_PASSWORD'),
                 secret('ARMSasEndpoint', 'ARM_SAS_ENDPOINT'),
-                secret('DETSSasURLEndpoint', 'DETS_SAS_URL_ENDPOINT')
-
+                secret('DETSSasURLEndpoint', 'DETS_SAS_URL_ENDPOINT'),
+                secret('DartsInboundStorageSasUrl', 'DARTS_INBOUND_STORAGE_SAS_URL'),
+                secret('DartsUnstructuredStorageSasUrl', 'DARTS_UNSTRUCTURED_STORAGE_SAS_URL')
         ],
 ]
 

Jenkinsfile_parameterized

@@ -40,7 +40,9 @@ def secrets = [
                 secret('AzureAdB2CFuncTestROPCGlobalUsername', 'AZURE_AD_FUNCTIONAL_TEST_GLOBAL_USERNAME'),
                 secret('AzureAdB2CFuncTestROPCGlobalPassword', 'AZURE_AD_FUNCTIONAL_TEST_GLOBAL_PASSWORD'),
                 secret('ARMSasEndpoint', 'ARM_SAS_ENDPOINT'),
-                secret('DETSSasURLEndpoint', 'DETS_SAS_URL_ENDPOINT')
+                secret('DETSSasURLEndpoint', 'DETS_SAS_URL_ENDPOINT'),
+                secret('DartsInboundStorageSasUrl', 'DARTS_INBOUND_STORAGE_SAS_URL'),
+                secret('DartsUnstructuredStorageSasUrl', 'DARTS_UNSTRUCTURED_STORAGE_SAS_URL')
         ],
 ]
 

README.md

@@ -56,6 +56,8 @@ The required value of each variable is stored in Azure Key Vault as a Secret.
 | AAD_TENANT_ID_JUSTICE                    | AzureADTenantIdJustice                    |
 | AAD_CLIENT_ID_JUSTICE                    | AzureADClientIdJustice                    |
 | AAD_CLIENT_SECRET_JUSTICE                | AzureADClientSecretJustice                |
+| DARTS_INBOUND_STORAGE_SAS_URL            | DartsInboundStorageSasUrl                 |
+| DARTS_UNSTRUCTURED_STORAGE_SAS_URL       | DartsUnstructuredStorageSasUrl            |
 
 There are few attributes which doesn't use Azure Keyvault secrets. Those environment variable values are controlled dynamically via Flux config
 
@@ -124,6 +126,8 @@ export ARM_SAS_ENDPOINT=
 export ARM_URL=
 export ARM_USERNAME=
 export ARM_PASSWORD=
+export DARTS_INBOUND_STORAGE_SAS_URL=
+export DARTS_UNSTRUCTURED_STORAGE_SAS_URL=
 ```
 
 ### Storage Account

bin/secrets-stg.sh

@@ -37,4 +37,6 @@ export AAD_TENANT_ID_JUSTICE="$(az keyvault secret show --vault-name darts-stg -
 export AAD_CLIENT_ID_JUSTICE="$(az keyvault secret show --vault-name darts-stg --name AzureADClientIdJustice | jq .value -r)"
 export AAD_CLIENT_SECRET_JUSTICE="$(az keyvault secret show --vault-name darts-stg --name AzureADClientSecretJustice | jq .value -r)"
 export MAX_FILE_UPLOAD_SIZE_MEGABYTES="$(az keyvault secret show --vault-name darts-stg --name MaxFileUploadSizeInMegabytes | jq .value -r)"
-export MAX_FILE_UPLOAD_REQUEST_SIZE_MEGABYTES="$(az keyvault secret show --vault-name darts-stg --name MaxFileUploadRequestSizeInMegabytes | jq .value -r)"
+export MAX_FILE_UPLOAD_REQUEST_SIZE_MEGABYTES="$(az keyvault secret show --vault-name darts-stg --name MaxFileUploadRequestSizeInMegabytes | jq .value -r)"
+export DARTS_INBOUND_STORAGE_SAS_URL="$(az keyvault secret show --vault-name darts-stg --name DartsInboundStorageSasUrl | jq .value -r)"
+export DARTS_UNSTRUCTURED_STORAGE_SAS_URL="$(az keyvault secret show --vault-name darts-stg --name DartsUnstructuredStorageSasUrl | jq .value -r)"

build.gradle

@@ -266,7 +266,7 @@ sonarqube {
     property "sonar.projectName", "DARTS :: darts-api"
     property "sonar.projectKey", "uk.gov.hmcts.reform:darts-api"
     property "sonar.exclusions", coverageExclusions.join(', ')
-    property 'sonar.coverage.exclusions', "**/entity/*,**/dto/*"
+    property 'sonar.coverage.exclusions', "**/entity/*,**/dto/*,**/AzureCopyUtil.java"
     //duplicate code here due to OpenAPI codegen that creates identical
     // objects (Transcript) in different packages
     property "sonar.cpd.exclusions", "**/TranscriptionMapper.java"

charts/darts-api/Chart.yaml

@@ -3,7 +3,7 @@ appVersion: "1.0"
 description: A Helm chart for darts-api App
 name: darts-api
 home: https://github.com/hmcts/darts-api
-version: 0.0.78
+version: 0.0.79
 maintainers:
   - name: HMCTS darts team
 dependencies:

charts/darts-api/values.dev.template.yaml

@@ -89,6 +89,10 @@ java:
           alias: ARM_USERNAME
         - name: ArmPassword
           alias: ARM_PASSWORD
+        - name: DartsInboundStorageSasUrl
+          alias: DARTS_INBOUND_STORAGE_SAS_URL
+        - name: DartsUnstructuredStorageSasUrl
+          alias: DARTS_UNSTRUCTURED_STORAGE_SAS_URL
   environment:
     ENABLE_FLYWAY: true
     RUN_DB_MIGRATION_ON_STARTUP: true

charts/darts-api/values.yaml

@@ -90,6 +90,10 @@ java:
           alias: MAX_FILE_UPLOAD_SIZE_MEGABYTES
         - name: MaxFileUploadRequestSizeInMegabytes
           alias: MAX_FILE_UPLOAD_REQUEST_SIZE_MEGABYTES
+        - name: DartsInboundStorageSasUrl
+          alias: DARTS_INBOUND_STORAGE_SAS_URL
+        - name: DartsUnstructuredStorageSasUrl
+          alias: DARTS_UNSTRUCTURED_STORAGE_SAS_URL
   environment:
     NOTIFICATION_SCHEDULER_CRON: "3 */2 * * * MON-FRI"
     POSTGRES_SSL_MODE: require
@@ -192,6 +196,10 @@ function:
           alias: ARM_USERNAME
         - name: ArmPassword
           alias: ARM_PASSWORD
+        - name: DartsInboundStorageSasUrl
+          alias: DARTS_INBOUND_STORAGE_SAS_URL
+        - name: DartsUnstructuredStorageSasUrl
+          alias: DARTS_UNSTRUCTURED_STORAGE_SAS_URL
   environment:
     ATS_MODE: true
     API_MODE: false

docker-compose-local.yml

@@ -52,6 +52,8 @@ services:
       - ACTIVE_DIRECTORY_B2C_AUTH_URI=https://hmctsstgextid.b2clogin.com/hmctsstgextid.onmicrosoft.com
       - MAX_FILE_UPLOAD_SIZE_MEGABYTES=350
       - MAX_FILE_UPLOAD_REQUEST_SIZE_MEGABYTES=360
+      - DARTS_INBOUND_STORAGE_SAS_URL
+      - DARTS_UNSTRUCTURED_STORAGE_SAS_URL
     build:
       context: .
       dockerfile: Dockerfile

src/functionalTest/java/uk/gov/hmcts/darts/datamanagement/DataManagementServiceTest.java

@@ -13,6 +13,8 @@
 import org.springframework.test.context.ActiveProfiles;
 import uk.gov.hmcts.darts.common.datamanagement.component.impl.DownloadResponseMetaData;
 import uk.gov.hmcts.darts.common.datamanagement.enums.DatastoreContainerType;
+import uk.gov.hmcts.darts.common.exception.AzureDeleteBlobException;
+import uk.gov.hmcts.darts.datamanagement.config.DataManagementConfiguration;
 import uk.gov.hmcts.darts.datamanagement.exception.FileNotDownloadedException;
 import uk.gov.hmcts.darts.datamanagement.service.DataManagementService;
 
@@ -42,6 +44,8 @@ class DataManagementServiceTest {
 
     @Autowired
     DataManagementService dataManagementService;
+    @Autowired
+    DataManagementConfiguration dataManagementConfiguration;
 
     @Test
     void saveBinaryDataToBlobStorage() {
@@ -116,4 +120,28 @@ void saveBlobDataShouldSucceedAndReturnUuid() {
         assertTrue(StringUtils.isNotEmpty(uuid.toString()));
     }
 
+    @Test
+    void copyBetweenStorageContainers() throws AzureDeleteBlobException {
+
+        byte[] testStringInBytes = TEST_BINARY_STRING.getBytes(StandardCharsets.UTF_8);
+        BinaryData data = BinaryData.fromBytes(testStringInBytes);
+
+        var sourceUuid = dataManagementService.saveBlobData(dataManagementConfiguration.getInboundContainerName(), data);
+
+        UUID destinationUuid = dataManagementService.copyBlobData(
+            dataManagementConfiguration.getInboundContainerName(),
+            dataManagementConfiguration.getUnstructuredContainerName(),
+            sourceUuid);
+
+        var blobData = dataManagementService.getBlobData(
+            dataManagementConfiguration.getUnstructuredContainerName(),
+            destinationUuid
+        );
+
+        dataManagementService.deleteBlobData(dataManagementConfiguration.getInboundContainerName(), sourceUuid);
+        dataManagementService.deleteBlobData(dataManagementConfiguration.getUnstructuredContainerName(), destinationUuid);
+
+        assertEquals(TEST_BINARY_STRING, blobData.toString());
+    }
+
 }

src/integrationTest/java/uk/gov/hmcts/darts/retention/service/ApplyRetentionCaseAssociatedObjectsProcessorIntTest.java

@@ -271,9 +271,9 @@ void testSuccessfullyApplyRetentionToCaseTranscriptionDocuments() {
         tr2.addHearing(hear2);
         transcriptionRepository.save(tr2);
 
-        transcriptionStub.updateTranscriptionWithDocument(tr1, STORED, INBOUND);
-        transcriptionStub.updateTranscriptionWithDocument(tr1, STORED, INBOUND);
-        transcriptionStub.updateTranscriptionWithDocument(tr2, STORED, INBOUND);
+        transcriptionStub.updateTranscriptionWithDocument(tr1, STORED, INBOUND, UUID.randomUUID());
+        transcriptionStub.updateTranscriptionWithDocument(tr1, STORED, INBOUND, UUID.randomUUID());
+        transcriptionStub.updateTranscriptionWithDocument(tr2, STORED, INBOUND, UUID.randomUUID());
 
         var trDoc1 = tr1.getTranscriptionDocumentEntities().get(0);
         eodStub.createAndSaveExternalObjectDirectory(trDoc1.getId(), EodHelper.armDropZoneStatus(), EodHelper.armLocation());