Feature/refactor presigned url usage (#1839)

* cosys-integration add endpoint without presigned url parameter * cosys-integration add new code to tests * mail-integration init endpoint without presigned url parameter * docs endpoints without presigned url parameter * cosys-integration update endpoint without presigned url parameter * mail-integration update endpoint without presigned url parameter * s3-integration mark streaming presigned url as deprecated * update s3 example process without presigned url usage * cosys-integration update endpoint without presigned url parameter * mail-integration update endpoint without presigned url * mail-integration update endpoint without presigned url * mail-integration fix application.yml
it-at-m · Jul 12, 2024 · f3b6193 · f3b6193
1 parent 5926e61
commit f3b6193
Show file tree

Hide file tree

Showing 55 changed files with 2,048 additions and 616 deletions.
diff --git a/...giwf-engine-service/src/main/resources/prozesse/example/s3-integration/S3TestProzess.bpmn b/...giwf-engine-service/src/main/resources/prozesse/example/s3-integration/S3TestProzess.bpmn
diff --git a/...ervice/src/main/resources/prozesse/example/s3-integration/S3TestProzessPresignedUrls.bpmn b/...ervice/src/main/resources/prozesse/example/s3-integration/S3TestProzessPresignedUrls.bpmn
diff --git a/...ervice/src/main/resources/prozesse/example/s3-integration/s3-presigned.processconfig.json b/...ervice/src/main/resources/prozesse/example/s3-integration/s3-presigned.processconfig.json
@@ -0,0 +1,14 @@
+{
+  "key": "feature-s3-integration-presigned",
+  "statusDokument": "",
+  "configs": [
+    {
+      "key": "app_file_paths_readonly",
+      "value": ""
+    },
+    {
+      "key": "app_file_paths",
+      "value": "test"
+    }
+  ]
+}
diff --git a/...va/de/muenchen/oss/digiwf/cosys/integration/adapter/in/streaming/GenerateDocumentDTO.java b/...va/de/muenchen/oss/digiwf/cosys/integration/adapter/in/streaming/GenerateDocumentDTO.java
@@ -0,0 +1,31 @@
+package de.muenchen.oss.digiwf.cosys.integration.adapter.in.streaming;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import de.muenchen.oss.digiwf.cosys.integration.domain.model.GenerateDocument;
+import jakarta.validation.constraints.NotBlank;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+
+@EqualsAndHashCode(callSuper = true)
+@Data
+@NoArgsConstructor
+public class GenerateDocumentDTO extends GenerateDocument {
+    @NotBlank
+    private String fileContext;
+
+    @NotBlank
+    private String filePath;
+
+    public GenerateDocumentDTO(
+            @NotBlank(message = "client is mandatory") String client,
+            @NotBlank(message = "role is mandatory") String role,
+            @NotBlank(message = "guid is mandatory") String guid,
+            JsonNode variables,
+            String fileContext,
+            String filePath) {
+        super(client, role, guid, variables);
+        this.fileContext = fileContext;
+        this.filePath = filePath;
+    }
+}
diff --git a/...n/oss/digiwf/cosys/integration/adapter/in/streaming/GenerateDocumentPresignedUrlsDTO.java b/...n/oss/digiwf/cosys/integration/adapter/in/streaming/GenerateDocumentPresignedUrlsDTO.java
@@ -0,0 +1,36 @@
+package de.muenchen.oss.digiwf.cosys.integration.adapter.in.streaming;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import de.muenchen.oss.digiwf.cosys.integration.domain.model.DocumentStorageUrl;
+import de.muenchen.oss.digiwf.cosys.integration.domain.model.GenerateDocument;
+import jakarta.validation.Valid;
+import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.Size;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Deprecated
+@EqualsAndHashCode(callSuper = true)
+@Data
+@NoArgsConstructor
+public class GenerateDocumentPresignedUrlsDTO extends GenerateDocument {
+    /**
+     * A list of presigned urls that are used to save the cosys documents in a s3 storage
+     */
+    @Valid
+    @Size(min = 1, max = 1)
+    private List<DocumentStorageUrl> documentStorageUrls;
+
+    public GenerateDocumentPresignedUrlsDTO(
+            @NotBlank(message = "client is mandatory") String client,
+            @NotBlank(message = "role is mandatory") String role,
+            @NotBlank(message = "guid is mandatory") String guid,
+            JsonNode variables,
+            List<DocumentStorageUrl> documentStorageUrls) {
+        super(client, role, guid, variables);
+        this.documentStorageUrls = documentStorageUrls;
+    }
+}
diff --git a/.../java/de/muenchen/oss/digiwf/cosys/integration/adapter/in/streaming/StreamingAdapter.java b/.../java/de/muenchen/oss/digiwf/cosys/integration/adapter/in/streaming/StreamingAdapter.java
@@ -2,7 +2,6 @@
 
 
 import de.muenchen.oss.digiwf.cosys.integration.application.port.in.CreateDocumentInPort;
-import de.muenchen.oss.digiwf.cosys.integration.domain.model.GenerateDocument;
 import de.muenchen.oss.digiwf.message.process.api.ErrorApi;
 import de.muenchen.oss.digiwf.message.process.api.ProcessApi;
 import de.muenchen.oss.digiwf.message.process.api.error.BpmnError;
@@ -13,6 +12,7 @@
 import org.springframework.messaging.Message;
 
 import java.util.Map;
+import java.util.Objects;
 import java.util.function.Consumer;
 
 import static de.muenchen.oss.digiwf.message.common.MessageConstants.*;
@@ -26,26 +26,50 @@ public class StreamingAdapter {
     private final ProcessApi processApi;
     private final ErrorApi errorApi;
 
-    public Consumer<Message<GenerateDocument>> createCosysDocument() {
-        return message -> {
-            try {
-                log.info("Processing generate document request from eventbus");
-                final GenerateDocument document = message.getPayload();
-                log.debug("Generate document request: {}", document);
-                this.createDocumentInPort.createDocument(document);
-                this.processApi.correlateMessage(
-                        message.getHeaders().get(DIGIWF_PROCESS_INSTANCE_ID, String.class),
-                        message.getHeaders().get(TYPE, String.class),
-                        message.getHeaders().get(DIGIWF_INTEGRATION_NAME, String.class),
-                        Map.of("status", true));
-            } catch (final BpmnError bpmnError) {
-                this.errorApi.handleBpmnError(message.getHeaders(), bpmnError);
-            } catch (final ValidationException validationException) {
-                this.errorApi.handleBpmnError(message.getHeaders(), new BpmnError("VALIDATION_ERROR", validationException.getMessage()));
-            } catch (final IncidentError incidentError) {
-                this.errorApi.handleIncident(message.getHeaders(), incidentError);
-            }
-        };
+    @Deprecated
+    public Consumer<Message<GenerateDocumentPresignedUrlsDTO>> createCosysDocument() {
+        return message -> withErrorHandling(message, () -> {
+            log.info("Processing generate document request from eventbus");
+            final GenerateDocumentPresignedUrlsDTO document = message.getPayload();
+            log.debug("Generate document request: {}", document);
+            this.createDocumentInPort.createDocument(document, document.getDocumentStorageUrls());
+            this.correlateMessage(
+                    message.getHeaders(),
+                    Map.of("status", true)
+            );
+        });
     }
 
+    public Consumer<Message<GenerateDocumentDTO>> createCosysDocumentV2() {
+        return message -> withErrorHandling(message, () -> {
+            log.info("Processing generate document request from eventbus");
+            final GenerateDocumentDTO document = message.getPayload();
+            log.debug("Generate document request: {}", document);
+            this.createDocumentInPort.createDocument(document, document.getFileContext(), document.getFilePath());
+            this.correlateMessage(
+                    message.getHeaders(),
+                    Map.of("status", true)
+            );
+        });
+    }
+
+    private void withErrorHandling(final Message<?> message, final Runnable runnable) {
+        try {
+            runnable.run();
+        } catch (final BpmnError bpmnError) {
+            this.errorApi.handleBpmnError(message.getHeaders(), bpmnError);
+        } catch (final ValidationException validationException) {
+            this.errorApi.handleBpmnError(message.getHeaders(), new BpmnError("VALIDATION_ERROR", validationException.getMessage()));
+        } catch (final IncidentError incidentError) {
+            this.errorApi.handleIncident(message.getHeaders(), incidentError);
+        }
+    }
+
+    private void correlateMessage(final Map<String, Object> originMessageHeaders, final Map<String, Object> message) {
+        String processInstanceId = Objects.requireNonNull(originMessageHeaders.get(DIGIWF_PROCESS_INSTANCE_ID).toString());
+        String type = Objects.requireNonNull(originMessageHeaders.get(TYPE).toString());
+        String integrationName = Objects.requireNonNull(originMessageHeaders.get(DIGIWF_INTEGRATION_NAME).toString());
+        log.info("sending response message for process {}: {}", processInstanceId, message);
+        this.processApi.correlateMessage(processInstanceId, type, integrationName, message);
+    }
 }
diff --git a/...core/src/main/java/de/muenchen/oss/digiwf/cosys/integration/adapter/out/s3/S3Adapter.java b/...core/src/main/java/de/muenchen/oss/digiwf/cosys/integration/adapter/out/s3/S3Adapter.java
@@ -2,31 +2,40 @@
 
 import de.muenchen.oss.digiwf.cosys.integration.application.port.out.SaveFileToStorageOutPort;
 import de.muenchen.oss.digiwf.cosys.integration.domain.model.DocumentStorageUrl;
-import de.muenchen.oss.digiwf.cosys.integration.domain.model.GenerateDocument;
 import de.muenchen.oss.digiwf.message.process.api.error.BpmnError;
 import de.muenchen.oss.digiwf.s3.integration.client.exception.DocumentStorageClientErrorException;
 import de.muenchen.oss.digiwf.s3.integration.client.exception.DocumentStorageException;
 import de.muenchen.oss.digiwf.s3.integration.client.exception.DocumentStorageServerErrorException;
+import de.muenchen.oss.digiwf.s3.integration.client.repository.DocumentStorageFileRepository;
 import de.muenchen.oss.digiwf.s3.integration.client.repository.transfer.S3FileTransferRepository;
 import de.muenchen.oss.digiwf.s3.integration.client.service.FileService;
+import de.muenchen.oss.digiwf.s3.integration.client.service.S3StorageUrlProvider;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import lombok.val;
 import org.springframework.util.unit.DataSize;
 
+import java.util.List;
+
 @Slf4j
 @RequiredArgsConstructor
 public class S3Adapter implements SaveFileToStorageOutPort {
 
     private static final String S3_FILE_SAVE_ERROR = "S3_FILE_SAVE_ERROR";
     private static final String S3_FILE_SIZE_ERROR = "S3_FILE_SIZE_ERROR";
     private final S3FileTransferRepository s3FileTransferRepository;
+    private final DocumentStorageFileRepository documentStorageFileRepository;
     private final FileService fileService;
+    private final S3StorageUrlProvider s3DomainService;
 
     @Override
-    public void saveDocumentInStorage(final GenerateDocument generateDocument, final byte[] data) {
+    public void saveDocumentInStorage(
+            final List<DocumentStorageUrl> documentStorageUrls,
+            final byte[] data
+    ) {
         try {
             validateFileSize(data);
-            for (final DocumentStorageUrl presignedUrl : generateDocument.getDocumentStorageUrls()) {
+            for (final DocumentStorageUrl presignedUrl : documentStorageUrls) {
                 if (presignedUrl.getAction().equalsIgnoreCase("POST")) {
                     this.s3FileTransferRepository.saveFile(presignedUrl.getUrl(), data);
                 } else if (presignedUrl.getAction().equalsIgnoreCase("PUT")) {
@@ -36,12 +45,30 @@ public void saveDocumentInStorage(final GenerateDocument generateDocument, final
                 }
             }
         } catch (final DocumentStorageClientErrorException | DocumentStorageServerErrorException |
-                DocumentStorageException ex) {
+                       DocumentStorageException ex) {
             log.debug("Document could not be saved.", ex);
             throw new BpmnError(S3_FILE_SAVE_ERROR, ex.getMessage());
         }
     }
 
+    @Override
+    public void saveDocumentInStorage(
+            final String fileContext,
+            final String filePath,
+            final byte[] data
+    ) {
+        final String s3Storage = s3DomainService.getDefaultDocumentStorageUrl();
+        val fullFilePath = String.format("%s/%s", fileContext, filePath).replace("//", "/");
+        try {
+            validateFileSize(data);
+            documentStorageFileRepository.saveFile(fullFilePath, data, 1, s3Storage);
+        } catch (DocumentStorageException | DocumentStorageClientErrorException |
+                 DocumentStorageServerErrorException e) {
+            log.debug("Document could not be saved.", e);
+            throw new BpmnError(S3_FILE_SAVE_ERROR, e.getMessage());
+        }
+    }
+
     private void validateFileSize(final byte[] data) {
         if (!fileService.isValidFileSize(data))
             throw new BpmnError(S3_FILE_SIZE_ERROR,

diff --git a/...va/de/muenchen/oss/digiwf/cosys/integration/application/port/in/CreateDocumentInPort.java b/...va/de/muenchen/oss/digiwf/cosys/integration/application/port/in/CreateDocumentInPort.java
@@ -1,10 +1,24 @@
 package de.muenchen.oss.digiwf.cosys.integration.application.port.in;
 
+import de.muenchen.oss.digiwf.cosys.integration.domain.model.DocumentStorageUrl;
 import de.muenchen.oss.digiwf.cosys.integration.domain.model.GenerateDocument;
 import jakarta.validation.Valid;
+import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.Size;
+
+import java.util.List;
 
 public interface CreateDocumentInPort {
 
-    void createDocument(@Valid final GenerateDocument generateDocument);
+    @Deprecated
+    void createDocument(
+            @Valid final GenerateDocument generateDocument,
+            @Valid @Size(min = 1, max = 1) final List<DocumentStorageUrl> documentStorageUrls
+    );
 
+    void createDocument(
+            @Valid final GenerateDocument generateDocument,
+            @NotBlank final String fileContext,
+            @NotBlank final String filePath
+    );
 }
diff --git a/.../muenchen/oss/digiwf/cosys/integration/application/port/out/SaveFileToStorageOutPort.java b/.../muenchen/oss/digiwf/cosys/integration/application/port/out/SaveFileToStorageOutPort.java
@@ -1,9 +1,20 @@
 package de.muenchen.oss.digiwf.cosys.integration.application.port.out;
 
-import de.muenchen.oss.digiwf.cosys.integration.domain.model.GenerateDocument;
+import de.muenchen.oss.digiwf.cosys.integration.domain.model.DocumentStorageUrl;
+
+import java.util.List;
 
 public interface SaveFileToStorageOutPort {
+    @Deprecated
+    void saveDocumentInStorage(
+            final List<DocumentStorageUrl> documentStorageUrls,
+            final byte[] data
+    );
 
-    void saveDocumentInStorage(final GenerateDocument generateDocument, final byte[] data);
+    void saveDocumentInStorage(
+            final String fileContext,
+            final String filePath,
+            final byte[] data
+    );
 
 }
diff --git a/...a/de/muenchen/oss/digiwf/cosys/integration/application/usecase/CreateDocumentUseCase.java b/...a/de/muenchen/oss/digiwf/cosys/integration/application/usecase/CreateDocumentUseCase.java
@@ -3,11 +3,16 @@
 import de.muenchen.oss.digiwf.cosys.integration.application.port.in.CreateDocumentInPort;
 import de.muenchen.oss.digiwf.cosys.integration.application.port.out.GenerateDocumentOutPort;
 import de.muenchen.oss.digiwf.cosys.integration.application.port.out.SaveFileToStorageOutPort;
+import de.muenchen.oss.digiwf.cosys.integration.domain.model.DocumentStorageUrl;
 import de.muenchen.oss.digiwf.cosys.integration.domain.model.GenerateDocument;
 import jakarta.validation.Valid;
+import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.Size;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
+import java.util.List;
+
 
 @Slf4j
 @RequiredArgsConstructor
@@ -22,11 +27,30 @@ public class CreateDocumentUseCase implements CreateDocumentInPort {
      *
      * @param generateDocument Data for generating documents
      */
+    @Deprecated
     @Override
-    public void createDocument(@Valid final GenerateDocument generateDocument) {
+    public void createDocument(
+            @Valid final GenerateDocument generateDocument,
+            @Valid @Size(min = 1, max = 1) final List<DocumentStorageUrl> documentStorageUrls
+    ) {
         final byte[] data = this.generateDocumentOutPort.generateCosysDocument(generateDocument).block();
-        this.saveFileToStorageOutPort.saveDocumentInStorage(generateDocument, data);
+        this.saveFileToStorageOutPort.saveDocumentInStorage(documentStorageUrls, data);
     }
 
-
+    /**
+     * Generate a document in Cosys and save it in S3 using file context and path.
+     *
+     * @param generateDocument Data for generating documents
+     * @param fileContext      File context to save document to
+     * @param filePath         Path to save document to
+     */
+    @Override
+    public void createDocument(
+            @Valid final GenerateDocument generateDocument,
+            @NotBlank final String fileContext,
+            @NotBlank final String filePath
+    ) {
+        final byte[] data = this.generateDocumentOutPort.generateCosysDocument(generateDocument).block();
+        this.saveFileToStorageOutPort.saveDocumentInStorage(fileContext, filePath, data);
+    }
 }
diff --git a/...c/main/java/de/muenchen/oss/digiwf/cosys/integration/domain/model/DocumentStorageUrl.java b/...c/main/java/de/muenchen/oss/digiwf/cosys/integration/domain/model/DocumentStorageUrl.java
@@ -7,6 +7,7 @@
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
 
+@Deprecated
 @Data
 @Builder
 @RequiredArgsConstructor