Merge pull request #8 from DiSSCo/feature/implement-prov-o

Implement new CreateUpdateTombstoneEvent
DiSSCo · Jul 16, 2024 · b3a3286 · b3a3286
2 parents 33bf2e1 + c00820a
commit b3a3286
Show file tree

Hide file tree

Showing 10 changed files with 439 additions and 166 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -35,7 +35,7 @@ jobs:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar 
                   -Dsonar.projectKey=DiSSCo_${{ github.event.repository.name }}
-                  -Dsonar.coverage.exclusions=**/properties/**,**/configuration/**,**/exception/**
+                  -Dsonar.coverage.exclusions=**/properties/**,**/configuration/**,**/exception/**,**/maven/**
       - name: Login to Public ECR
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v1

diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.springframework.boot</groupId>
     <artifactId>spring-boot-starter-parent</artifactId>
-    <version>3.2.4</version>
+    <version>3.3.1</version>
     <relativePath/> <!-- lookup parent from repository -->
   </parent>
   <groupId>eu.dissco.core</groupId>
@@ -16,7 +16,7 @@
   <description>DiSSCo core consumer for event provenance records</description>
   <properties>
     <java.version>17</java.version>
-    <testcontainers.version>1.19.7</testcontainers.version>
+    <testcontainers.version>1.19.8</testcontainers.version>
     <sonar.organization>dissco</sonar.organization>
     <sonar.host.url>https://sonarcloud.io</sonar.host.url>
     <sonar.coverage.jacoco.xmlReportPaths>../app-it/target/site/jacoco-aggregate/jacoco.xml
@@ -117,6 +117,17 @@
           </excludes>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <version>3.3.0</version>
+        <configuration>
+          <mainClass>eu.dissco.core.provenanceservice.maven.MavenRunner</mainClass>
+          <arguments>
+            <argument>https://schemas.dissco.tech/schemas/fdo-type/create-update-tombstone-event/0.3.0/create-update-tombstone-event.json</argument>
+          </arguments>
+        </configuration>
+      </plugin>
       <plugin>
         <groupId>org.jacoco</groupId>
         <artifactId>jacoco-maven-plugin</artifactId>
@@ -155,6 +166,28 @@
           <useSystemClassLoader>false</useSystemClassLoader>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.jsonschema2pojo</groupId>
+        <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+        <version>1.2.1</version>
+        <configuration>
+          <sourceDirectory>${basedir}/src/main/resources/json-schema/</sourceDirectory>
+          <targetPackage>eu.dissco.core.provenanceservice.schema</targetPackage>
+          <generateBuilders>true</generateBuilders>
+          <includeConstructors>true</includeConstructors>
+          <includeDynamicBuilders>true</includeDynamicBuilders>
+          <includeAdditionalProperties>true</includeAdditionalProperties>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>
+                generate
+              </goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 

diff --git a/src/main/java/eu/dissco/core/provenanceservice/domain/CreateUpdateDeleteEvent.java b/src/main/java/eu/dissco/core/provenanceservice/domain/CreateUpdateDeleteEvent.java
diff --git a/src/main/java/eu/dissco/core/provenanceservice/maven/MavenRunner.java b/src/main/java/eu/dissco/core/provenanceservice/maven/MavenRunner.java
@@ -0,0 +1,58 @@
+package eu.dissco.core.provenanceservice.maven;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.web.bind.annotation.RequestMethod;
+
+public class MavenRunner {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(MavenRunner.class);
+
+  public static void main(String[] args) {
+    LOGGER.info("Starting the MavenRunner to download and parse json schemas");
+    for (String schemaUrl : args) {
+      LOGGER.info("Processing json schema: {}", schemaUrl);
+      var fileName = schemaUrl.substring(schemaUrl.lastIndexOf('/') + 1);
+      String outputFilePath = "src/main/resources/json-schema/" + fileName;
+      try {
+        String schema = downloadSchema(schemaUrl);
+        saveSchemaToFile(schema, outputFilePath);
+        LOGGER.info("JSON schema downloaded and saved to: {} ", outputFilePath);
+      } catch (IOException e) {
+        LOGGER.error("Error downloading or saving the JSON schema", e);
+      }
+    }
+  }
+
+  private static String downloadSchema(String schemaUrl) throws IOException {
+    StringBuilder result = new StringBuilder();
+    URL url = new URL(schemaUrl);
+    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    conn.setRequestMethod(RequestMethod.GET.name());
+    try (BufferedReader reader = new BufferedReader(
+        new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) {
+      String line;
+      while ((line = reader.readLine()) != null) {
+        result.append(line).append("\n");
+      }
+    }
+    return result.toString();
+  }
+
+  private static void saveSchemaToFile(String schema, String filePath) throws IOException {
+    try (BufferedWriter writer = new BufferedWriter(
+        new OutputStreamWriter(new FileOutputStream(filePath), StandardCharsets.UTF_8))) {
+      writer.write(schema);
+    }
+  }
+
+}
diff --git a/src/main/java/eu/dissco/core/provenanceservice/repository/EventRepository.java b/src/main/java/eu/dissco/core/provenanceservice/repository/EventRepository.java
@@ -4,7 +4,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.mongodb.client.MongoDatabase;
 import com.mongodb.client.model.ReplaceOptions;
-import eu.dissco.core.provenanceservice.domain.CreateUpdateDeleteEvent;
+import eu.dissco.core.provenanceservice.schema.CreateUpdateTombstoneEvent;
 import lombok.RequiredArgsConstructor;
 import org.bson.Document;
 import org.springframework.stereotype.Repository;
@@ -16,7 +16,7 @@ public class EventRepository {
   private final MongoDatabase database;
   private final ObjectMapper mapper;
 
-  public boolean insertNewVersion(String versionId, CreateUpdateDeleteEvent event,
+  public boolean insertNewVersion(String versionId, CreateUpdateTombstoneEvent event,
       String collectionName)
       throws JsonProcessingException {
     var collection = database.getCollection(collectionName);

diff --git a/src/main/java/eu/dissco/core/provenanceservice/service/KafkaConsumerService.java b/src/main/java/eu/dissco/core/provenanceservice/service/KafkaConsumerService.java
@@ -21,10 +21,7 @@ public class KafkaConsumerService {
 
   private final ProcessingService processingService;
 
-  @RetryableTopic(
-      backoff = @Backoff(value = 3000L),
-      attempts = "3",
-      autoCreateTopics = "true")
+  @RetryableTopic(backoff = @Backoff(value = 3000L))
   @KafkaListener(topics = "${spring.kafka.consumer.topic}")
   public void getMessages(@Payload String message)
       throws JsonProcessingException, MongodbException, UnknownSubjectException {

diff --git a/src/main/java/eu/dissco/core/provenanceservice/service/ProcessingService.java b/src/main/java/eu/dissco/core/provenanceservice/service/ProcessingService.java
@@ -1,12 +1,11 @@
 package eu.dissco.core.provenanceservice.service;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import eu.dissco.core.provenanceservice.domain.CreateUpdateDeleteEvent;
 import eu.dissco.core.provenanceservice.exception.MongodbException;
 import eu.dissco.core.provenanceservice.exception.UnknownSubjectException;
 import eu.dissco.core.provenanceservice.repository.EventRepository;
+import eu.dissco.core.provenanceservice.schema.CreateUpdateTombstoneEvent;
 import java.util.HashMap;
 import java.util.Map;
 import lombok.RequiredArgsConstructor;
@@ -18,47 +17,45 @@
 @RequiredArgsConstructor
 public class ProcessingService {
 
+  private static final Map<String, String> SUBJECT_MAPPING = provideSubjectMapping();
   private final ObjectMapper mapper;
   private final EventRepository eventRepository;
-  private static final Map<String, String> SUBJECT_MAPPING = provideSubjectMapping();
 
   private static Map<String, String> provideSubjectMapping() {
     var map = new HashMap<String, String>();
-    map.put("DigitalSpecimen", "digital_specimen_provenance");
-    map.put("DigitalMediaObject", "digital_media_provenance");
-    map.put("Annotation", "annotation_provenance");
-    map.put("MachineAnnotationService", "machine_annotation_service_provenance");
-    map.put("Mapping", "mapping_provenance");
-    map.put("SourceSystem", "source_system_provenance");
+    map.put("ods:DigitalSpecimen", "digital_specimen_provenance");
+    map.put("ods:DigitalMedia", "digital_media_provenance");
+    map.put("ods:Annotation", "annotation_provenance");
+    map.put("ods:MachineAnnotationService", "machine_annotation_service_provenance");
+    map.put("ods:DataMapping", "data_mapping_provenance");
+    map.put("ods:SourceSystem", "source_system_provenance");
     return map;
   }
 
   public void handleMessage(String message)
       throws JsonProcessingException, MongodbException, UnknownSubjectException {
-    var event = mapper.readValue(message, CreateUpdateDeleteEvent.class);
-    var versionId = generateUniqueVersionId(event.eventRecord());
-    var collectionName = parseSubjectType(event.subjectType());
+    var event = mapper.readValue(message, CreateUpdateTombstoneEvent.class);
+    var versionId = event.getId();
+    var collectionName = parseSubjectType(event);
     var eventResult = eventRepository.insertNewVersion(versionId, event, collectionName);
     if (eventResult) {
-      log.info("Successfully processed event information for {}: {}", event.subjectType(),
+      log.info("Successfully processed event information for {}: {}",
+          event.getProvEntity().getType(),
           versionId);
     } else {
       log.warn("Failed to insert event into mongodb: {}", message);
       throw new MongodbException(message);
     }
   }
 
-  private String parseSubjectType(String subjectType) throws UnknownSubjectException {
-    var collectionName = SUBJECT_MAPPING.get(subjectType);
-    if (collectionName == null){
-      throw new UnknownSubjectException("SubjectType: " + subjectType + " is unknown");
+  private String parseSubjectType(CreateUpdateTombstoneEvent event) throws UnknownSubjectException {
+    var collectionName = SUBJECT_MAPPING.get(event.getProvEntity().getType());
+    if (collectionName == null) {
+      throw new UnknownSubjectException(
+          "SubjectType: " + event.getProvEntity().getType() + " is unknown");
     } else {
       return collectionName;
     }
   }
 
-  private String generateUniqueVersionId(JsonNode eventRecord) {
-    return eventRecord.get("id").asText() + "/" + eventRecord.get("version").asInt();
-  }
-
 }