Fixes for:

* CatalogueOfLife/data#719 * CatalogueOfLife/data#722 Work for * #1350 Fixes for prefix mapping and metadata loading
CatalogueOfLife · Nov 5, 2024 · 5d5b171 · 5d5b171
1 parent b2ee90c
commit 5d5b171
Show file tree

Hide file tree

Showing 18 changed files with 594 additions and 231 deletions.
diff --git a/matching-ws/Dockerfile b/matching-ws/Dockerfile
@@ -27,14 +27,11 @@ RUN git clone https://github.com/CatalogueOfLife/backend.git
 WORKDIR /app/backend
 RUN git checkout $GIT_BRANCH
 
-# Build all the CLB modules
-RUN mvn clean install package -DskipTests
-
 # Build the Maven project and create a exec file
 WORKDIR /app/backend/matching-ws
 
-# Run tests - full backend tests require additional services (e.g. ES)
-RUN mvn clean install package
+# Build all the CLB modules
+RUN mvn clean install package -DskipTests -DskipITs
 
 # Store git commit id and log
 RUN curl -o /app/backend/git.json -H "Accept: application/vnd.github+json" "https://api.github.com/repos/catalogueoflife/backend/commits/$(git rev-parse HEAD)"
@@ -106,4 +103,4 @@ RUN chown -R $USER:$USER /opt/gbif/$APP_ARTIFACT
 USER $USER
 EXPOSE $SERVER_PORT
 
-CMD java $JVM_OPTIONS -jar app.jar --server.port=$SERVER_PORT --working.dir=/opt/gbif/$APP_ARTIFACT --mode=RUN --spring.cloud.bootstrap.location=/opt/gbif/$APP_ARTIFACT/bootstrap.yaml
+CMD java $JVM_OPTIONS -jar app.jar --server.port=$SERVER_PORT --working.dir=/opt/gbif/$APP_ARTIFACT/ --mode=RUN --spring.cloud.bootstrap.location=/opt/gbif/$APP_ARTIFACT/bootstrap.yaml
diff --git a/matching-ws/pom.xml b/matching-ws/pom.xml
@@ -15,6 +15,7 @@
     <java.version>11</java.version>
     <lucene.version>9.10.0</lucene.version>
     <spring-boot.version>2.7.18</spring-boot.version>
+    <spring-cloud.version>2021.0.9</spring-cloud.version>
     <lombok.version>1.18.22</lombok.version>
     <spotless-maven-plugin.version>2.43.0</spotless-maven-plugin.version>
     <logback.version>1.2.13</logback.version>
@@ -284,11 +285,6 @@
       <artifactId>spring-boot-starter-validation</artifactId>
       <version>${spring-boot.version}</version>
     </dependency>
-    <dependency>
-      <groupId>net.openhft</groupId>
-      <artifactId>chronicle-map</artifactId>
-      <version>3.25ea6</version>
-    </dependency>
     <dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-web</artifactId>
@@ -356,14 +352,6 @@
       <artifactId>logstash-logback-encoder</artifactId>
       <version>${logstash-logback.version}</version>
     </dependency>
-    <!-- enable for dev -->
-<!--    <dependency>-->
-<!--      <groupId>org.springframework.boot</groupId>-->
-<!--      <artifactId>spring-boot-devtools</artifactId>-->
-<!--      <version>${spring-boot.version}</version>-->
-<!--      <scope>runtime</scope>-->
-<!--      <optional>true</optional>-->
-<!--    </dependency>-->
     <dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-configuration-processor</artifactId>

diff --git a/matching-ws/src/main/java/life/catalogue/matching/MatchingApplication.java b/matching-ws/src/main/java/life/catalogue/matching/MatchingApplication.java
@@ -87,7 +87,7 @@ public void run(ApplicationArguments args) {
   }
 
   private void initialiseWebapp() {
-    Optional<APIMetadata> metadata = matchingService.getAPIMetadata(false);
+    Optional<APIMetadata> metadata = matchingService.getAPIMetadata(true);
     if (metadata.isEmpty()) {
       log.error("No main index found. Cannot start web services");
       return;
@@ -136,7 +136,8 @@ private void runIndexingIfRequired(ApplicationArguments args) throws Exception {
         indexingService.indexIdentifiers(id);
     }
 
-    log.info("Indexing completed");
+    matchingService.getAPIMetadata(true);
+    log.info("Indexing ready");
   }
 
   private ExecutionMode getMode(ApplicationArguments args) {

diff --git a/matching-ws/src/main/java/life/catalogue/matching/controller/MatchController.java b/matching-ws/src/main/java/life/catalogue/matching/controller/MatchController.java
@@ -15,6 +15,7 @@
 import java.util.stream.Collectors;
 import life.catalogue.matching.model.*;
 import life.catalogue.matching.service.MatchingService;
+import life.catalogue.matching.util.IUCNUtils;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -112,9 +113,13 @@ public NameUsageMatch matchOldPaths(
     HttpServletRequest response) {
     return matchV2(
       usageKey,
-      taxonID,taxonConceptID,scientificNameID,
-      scientificName2, scientificName,
-      authorship, authorship2,
+      taxonID,
+      taxonConceptID,
+      scientificNameID,
+      scientificName2,
+      scientificName,
+      authorship,
+      authorship2,
       genericName,
       specificEpithet,
       infraspecificEpithet,
@@ -279,6 +284,7 @@ public NameUsageMatch matchV2(
         taxonID,
         taxonConceptID,
         scientificNameID,
+
         scientificName,
         scientificName2,
         authorship,
@@ -288,6 +294,7 @@ public NameUsageMatch matchV2(
         infraspecificEpithet,
         rank,
         rank2,
+
         classification,
         exclude,
         strict,
@@ -428,6 +435,7 @@ public Object matchFlatV1(
         taxonID,
         taxonConceptID,
         scientificNameID,
+
         scientificName,
         scientificName2,
         authorship,
@@ -437,6 +445,7 @@ public Object matchFlatV1(
         infraspecificEpithet,
         rank,
         rank2,
+
         classification,
         exclude != null ? exclude.stream().map(Object::toString).collect(Collectors.toSet()) : Set.of(),
         strict,
@@ -598,6 +607,7 @@ public Object matchV1(
       taxonID,
       taxonConceptID,
       scientificNameID,
+
       scientificName,
       scientificName2,
       authorship,
@@ -607,6 +617,7 @@ public Object matchV1(
       infraspecificEpithet,
       rank,
       rank2,
+
       classification,
       exclude != null ? exclude.stream().map(Object::toString).collect(Collectors.toSet()) : Set.of(),
       strict,
@@ -671,15 +682,15 @@ public Map<String, Object> iucnRedListV1(@PathVariable(value = "usageKey", requi
       return Map.of();
     }
     NameUsageMatch.Status status = statusList.get(0);
-    String formatted = formatIucn(status.getStatus());
+    String formatted = IUCNUtils.formatIucn(status.getStatus());
     if (formatted == null || formatted.isEmpty()) {
       return Map.of();
     }
 
     String scientificName = match.getAcceptedUsage() != null ? match.getAcceptedUsage().getCanonicalName() : match.getUsage().getCanonicalName();
 
     try {
-      IUCN iucn = IUCN.valueOf(formatted); // throws IllegalArgumentException if not found
+      IUCNUtils.IUCN iucn = IUCNUtils.IUCN.valueOf(formatted); // throws IllegalArgumentException if not found
       watch.stop();
       log("v1/species/iucnRedListCategory", usageKey, watch);
       return Map.of(
@@ -689,7 +700,7 @@ public Map<String, Object> iucnRedListV1(@PathVariable(value = "usageKey", requi
         "taxonomicStatus", NameUsageMatchV1.TaxonomicStatusV1.convert(
           match.getDiagnostics().getStatus()),
         "iucnTaxonID", status.getSourceId(),
-        "code", iucn.code
+        "code", iucn.getCode()
       );
     } catch (IllegalArgumentException e) {
       log.error("IUCN category not found: {}", formatted, e);
@@ -751,37 +762,6 @@ private static void addIfNotNull(StringJoiner joiner, Object value) {
     }
   }
 
-  String formatIucn(String original){
-    if (original == null) {
-      return null;
-    }
-    // Trim the string
-    String trimmed = original.trim();
-    // Convert to uppercase
-    String uppercased = trimmed.toUpperCase();
-    // Replace any whitespace with a single underscore
-    return uppercased.replaceAll("\\s+", "_");
-  }
-
-   enum IUCN {
-    EXTINCT("EX"),
-    EXTINCT_IN_THE_WILD("EW"),
-    CRITICALLY_ENDANGERED ("CR"),
-    ENDANGERED ("EN"),
-    VULNERABLE ("VU"),
-    NEAR_THREATENED ("NT"),
-    CONSERVATION_DEPENDENT ("CD"),
-    LEAST_CONCERN ("LC"),
-    DATA_DEFICIENT ("DD"),
-    NOT_EVALUATED ("NE");
-
-    private final String code;
-
-    IUCN(String code) {
-      this.code = code;
-    }
-  }
-
   @Data
   @Builder
   @NoArgsConstructor