Support dataset exclusion in nub configs

gbif · Jul 5, 2023 · 558f793 · 558f793
1 parent 1f386a3
commit 558f793
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 0 deletions.
diff --git a/checklistbank-cli/src/main/java/org/gbif/checklistbank/cli/nubbuild/NubConfiguration.java b/checklistbank-cli/src/main/java/org/gbif/checklistbank/cli/nubbuild/NubConfiguration.java
@@ -81,6 +81,13 @@ public boolean isDoubtful(String name) {
     return doubtful.contains(name.trim());
   }
 
+  /**
+   * Set of dataset keys to be excluded from processing that publisher sources like Plazi otherwise provide.
+   */
+  @NotNull
+  @Valid
+  public Set<UUID> datasetExclusions = new HashSet<>();
+
   /**
    * List of higher wrong homonyms that should be removed, regardless of which source they came from.
    * Map of a canonical name to its direct parent.

diff --git a/checklistbank-cli/src/main/java/org/gbif/checklistbank/nub/source/ClbSourceList.java b/checklistbank-cli/src/main/java/org/gbif/checklistbank/nub/source/ClbSourceList.java
@@ -133,6 +133,10 @@ private List<NubSource> loadSources() {
         LOG.warn("Duplicate source {} skipped", sd.key);
         continue;
       }
+      if (cfg.datasetExclusions.contains(sd.key)) {
+        LOG.warn("Source {} excluded in configs", sd.key);
+        continue;
+      }
       keys.add(sd.key);
       Dataset d = datasetService.get(sd.key);
       if (d != null) {
@@ -144,6 +148,10 @@ private List<NubSource> loadSources() {
         if (org != null) {
           int counter = 0;
           for (Dataset d2 : Iterables.publishedDatasets(org.getKey(), DatasetType.CHECKLIST, organizationService)) {
+            if (cfg.datasetExclusions.contains(d2)) {
+              LOG.warn("Source {} from publisher {} excluded in configs", d2, org.getTitle());
+              continue;
+            }
             if (!keys.contains(d2.getKey())) {
               sources.add(buildSource(d2, cfg, sd));
               counter++;
@@ -156,6 +164,10 @@ private List<NubSource> loadSources() {
           if (inst != null) {
             int counter = 0;
             for (Dataset d2 : Iterables.hostedDatasets(inst.getKey(), DatasetType.CHECKLIST, installationService)) {
+              if (cfg.datasetExclusions.contains(d2)) {
+                LOG.warn("Source {} from installation {} excluded in configs", d2, inst.getKey());
+                continue;
+              }
               if (!keys.contains(d2.getKey())) {
                 sources.add(buildSource(d2, cfg, sd));
                 counter++;