handle dataset not existing anymore

NINAnor · Jul 26, 2024 · 41d0dd6 · 41d0dd6
1 parent bb5b420
commit 41d0dd6
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 18 deletions.
diff --git a/metadata_catalogue/datasets/libs/harvesters.py b/metadata_catalogue/datasets/libs/harvesters.py
@@ -35,24 +35,36 @@ def harvest_dataset(dataset_id: int):
         if dataset.fetch_url:
             file = NamedTemporaryFile(delete=True)
             # TODO: handle network errors
-            stream = requests.get(dataset.fetch_url, stream=True)
-            for block in stream.iter_content(1024 * 8):
-                if not block:
-                    break
-                file.write(block)
+            try:
+                stream = requests.get(dataset.fetch_url, stream=True)
+                for block in stream.iter_content(1024 * 8):
+                    if not block:
+                        break
+                    file.write(block)
 
-            file.flush()
+                file.flush()
+
+                if dataset.fetch_type == Dataset.FetchType.DARWINCORE:
+                    try:
+                        handle_file_as_darwincore_zip(file, dataset)
+                        dataset.set_fetch_message("", append=True, success=True)
+                    except:
+                        dataset.set_fetch_message(
+                            traceback.format_exc(), append=True, logger_fn=logger.error, success=False
+                        )
+                else:
+                    dataset.set_fetch_message("Not implemented fetch type", logger_fn=logger.warn, success=False)
+            except requests.RequestException:
+                dataset.set_fetch_message(
+                    "Unable to fetch the dataset, will be hidden",
+                    append=True,
+                    logger_fn=logger.error,
+                    success=False,
+                    commit=False,
+                )
+                dataset.public = False
+                dataset.save()
 
-            if dataset.fetch_type == Dataset.FetchType.DARWINCORE:
-                try:
-                    handle_file_as_darwincore_zip(file, dataset)
-                    dataset.set_fetch_message("", append=True, success=True)
-                except:
-                    dataset.set_fetch_message(
-                        traceback.format_exc(), append=True, logger_fn=logger.error, success=False
-                    )
-            else:
-                dataset.set_fetch_message("Not implemented fetch type", logger_fn=logger.warn, success=False)
         else:
             dataset.set_fetch_message(
                 f"No fetch URL for dataset {dataset_id}, ignoring", logger_fn=logger.warn, success=False

diff --git a/metadata_catalogue/datasets/models.py b/metadata_catalogue/datasets/models.py
@@ -52,7 +52,7 @@ class FetchType(models.IntegerChoices):
     def __str__(self):
         return f"{self.id} - {self.name}"
 
-    def set_fetch_message(self, message, *args, append=False, success=None, logger_fn=None):
+    def set_fetch_message(self, message, *args, append=False, success=None, logger_fn=None, commit=True):
         text = ""
         if append and self.fetch_message:
             text = self.fetch_message
@@ -66,7 +66,8 @@ def set_fetch_message(self, message, *args, append=False, success=None, logger_f
         if logger_fn:
             logger_fn(message)
 
-        self.save()
+        if commit:
+            self.save()
 
     @hook(AFTER_CREATE)
     def create_metadata_content(self):