When reading multiple files, warn instead of raising on corrupted files.

tests/test_routines.py

@@ -1,6 +1,7 @@
 import numpy as np
 import pytest
 
+import xdas as xd
 from xdas.core.coordinates import Coordinates
 from xdas.core.dataarray import DataArray
 from xdas.core.routines import Bag, CompatibilityError, combine_by_coords
@@ -193,3 +194,23 @@ def test_expand_scalar_coordinate(self):
         assert dc.shape == (2, 10)
         assert dc.dims == ("space", "time")
         assert dc.coords["space"].values.tolist() == [0, 1]
+
+
+class TestOpenMFDataArray:
+    def test_warn_on_corrupted_files(self, tmp_path):
+        expected = DataArray(
+            np.random.rand(10, 5),
+            coords={
+                "time": np.arange(10),
+                "space": np.arange(5),
+            },  # TODO: should work without coords
+        )
+        for index, chunk in enumerate(xd.split(expected, 3, "time"), start=1):
+            chunk.to_netcdf(tmp_path / f"chunk_{index}.nc")
+        result = xd.open_mfdataarray(str(tmp_path / "*.nc"))  # TODO: should accept Path
+        assert result.equals(expected)
+        with (tmp_path / "corrupted.nc").open("wb") as f:
+            f.write(b"corrupted")
+        with pytest.warns(RuntimeWarning):
+            result = xd.open_mfdataarray(str(tmp_path / "*.nc"))
+        assert result.equals(expected)

xdas/core/routines.py

@@ -1,5 +1,6 @@
 import os
 import re
+import warnings
 from collections import defaultdict
 from concurrent.futures import ProcessPoolExecutor, as_completed
 from glob import glob
@@ -323,19 +324,27 @@ def open_mfdataarray(
         )
     max_workers = 1 if engine == "miniseed" else None  # TODO: dirty fix
     with ProcessPoolExecutor(max_workers=max_workers) as executor:
-        futures = [
-            executor.submit(open_dataarray, path, engine=engine, **kwargs)
+        futures_to_paths = {
+            executor.submit(open_dataarray, path, engine=engine, **kwargs): path
             for path in paths
-        ]
+        }
         if verbose:
             iterator = tqdm(
-                as_completed(futures),
-                total=len(futures),
+                as_completed(futures_to_paths),
+                total=len(futures_to_paths),
                 desc="Fetching metadata from files",
             )
         else:
-            iterator = as_completed(futures)
-        objs = [future.result() for future in iterator]
+            iterator = as_completed(futures_to_paths)
+        objs = []
+        for future in iterator:
+            try:
+                obj = future.result()
+            except Exception as e:
+                path = futures_to_paths[future]
+                warnings.warn(f"could not open {path}: {e}", RuntimeWarning)
+            else:
+                objs.append(obj)
     return combine_by_coords(objs, dim, tolerance, squeeze, None, verbose)