Early return for num_shards==0 in the Beam pipeline. (#778)

mlcommons · Dec 3, 2024 · 5fbea37 · 5fbea37
1 parent f533cee
commit 5fbea37
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/python/mlcroissant/mlcroissant/_src/operation_graph/execute.py b/python/mlcroissant/mlcroissant/_src/operation_graph/execute.py
@@ -195,6 +195,10 @@ def execute_operations_in_beam(
         enumerate(files)
     )
     num_shards = len(files)
+    if not num_shards:
+        raise ValueError(
+            f"Empty {record_set=}. No files found for filters={json.dumps(filters)}"
+        )
 
     # We don't know in advance the number of records per shards. So we just allocate the
     # maximum number which is `sys.maxsize // num_shards`. Taking the practical case of