Skip to content

Commit

Permalink
Add checks against empty partitions
Browse files Browse the repository at this point in the history
  • Loading branch information
adam-narozniak committed Feb 26, 2024
1 parent 1897cbc commit 5789d79
Showing 1 changed file with 15 additions and 0 deletions.
15 changes: 15 additions & 0 deletions datasets/flwr_datasets/partitioner/shard_partitioner.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,14 @@ def _determine_node_id_to_indices_if_needed(self) -> None: # pylint: disable=R0
math.ceil(len(self.dataset) / self._shard_size)
)
num_usable_shards_in_dataset = self._num_shards_used
if num_usable_shards_in_dataset < self._num_partitions:
raise ValueError(
"Based on the given arguments the creation of the partitions "
"is impossible. The implied number of partitions that can be "
"used is lower than the number of requested partitions "
"resulting in empty partitions. Please decrease the size of "
"shards: `shard_size`."
)
else:
raise ValueError(
"The keep_incomplete_shards need to be specified "
Expand All @@ -251,6 +259,13 @@ def _determine_node_id_to_indices_if_needed(self) -> None: # pylint: disable=R0
"keep_incomplete_shards is not correct."
)

if num_usable_shards_in_dataset < self._num_partitions:
raise ValueError(
"The specified configuration results in empty partitions because the "
"number of usable shards is smaller that the number partitions. "
"Try decreasing the shard size or the number of partitions. "
)

indices_on_which_to_split_shards = np.cumsum(
num_shards_per_node_array, dtype=int
)
Expand Down

0 comments on commit 5789d79

Please sign in to comment.