Skip to content

Commit

Permalink
Merge branch 'add-fds-partial-download-for-tests' into add-tests-for-…
Browse files Browse the repository at this point in the history
…ucf101-dataset
  • Loading branch information
adam-narozniak committed Jul 20, 2024
2 parents 0e6891c + e81680d commit 02e085f
Showing 1 changed file with 16 additions and 1 deletion.
17 changes: 16 additions & 1 deletion datasets/flwr_datasets/mock_utils_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,7 @@ def _load_mocked_dataset(
return datasets.DatasetDict(dataset_dict)


def _download_partial_dataset(
def _load_mocked_dataset_by_partial_download(
dataset_name: str,
split_name: str,
skip_take_list: List[Tuple[int, int]],
Expand Down Expand Up @@ -439,3 +439,18 @@ def _download_partial_dataset(
# to the fully downloaded dataset index: dataset[n+1: (n+1 + m)]
dataset_list.extend(list(dataset.skip(skip).take(take)))
return Dataset.from_list(dataset_list)


def _load_mocked_dataset_dict_by_partial_download(
dataset_name: str,
split_names: List[str],
skip_take_lists: List[List[Tuple[int, int]]],
subset_name: Optional[str] = None,
) -> DatasetDict:
"""Like _load_mocked_dataset_by_partial_download but for many splits."""
dataset_dict = {}
for split_name, skip_take_list in zip(split_names, skip_take_lists):
dataset_dict[split_name] = _load_mocked_dataset_by_partial_download(
dataset_name, split_name, skip_take_list, subset_name
)
return DatasetDict(dataset_dict)

0 comments on commit 02e085f

Please sign in to comment.