Specify predownload for streaming dataset

mosaicml · Jul 24, 2024 · 8738b18 · 8738b18
1 parent 300c51b
commit 8738b18
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/scripts/t5_precompute.py b/scripts/t5_precompute.py
@@ -64,6 +64,7 @@
     dataset = StreamingDataset(remote=remote_src,
                                local=os.path.join('/tmp', subdir_path),
                                download_timeout=300,
+                               predownload=64,
                                shuffle=False)
 
     # Get columns