More corrections

arcee-ai · Nov 23, 2023 · 55f03ef · 55f03ef
1 parent 14f65c5
commit 55f03ef
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/dalm/pipelines/reading_comprehension_pipeline.py b/dalm/pipelines/reading_comprehension_pipeline.py
@@ -165,7 +165,8 @@ def pipeline(
 
     dataset = datasets.Dataset.from_list(in_memory_dataset)
 
-    dataset.save_to_disk(output_dataset_name)
+    if not os.path.exists(output_dataset_name):
+        dataset.save_to_disk(output_dataset_name)
 
     train_generator(
         model_name=model_name,

diff --git a/dalm/training/generator_only/trainer.py b/dalm/training/generator_only/trainer.py
@@ -181,7 +181,6 @@ def train_generator(
         quantization_config=bnb_config,
         device_map={"": Accelerator().local_process_index},
         trust_remote_code=True,
-        use_auth_token=True,
     )
 
     base_model.config.use_cache = False