dpo nb lightly cleaned

Author: nelson-lojo

dpo.ipynb

@@ -223,10 +223,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Define template: `response_template_ids`, `collator`, `format_prompt`\n",
-    "response_template_ids = tokenizer.encode(response_template, add_special_tokens=False)[1:]\n",
-    "collator = DataCollatorForCompletionOnlyLM(response_template_ids, tokenizer=tokenizer)\n",
-    "\n",
     "def format_prompt(example) -> Tuple[str, str]:\n",
     "    return f\"{example['context']} \\n-- Question: {example['question']}{response_template}\", example['answer']"
    ]
@@ -323,7 +319,7 @@
     "        return out\n",
     "\n",
     "    dataset.set_transform(batched_mutate)\n",
-    "    return dataset # dataset.map(batched_mutate, batched=True, num_proc=num_proc, remove_columns=original_columns)"
+    "    return dataset"
    ]
   },
   {
@@ -427,18 +423,15 @@
     "    model_ref, # The model with peft adapters turned off will be used as a reference model if not provided\n",
     "    tokenizer=tokenizer,\n",
     "    train_dataset=ds,\n",
-    "    # eval_dataset=eval_dataset,\n",
-    "    beta=0.2, # TODO: determine\n",
+    "    beta=0.2, \n",
     "    max_length=2048,\n",
-    "    # max_target_length=248,\n",
     "    max_prompt_length=1500,\n",
     "    args=TrainingArguments(\n",
     "        output_dir=\"./dpo_results\",\n",
     "        optim=\"paged_adamw_32bit\",\n",
     "\n",
     "        max_grad_norm=0.3,\n",
     "        warmup_ratio=0.03,\n",
-    "        # group_by_length=True,\n",
     "        \n",
     "        learning_rate=2e-4,\n",
     "        weight_decay=0.001,\n",
@@ -533,7 +526,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Fine-tuned model\n",
+    "# Save trained model\n",
     "new_model = \"open_llama_3b_v2_sft_plus_dpo\"\n",
     "trainer.model.save_pretrained(new_model)"
    ]