Z01_LLaMA-Factory_cmd.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fe8650ad-9a9b-4377-b138-7cc169109d64",
   "metadata": {},
   "source": [
    "# LLaMA-Factory (CMD)\n",
    "https://github.com/hiyouga/LLaMA-Factory"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4773e2c0-ce68-4c80-82a2-c632479c1eaf",
   "metadata": {},
   "source": [
    "## 初始環境設定"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b113b8a-c6ff-43f8-a097-30d7d48629fd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "HOME = str(Path.home())\n",
    "Add_Binarry_Path=HOME+'/.local/bin'\n",
    "os.environ['PATH']=os.environ['PATH']+':'+Add_Binarry_Path\n",
    "current_foldr=!pwd\n",
    "current_foldr=current_foldr[0]\n",
    "current_foldr"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf2c4620-b2a9-40fd-8b1d-01bda0b65691",
   "metadata": {},
   "source": [
    "## 套件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d8a4bb1-bed5-4487-a90a-9806fd23f91e",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install cohere gdown kaleido langchain openai pyngrok pypdf python-dotenv sentence-transformers tiktoken -q\n",
    "!pip install accelerate bitsandbytes hf_transfer huggingface_hub optimum transformers==4.36.2 -q \n",
    "!pip install appdirs black black[jupyter] datasets fire loralib sentencepiece gradio==3.48.0 -q\n",
    "!pip install fastapi jieba matplotlib nltk peft==0.7.0 protobuf pydantic rouge-chinese scipy sse-starlette trl==0.7.6 uvicorn -q \n",
    "!pip install deepspeed -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dda25d8f-63b5-45ca-b6fa-78b1f2a48091",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# HF_TOKEN\n",
    "import os\n",
    "os.environ[\"HF_TOKEN\"] = \"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\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4416747b-f7fb-48b8-9042-6ac5e874de9c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Download dataset\n",
    "from huggingface_hub import hf_hub_download\n",
    "import pandas as pd\n",
    "REPO_ID = \"dikw/hh_rlhf_cn\"\n",
    "FILENAME = \"harmless_base_cn_train.jsonl\"\n",
    "downloaded_model_path=hf_hub_download(repo_id=REPO_ID, filename=FILENAME, repo_type=\"dataset\")\n",
    "print(downloaded_model_path)'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e33883e-9030-4f17-9f88-fd9274d611d9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 所有資料內容\n",
    "import pandas as pd \n",
    "file_path='./harmless_base_cn_train.jsonl'\n",
    "df = pd.read_json(path_or_buf=file_path, lineas=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71ad6a36-0409-498e-8126-0f6b05c5b2c1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!wandb offline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7993f8df-9768-4aa9-b444-4b23a0e4cbd4",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Continue Pretraining (CP)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6696b36-fd9f-46ed-a663-e74424c66458",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "#METHOD 01 Python\n",
    "rm -rf  path_to_pt_checkpoint\n",
    "MODEL_ID=\"/work/g00cjz00/github/LLaMA-Factory/Llama-2-7b-hf\"\n",
    "CUDA_VISIBLE_DEVICES=0 python src/train_bash.py \\\n",
    "    --stage pt \\\n",
    "    --do_train \\\n",
    "    --model_name_or_path ${MODEL_ID} \\\n",
    "    --dataset wiki_demo \\\n",
    "    --finetuning_type lora \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir path_to_pt_checkpoint \\\n",
    "    --overwrite_cache \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --logging_steps 10 \\\n",
    "    --save_steps 1000 \\\n",
    "    --learning_rate 5e-5 \\\n",
    "    --num_train_epochs 1.0 \\\n",
    "    --plot_loss \\\n",
    "    --fp16 \\\n",
    "    --max_samples 500 \\\n",
    "    --plot_loss True "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fef30d03-2521-419e-a820-558fcdc8815d",
   "metadata": {},
   "source": [
    "## Supervised Fine-Tuning (SFT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b28ff3fe-321c-4d52-89e6-d1a7580c4a69",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "#METHOD 01 Python\n",
    "rm -rf path_to_sft_checkpoint\n",
    "MODEL_ID=\"/work/u00cjz00/slurm_jobs/github/models/CKIP-Llama-2-7b-chat\"\n",
    "CUDA_VISIBLE_DEVICES=0 python src/train_bash.py \\\n",
    "    --stage sft \\\n",
    "    --do_train \\\n",
    "    --model_name_or_path ${MODEL_ID} \\\n",
    "    --dataset alpaca_gpt4_en \\\n",
    "    --template default \\\n",
    "    --finetuning_type lora \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir path_to_sft_checkpoint \\\n",
    "    --overwrite_cache \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --logging_steps 10 \\\n",
    "    --save_steps 1000 \\\n",
    "    --learning_rate 5e-5 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --fp16 \\\n",
    "    --max_samples 500 \\\n",
    "    --plot_loss True "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5452c2d5-6fe6-4b32-9a00-2ecc9f41b57d",
   "metadata": {},
   "source": [
    "## Direct Preference Optimization (DPO)\n",
    "LLM的直接偏好優化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3238e06-a8da-468b-8712-a3b090112162",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "#METHOD 01\n",
    "rm -rf path_to_dpo_checkpoint\n",
    "MODEL_ID=\"/work/g00cjz00/github/LLaMA-Factory/Llama-2-7b-hf\"\n",
    "CUDA_VISIBLE_DEVICES=0 python src/train_bash.py \\\n",
    "    --stage dpo \\\n",
    "    --do_train \\\n",
    "    --model_name_or_path ${MODEL_ID} \\\n",
    "    --adapter_name_or_path path_to_sft_checkpoint \\\n",
    "    --create_new_adapter \\\n",
    "    --dataset comparison_gpt4_en \\\n",
    "    --template default \\\n",
    "    --finetuning_type lora \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir path_to_dpo_checkpoint \\\n",
    "    --per_device_train_batch_size 2 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --logging_steps 10 \\\n",
    "    --save_steps 1000 \\\n",
    "    --learning_rate 1e-5 \\\n",
    "    --num_train_epochs 1.0 \\\n",
    "    --fp16 \\\n",
    "    --max_samples 500 \\\n",
    "    --plot_loss True "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5973cdc3-097b-4d0f-a01e-3560dbfde107",
   "metadata": {},
   "source": [
    "## Reward Modeling (RW), RLHF \n",
    "- 反映人类对生成回答偏好"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8d32efd-9925-4f04-b7f5-a756fc4641d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "#METHOD 01\n",
    "rm -rf path_to_rm_checkpoint\n",
    "\n",
    "MODEL_ID=\"/work/g00cjz00/github/LLaMA-Factory/Llama-2-7b-hf\"\n",
    "\n",
    "CUDA_VISIBLE_DEVICES=0 python src/train_bash.py \\\n",
    "    --stage rm \\\n",
    "    --do_train \\\n",
    "    --model_name_or_path ${MODEL_ID} \\\n",
    "    --adapter_name_or_path path_to_sft_checkpoint \\\n",
    "    --create_new_adapter \\\n",
    "    --dataset comparison_gpt4_en \\\n",
    "    --template default \\\n",
    "    --finetuning_type lora \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir path_to_rm_checkpoint \\\n",
    "    --per_device_train_batch_size 2 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --logging_steps 10 \\\n",
    "    --save_steps 1000 \\\n",
    "    --learning_rate 1e-6 \\\n",
    "    --num_train_epochs 1.0 \\\n",
    "    --plot_loss \\\n",
    "    --fp16"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3170e12-be06-4e8b-9a89-5e135513ed9b",
   "metadata": {},
   "source": [
    "## PPO Training (PPO), RLHF\n",
    "- 強化學習（PPO）來最大化預測的報酬（reward）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4462fd4-f02a-4981-9d5d-50f282105859",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "#METHOD 01\n",
    "rm -rf path_to_ppo_checkpoint\n",
    "\n",
    "MODEL_ID=\"/work/g00cjz00/github/LLaMA-Factory/Llama-2-7b-hf\"\n",
    "\n",
    "CUDA_VISIBLE_DEVICES=0 python src/train_bash.py \\\n",
    "    --stage ppo \\\n",
    "    --do_train \\\n",
    "    --model_name_or_path ${MODEL_ID} \\\n",
    "    --adapter_name_or_path path_to_sft_checkpoint \\\n",
    "    --create_new_adapter \\\n",
    "    --dataset alpaca_gpt4_en \\\n",
    "    --template default \\\n",
    "    --finetuning_type lora \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --reward_model path_to_rm_checkpoint \\\n",
    "    --output_dir path_to_ppo_checkpoint \\\n",
    "    --per_device_train_batch_size 2 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --top_k 0 \\\n",
    "    --top_p 0.9 \\\n",
    "    --logging_steps 10 \\\n",
    "    --save_steps 1000 \\\n",
    "    --learning_rate 1e-5 \\\n",
    "    --num_train_epochs 1.0 \\\n",
    "    --plot_loss \\\n",
    "    --fp16"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca2094c8-ce63-4357-9611-84f9bf6e35e6",
   "metadata": {},
   "source": [
    "## Merge LoRA weights and export model\n",
    "- Merging LoRA weights into a quantized model is not supported.\n",
    "- Use --export_quantization_bit 4 and --export_quantization_dataset data/c4_demo.json to quantize the model after merging the LoRA weights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b28c9e6a-a7fd-4a72-a7d3-06eb156229d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "#METHOD 02\n",
    "rm -rf path_to_export\n",
    "\n",
    "MODEL_ID=\"/work/g00cjz00/github/LLaMA-Factory/Llama-2-7b-hf\"\n",
    "\n",
    "python src/export_model.py \\\n",
    "    --model_name_or_path ${MODEL_ID} \\\n",
    "    --adapter_name_or_path path_to_checkpoint \\\n",
    "    --template default \\\n",
    "    --finetuning_type lora \\\n",
    "    --export_dir path_to_export \\\n",
    "    --export_size 2 \\\n",
    "    --export_legacy_format False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "094ff585-ca23-4a97-987a-964b4d2c0250",
   "metadata": {},
   "source": [
    "## API\n",
    "- Visit http://localhost:8000/docs for API documentation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6c3d2b7-fa8b-4e40-94e4-19aaf4be0827",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "MODEL_ID=\"/work/g00cjz00/github/LLaMA-Factory/Llama-2-7b-hf\"\n",
    "\n",
    "python src/api_demo.py \\\n",
    "    --model_name_or_path ${MODEL_ID} \\\n",
    "    --adapter_name_or_path path_to_sft_checkpoint \\\n",
    "    --template default \\\n",
    "    --finetuning_type lora"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "379e7924-af86-4a0c-b479-3e3d2d3aeb69",
   "metadata": {},
   "source": [
    "## CLI Demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ead662d4-488d-4d4a-a4d8-737d22664030",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "MODEL_ID=\"/work/g00cjz00/github/LLaMA-Factory/Llama-2-7b-hf\"\n",
    "\n",
    "python src/web_demo.py \\\n",
    "    --model_name_or_path ${MODEL_ID} \\\n",
    "    --adapter_name_or_path path_to_sft_checkpoint \\\n",
    "    --template default \\\n",
    "    --finetuning_type lora"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f52dd50d-e491-43f7-9b2e-9481db5cac8f",
   "metadata": {},
   "source": [
    "## Web Demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60d6e0a4-1a24-4c3b-bd43-e080f0798e2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "MODEL_ID=\"/work/g00cjz00/github/LLaMA-Factory/Llama-2-7b-hf\"\n",
    "\n",
    "python src/web_demo.py \\\n",
    "    --model_name_or_path ${MODEL_ID} \\\n",
    "    --adapter_name_or_path path_to_checkpoint \\\n",
    "    --template default \\\n",
    "    --finetuning_type lora"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3afad47e-013f-4e83-99a3-9ced2aa81161",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab65040d-b4fa-42c0-9afd-56d4b5ce523b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "MODEL_ID=\"/work/g00cjz00/github/LLaMA-Factory/Llama-2-7b-hf\"\n",
    "\n",
    "CUDA_VISIBLE_DEVICES=0 python src/evaluate.py \\\n",
    "    --model_name_or_path ${MODEL_ID} \\\n",
    "    --adapter_name_or_path path_to_sft_checkpoint \\\n",
    "    --template vanilla \\\n",
    "    --finetuning_type lora\n",
    "    --task mmlu \\\n",
    "    --split test \\\n",
    "    --lang en \\\n",
    "    --n_shot 5 \\\n",
    "    --batch_size 4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cda6f93-6633-4dd3-ac4f-58f8ea59ecc8",
   "metadata": {},
   "source": [
    "## Predict\n",
    "- Use --per_device_train_batch_size=1 for LLaMA-2 models in fp16 predict.\n",
    "- We recommend using --per_device_eval_batch_size=1 and --max_target_length 128 at 4/8-bit predict."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c216daa2-e71e-4b36-9ff9-83a669dac009",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "rm -rf path_to_predict_result\n",
    "MODEL_ID=\"/work/u00cjz00/slurm_jobs/github/models/CKIP-Llama-2-7b-chat\"\n",
    "\n",
    "CUDA_VISIBLE_DEVICES=1 python src/train_bash.py \\\n",
    "    --stage sft \\\n",
    "    --do_predict \\\n",
    "    --model_name_or_path ${MODEL_ID} \\\n",
    "    --adapter_name_or_path path_to_sft_checkpoint \\\n",
    "    --dataset alpaca_gpt4_en \\\n",
    "    --template default \\\n",
    "    --finetuning_type lora \\\n",
    "    --output_dir path_to_predict_result \\\n",
    "    --per_device_eval_batch_size 8 \\\n",
    "    --max_samples 100 \\\n",
    "    --predict_with_generate \\\n",
    "    --fp16 \\\n",
    "    --max_samples 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8ce9f5f-f35e-4266-94e5-e11a1643aa82",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa792446-6e90-4df6-9b53-a07f508b0888",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "#METHOD 04 ds_config.json\n",
    "cat << \\EOF >  ds_config.json\n",
    "{\n",
    "  \"train_batch_size\": \"auto\",\n",
    "  \"train_micro_batch_size_per_gpu\": \"auto\",\n",
    "  \"gradient_accumulation_steps\": \"auto\",\n",
    "  \"gradient_clipping\": \"auto\",\n",
    "  \"zero_allow_untested_optimizer\": true,\n",
    "  \"fp16\": {\n",
    "    \"enabled\": \"auto\",\n",
    "    \"loss_scale\": 0,\n",
    "    \"initial_scale_power\": 16,\n",
    "    \"loss_scale_window\": 1000,\n",
    "    \"hysteresis\": 2,\n",
    "    \"min_loss_scale\": 1\n",
    "  },  \n",
    "  \"zero_optimization\": {\n",
    "    \"stage\": 2,\n",
    "    \"allgather_partitions\": true,\n",
    "    \"allgather_bucket_size\": 5e8,\n",
    "    \"reduce_scatter\": true,\n",
    "    \"reduce_bucket_size\": 5e8,\n",
    "    \"overlap_comm\": false,\n",
    "    \"contiguous_gradients\": true\n",
    "  }\n",
    "}\n",
    "EOF\n",
    "\n",
    "#METHOD 04 Deepspeed\n",
    "rm -rf path_to_predict_result\n",
    "MODEL_ID=\"/work/u00cjz00/slurm_jobs/github/models/CKIP-Llama-2-7b-chat\"\n",
    "export PATH=$PATH:$HOME/.local/bin;\n",
    "deepspeed --num_gpus 2 --master_port=9901 src/train_bash.py \\\n",
    "    --deepspeed ds_config.json \\\n",
    "    --stage sft \\\n",
    "    --do_predict \\\n",
    "    --model_name_or_path ${MODEL_ID} \\\n",
    "    --adapter_name_or_path path_to_sft_checkpoint \\\n",
    "    --dataset alpaca_gpt4_en \\\n",
    "    --template default \\\n",
    "    --finetuning_type lora \\\n",
    "    --output_dir path_to_predict_result \\\n",
    "    --per_device_eval_batch_size 8 \\\n",
    "    --max_samples 100 \\\n",
    "    --predict_with_generate \\\n",
    "    --fp16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a10bb55-f74b-4cf6-b591-1f564b495a83",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "487d98b5-2f7f-4808-8696-069b5a896991",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!srun --mpi=pmi2 echo ${SLURM_PROCID}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6baac721-499d-4a5b-8921-41a4beeda1a6",
   "metadata": {},
   "source": [
    "## JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ef9523b-1546-456e-9787-9edb0de00d92",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 所有資料內容\n",
    "import pandas as pd \n",
    "file_path='./path_to_predict_result/generated_predictions.jsonl'\n",
    "df = pd.read_json(path_or_buf=file_path, lineas=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e414f47-b1a5-4131-b5dc-c09a4362201c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 所有資料內容\n",
    "import pandas as pd \n",
    "file_path='./path_to_predict_result/all_results.json'\n",
    "df = pd.read_json(path_or_buf=file_path)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4dd7eca8-a38c-450c-ac3a-a20d950486bf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Image_S_work-genai11_pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv",
   "language": "python",
   "name": "s_work-genai11_pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}