preference-tune-llm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mnoahpunintended\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.17.8"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/teamspace/studios/this_studio/wandb/run-20240905_115344-1c7v3d0g</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/noahpunintended/orpo-ft/runs/1c7v3d0g' target=\"_blank\">swept-shadow-2</a></strong> to <a href='https://wandb.ai/noahpunintended/orpo-ft' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/noahpunintended/orpo-ft' target=\"_blank\">https://wandb.ai/noahpunintended/orpo-ft</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/noahpunintended/orpo-ft/runs/1c7v3d0g' target=\"_blank\">https://wandb.ai/noahpunintended/orpo-ft/runs/1c7v3d0g</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gc\n",
    "import os\n",
    "\n",
    "import torch\n",
    "import wandb\n",
    "from datasets import load_dataset\n",
    "from peft import LoraConfig, PeftModel, prepare_model_for_kbit_training\n",
    "from transformers import (\n",
    "    AutoModelForCausalLM,\n",
    "    AutoTokenizer,\n",
    "    BitsAndBytesConfig,\n",
    "    TrainingArguments,\n",
    "    pipeline,\n",
    ")\n",
    "from trl import ORPOConfig, ORPOTrainer, setup_chat_format\n",
    "\n",
    "wandb.init(project=\"orpo-ft\")\n",
    "\n",
    "attn_implementation = \"eager\"\n",
    "torch_dtype = torch.float16\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model = \"HuggingFaceTB/SmolLM-360M-instruct\"\n",
    "# Load tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(base_model)\n",
    "\n",
    "\n",
    "base_model = \"HuggingFaceTB/SmolLM-360M\"\n",
    "\n",
    "# Load model\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    base_model,\n",
    "    device_map=\"auto\",\n",
    "    attn_implementation=attn_implementation\n",
    ")\n",
    "model, tokenizer = setup_chat_format(model, tokenizer)\n",
    "\n",
    "tokenizer.model_max_length = 2048\n",
    "tokenizer.pad_token = \"<|endoftext|>\" \n",
    "tokenizer.pad_token_id = tokenizer.convert_tokens_to_ids(tokenizer.pad_token )\n",
    "tokenizer.padding_side = 'right'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = \"HuggingFaceH4/orca_dpo_pairs\"\n",
    "dataset = load_dataset(dataset_name, split=\"all\")\n",
    "dataset = dataset.shuffle(seed=42)#.select(range(100))\n",
    "\n",
    "def format_chat_template(row):\n",
    "    row[\"chosen\"] = tokenizer.apply_chat_template(row[\"chosen\"], tokenize=False)\n",
    "    row[\"rejected\"] = tokenizer.apply_chat_template(row[\"rejected\"], tokenize=False)\n",
    "    return row\n",
    "\n",
    "dataset = dataset.map(\n",
    "    format_chat_template,\n",
    "    num_proc= os.cpu_count(),\n",
    ")\n",
    "dataset = dataset.train_test_split(test_size=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "system\n",
      "You are Pi-Card, the Raspberry Pi voice assistant.\n",
      "user\n",
      "Square root of 100?\n",
      "assistant\n",
      "\n",
      "## 1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.\n"
     ]
    }
   ],
   "source": [
    "# test an example model forward pass\n",
    "model.eval();\n",
    "\n",
    "#example_prompts = [\"What does your name stand for?\" , \"Tell me some fun facts about space\", \"Who are you?\", \"What can you do for me?\", \"Who won the US presidential election?\"]\n",
    "example_prompts = [\"Square root of 100?\"]\n",
    "for prompt in example_prompts:\n",
    "  chat_prompt = f\"\"\"<|im_start|>system\\nYou are Pi-Card, the Raspberry Pi voice assistant.<|im_end|>\\n<|im_start|>user\\n{prompt}<|im_end|>\\n<|im_start|>assistant\\n\"\"\"\n",
    "#    chat_prompt = f\"\"\"<|im_start|>user\\n{prompt}<|im_end|>\\n<|im_start|>assistant\\n\"\"\"\n",
    "\n",
    "  input_ids = tokenizer.encode(chat_prompt, return_tensors='pt')\n",
    "  # send to cuda\n",
    "  input_ids = input_ids.to(model.device)\n",
    "\n",
    "  output = model.generate(input_ids, max_length=1000, pad_token_id=tokenizer.eos_token_id, do_sample=False)\n",
    "\n",
    "  output_text = tokenizer.decode(output[0], skip_special_tokens=True) \n",
    "  print(output_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/transformers/training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n",
      "/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/trl/trainer/orpo_trainer.py:255: UserWarning: When using DPODataCollatorWithPadding, you should set `remove_unused_columns=False` in your TrainingArguments we have set it for you, but you should do it yourself in the future.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5c96ae5d1a14d088f4e5af085624d80",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/12730 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (2453 > 2048). Running this sequence through the model will result in indexing errors\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bdcb7c540d4543728ad3af6cb57379b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/129 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.\n",
      "Could not estimate the number of tokens of the input, floating-point operations will not be computed\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='4773' max='4773' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [4773/4773 2:18:08, Epoch 2/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>Samples Per Second</th>\n",
       "      <th>Steps Per Second</th>\n",
       "      <th>Rewards/chosen</th>\n",
       "      <th>Rewards/rejected</th>\n",
       "      <th>Rewards/accuracies</th>\n",
       "      <th>Rewards/margins</th>\n",
       "      <th>Logps/rejected</th>\n",
       "      <th>Logps/chosen</th>\n",
       "      <th>Logits/rejected</th>\n",
       "      <th>Logits/chosen</th>\n",
       "      <th>Nll Loss</th>\n",
       "      <th>Log Odds Ratio</th>\n",
       "      <th>Log Odds Chosen</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>955</td>\n",
       "      <td>1.220100</td>\n",
       "      <td>1.303726</td>\n",
       "      <td>10.908000</td>\n",
       "      <td>11.826000</td>\n",
       "      <td>5.959000</td>\n",
       "      <td>-0.064073</td>\n",
       "      <td>-1.354176</td>\n",
       "      <td>0.884615</td>\n",
       "      <td>1.290103</td>\n",
       "      <td>-13.541759</td>\n",
       "      <td>-0.640728</td>\n",
       "      <td>-3.047161</td>\n",
       "      <td>-1.000978</td>\n",
       "      <td>1.294628</td>\n",
       "      <td>-0.117854</td>\n",
       "      <td>13.883141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1910</td>\n",
       "      <td>1.031600</td>\n",
       "      <td>1.278734</td>\n",
       "      <td>10.862400</td>\n",
       "      <td>11.876000</td>\n",
       "      <td>5.984000</td>\n",
       "      <td>-0.063308</td>\n",
       "      <td>-1.568314</td>\n",
       "      <td>0.830769</td>\n",
       "      <td>1.505006</td>\n",
       "      <td>-15.683137</td>\n",
       "      <td>-0.633084</td>\n",
       "      <td>-3.196254</td>\n",
       "      <td>-1.271801</td>\n",
       "      <td>1.269806</td>\n",
       "      <td>-0.117618</td>\n",
       "      <td>16.066620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2865</td>\n",
       "      <td>0.945500</td>\n",
       "      <td>1.265010</td>\n",
       "      <td>10.995200</td>\n",
       "      <td>11.732000</td>\n",
       "      <td>5.912000</td>\n",
       "      <td>-0.062469</td>\n",
       "      <td>-1.654430</td>\n",
       "      <td>0.846154</td>\n",
       "      <td>1.591961</td>\n",
       "      <td>-16.544298</td>\n",
       "      <td>-0.624688</td>\n",
       "      <td>-3.210003</td>\n",
       "      <td>-1.328838</td>\n",
       "      <td>1.256219</td>\n",
       "      <td>-0.117552</td>\n",
       "      <td>16.950958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3820</td>\n",
       "      <td>1.002100</td>\n",
       "      <td>1.281111</td>\n",
       "      <td>10.964400</td>\n",
       "      <td>11.765000</td>\n",
       "      <td>5.928000</td>\n",
       "      <td>-0.063985</td>\n",
       "      <td>-1.737926</td>\n",
       "      <td>0.892308</td>\n",
       "      <td>1.673941</td>\n",
       "      <td>-17.379267</td>\n",
       "      <td>-0.639854</td>\n",
       "      <td>-3.086110</td>\n",
       "      <td>-1.318003</td>\n",
       "      <td>1.272396</td>\n",
       "      <td>-0.117518</td>\n",
       "      <td>17.764767</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "NameError",
     "evalue": "name 'new_model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[5], line 28\u001b[0m\n\u001b[1;32m     20\u001b[0m trainer \u001b[38;5;241m=\u001b[39m ORPOTrainer(\n\u001b[1;32m     21\u001b[0m     model\u001b[38;5;241m=\u001b[39mmodel,\n\u001b[1;32m     22\u001b[0m     args\u001b[38;5;241m=\u001b[39morpo_args,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     25\u001b[0m     tokenizer\u001b[38;5;241m=\u001b[39mtokenizer,\n\u001b[1;32m     26\u001b[0m )\n\u001b[1;32m     27\u001b[0m trainer\u001b[38;5;241m.\u001b[39mtrain()\n\u001b[0;32m---> 28\u001b[0m trainer\u001b[38;5;241m.\u001b[39msave_model(\u001b[43mnew_model\u001b[49m)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'new_model' is not defined"
     ]
    }
   ],
   "source": [
    "orpo_args = ORPOConfig(\n",
    "    learning_rate=8e-5,\n",
    "    beta=0.1,\n",
    "    lr_scheduler_type=\"linear\",\n",
    "    max_length=1024,\n",
    "    max_prompt_length=512,\n",
    "    per_device_train_batch_size=2,\n",
    "    per_device_eval_batch_size=2,\n",
    "    gradient_accumulation_steps=4,\n",
    "    optim=\"paged_adamw_8bit\",\n",
    "    num_train_epochs=3,\n",
    "    evaluation_strategy=\"steps\",\n",
    "    eval_steps=0.2,\n",
    "    logging_steps=1,\n",
    "    warmup_steps=10,\n",
    "    report_to=\"wandb\",\n",
    "    output_dir=\"./orpo_checkpoint_dir/\",\n",
    ")\n",
    "\n",
    "trainer = ORPOTrainer(\n",
    "    model=model,\n",
    "    args=orpo_args,\n",
    "    train_dataset=dataset[\"train\"],\n",
    "    eval_dataset=dataset[\"test\"],\n",
    "    tokenizer=tokenizer,\n",
    ")\n",
    "trainer.train()\n",
    "trainer.save_model(new_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_model = \"smol_orpo\"\n",
    "trainer.save_model(new_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "system\n",
      "You are Pi-Card, the Raspberry Pi voice assistant.\n",
      "user\n",
      "Can you tell me 5 fun facts about space?\n",
      "assistant\n",
      "Sure! Here are five fun facts about space:\n",
      "\n",
      "1. Space is the final frontier: It's the vast expanse of space that exists beyond Earth's atmosphere.\n",
      "2. Space is a place for exploration: Astronauts and scientists study space to learn more about the universe and its mysteries.\n",
      "3. Space is full of wonders: From black holes to distant galaxies, space is filled with incredible phenomena that are still being discovered.\n",
      "4. Space is a big place: The universe is vast, with billions of galaxies and countless stars, planets, and other celestial bodies.\n",
      "5. Space is a big business: The space industry is a billion-dollar business, with companies like SpaceX and Blue Origin investing heavily in space exploration and technology.\n",
      "\n",
      "These are just a few fun facts about space. There is so much more to learn about this fascinating and mysterious place!\n",
      "------\n"
     ]
    }
   ],
   "source": [
    "# test an example model forward pass\n",
    "model.eval();\n",
    "\n",
    "#example_prompts = [\"What does your name stand for?\" , \"Tell me some fun facts about space\", \"Who are you?\", \"What can you do for me?\", \"Who won the US presidential election?\"]\n",
    "example_prompts = [\"Can you tell me 5 fun facts about space?\"]\n",
    "for prompt in example_prompts:\n",
    "  chat_prompt = f\"\"\"<|im_start|>system\\nYou are Pi-Card, the Raspberry Pi voice assistant.<|im_end|>\\n<|im_start|>user\\n{prompt}<|im_end|>\\n<|im_start|>assistant\\n\"\"\"\n",
    "#    chat_prompt = f\"\"\"<|im_start|>user\\n{prompt}<|im_end|>\\n<|im_start|>assistant\\n\"\"\"\n",
    "\n",
    "  input_ids = tokenizer.encode(chat_prompt, return_tensors='pt')\n",
    "  # send to cuda\n",
    "  input_ids = input_ids.to(model.device)\n",
    "\n",
    "  output = model.generate(input_ids, max_length=1000, pad_token_id=tokenizer.eos_token_id, do_sample=False)\n",
    "\n",
    "  output_text = tokenizer.decode(output[0], skip_special_tokens=True) \n",
    "  print(output_text)\n",
    "  print('------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:hf-to-gguf:Loading model: checkpoint-4773\n",
      "INFO:gguf.gguf_writer:gguf: This GGUF file is for Little Endian only\n",
      "INFO:hf-to-gguf:Exporting model...\n",
      "INFO:hf-to-gguf:gguf: loading model part 'model.safetensors'\n",
      "INFO:hf-to-gguf:token_embd.weight,           torch.float32 --> F16, shape = {960, 49152}\n",
      "INFO:hf-to-gguf:blk.0.attn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.0.ffn_down.weight,       torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.0.ffn_gate.weight,       torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.0.ffn_up.weight,         torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.0.ffn_norm.weight,       torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.0.attn_k.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.0.attn_output.weight,    torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.0.attn_q.weight,         torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.0.attn_v.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.1.attn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.1.ffn_down.weight,       torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.1.ffn_gate.weight,       torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.1.ffn_up.weight,         torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.1.ffn_norm.weight,       torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.1.attn_k.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.1.attn_output.weight,    torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.1.attn_q.weight,         torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.1.attn_v.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.10.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.10.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.10.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.10.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.10.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.10.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.10.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.10.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.10.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.11.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.11.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.11.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.11.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.11.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.11.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.11.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.11.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.11.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.12.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.12.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.12.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.12.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.12.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.12.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.12.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.12.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.12.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.13.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.13.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.13.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.13.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.13.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.13.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.13.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.13.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.13.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.14.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.14.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.14.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.14.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.14.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.14.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.14.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.14.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.14.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.15.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.15.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.15.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.15.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.15.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.15.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.15.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.15.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.15.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.16.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.16.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.16.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.16.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.16.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.16.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.16.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.16.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.16.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.17.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.17.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.17.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.17.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.17.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.17.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.17.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.17.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.17.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.18.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.18.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.18.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.18.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.18.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.18.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.18.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.18.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.18.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.19.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.19.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.19.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.19.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.19.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.19.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.19.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.19.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.19.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.2.attn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.2.ffn_down.weight,       torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.2.ffn_gate.weight,       torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.2.ffn_up.weight,         torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.2.ffn_norm.weight,       torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.2.attn_k.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.2.attn_output.weight,    torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.2.attn_q.weight,         torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.2.attn_v.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.20.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.20.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.20.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.20.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.20.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.20.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.20.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.20.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.20.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.21.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.21.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.21.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.21.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.21.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.21.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.21.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.21.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.21.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.22.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.22.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.22.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.22.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.22.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.22.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.22.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.22.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.22.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.23.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.23.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.23.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.23.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.23.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.23.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.23.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.23.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.23.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.24.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.24.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.24.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.24.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.24.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.24.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.24.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.24.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.24.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.25.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.25.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.25.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.25.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.25.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.25.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.25.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.25.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.25.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.26.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.26.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.26.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.26.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.26.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.26.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.26.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.26.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.26.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.27.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.27.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.27.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.27.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.27.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.27.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.27.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.27.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.27.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.28.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.28.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.28.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.28.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.28.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.28.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.28.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.28.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.28.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.29.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.29.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.29.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.29.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.29.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.29.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.29.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.29.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.29.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.3.attn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.3.ffn_down.weight,       torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.3.ffn_gate.weight,       torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.3.ffn_up.weight,         torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.3.ffn_norm.weight,       torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.3.attn_k.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.3.attn_output.weight,    torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.3.attn_q.weight,         torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.3.attn_v.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.30.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.30.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.30.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.30.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.30.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.30.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.30.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.30.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.30.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.31.attn_norm.weight,     torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.31.ffn_down.weight,      torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.31.ffn_gate.weight,      torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.31.ffn_up.weight,        torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.31.ffn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.31.attn_k.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.31.attn_output.weight,   torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.31.attn_q.weight,        torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.31.attn_v.weight,        torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.4.attn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.4.ffn_down.weight,       torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.4.ffn_gate.weight,       torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.4.ffn_up.weight,         torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.4.ffn_norm.weight,       torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.4.attn_k.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.4.attn_output.weight,    torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.4.attn_q.weight,         torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.4.attn_v.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.5.attn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.5.ffn_down.weight,       torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.5.ffn_gate.weight,       torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.5.ffn_up.weight,         torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.5.ffn_norm.weight,       torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.5.attn_k.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.5.attn_output.weight,    torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.5.attn_q.weight,         torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.5.attn_v.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.6.attn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.6.ffn_down.weight,       torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.6.ffn_gate.weight,       torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.6.ffn_up.weight,         torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.6.ffn_norm.weight,       torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.6.attn_k.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.6.attn_output.weight,    torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.6.attn_q.weight,         torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.6.attn_v.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.7.attn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.7.ffn_down.weight,       torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.7.ffn_gate.weight,       torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.7.ffn_up.weight,         torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.7.ffn_norm.weight,       torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.7.attn_k.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.7.attn_output.weight,    torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.7.attn_q.weight,         torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.7.attn_v.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.8.attn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.8.ffn_down.weight,       torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.8.ffn_gate.weight,       torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.8.ffn_up.weight,         torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.8.ffn_norm.weight,       torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.8.attn_k.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.8.attn_output.weight,    torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.8.attn_q.weight,         torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.8.attn_v.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.9.attn_norm.weight,      torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.9.ffn_down.weight,       torch.float32 --> F16, shape = {2560, 960}\n",
      "INFO:hf-to-gguf:blk.9.ffn_gate.weight,       torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.9.ffn_up.weight,         torch.float32 --> F16, shape = {960, 2560}\n",
      "INFO:hf-to-gguf:blk.9.ffn_norm.weight,       torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:blk.9.attn_k.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:blk.9.attn_output.weight,    torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.9.attn_q.weight,         torch.float32 --> F16, shape = {960, 960}\n",
      "INFO:hf-to-gguf:blk.9.attn_v.weight,         torch.float32 --> F16, shape = {960, 320}\n",
      "INFO:hf-to-gguf:output_norm.weight,          torch.float32 --> F32, shape = {960}\n",
      "INFO:hf-to-gguf:Set meta model\n",
      "INFO:hf-to-gguf:Set model parameters\n",
      "INFO:hf-to-gguf:gguf: context length = 2048\n",
      "INFO:hf-to-gguf:gguf: embedding length = 960\n",
      "INFO:hf-to-gguf:gguf: feed forward length = 2560\n",
      "INFO:hf-to-gguf:gguf: head count = 15\n",
      "INFO:hf-to-gguf:gguf: key-value head count = 5\n",
      "INFO:hf-to-gguf:gguf: rope theta = 10000.0\n",
      "INFO:hf-to-gguf:gguf: rms norm epsilon = 1e-05\n",
      "INFO:hf-to-gguf:gguf: file type = 1\n",
      "INFO:hf-to-gguf:Set model tokenizer\n",
      "INFO:gguf.vocab:Adding 48900 merge(s).\n",
      "INFO:gguf.vocab:Setting special token type bos to 1\n",
      "INFO:gguf.vocab:Setting special token type eos to 2\n",
      "INFO:gguf.vocab:Setting special token type unk to 0\n",
      "INFO:gguf.vocab:Setting special token type pad to 0\n",
      "INFO:gguf.vocab:Setting chat_template to {% for message in messages %}{{'<|im_start|>' + message['role'] + '\n",
      "' + message['content'] + '<|im_end|>' + '\n",
      "'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n",
      "' }}{% endif %}\n",
      "INFO:hf-to-gguf:Set model quantization version\n",
      "INFO:gguf.gguf_writer:Writing the following files:\n",
      "INFO:gguf.gguf_writer:picard-orpo-0.36b-f16.gguf: n_tensors = 290, total_size = 723.8M\n",
      "Writing: 100%|█████████████████████████████| 724M/724M [00:02<00:00, 291Mbyte/s]\n",
      "INFO:hf-to-gguf:Model successfully exported to picard-orpo-0.36b-f16.gguf\n"
     ]
    }
   ],
   "source": [
    "# Create gguf file\n",
    "\n",
    "# Please note you'll need to update the checkpoint path and model names to the one you want to convert & save\n",
    "!python llama.cpp/convert_hf_to_gguf.py orpo_checkpoint_dir/checkpoint-4773 --outfile picard-orpo-0.36b-f16.gguf --outtype f16\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}