F01_datasets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ff818959-0fd5-4a28-b09d-8af2b6e47bfe",
   "metadata": {},
   "source": [
    "##  Finetuning, Create Datasets"
   ]
  },
  {
   "cell_type": "raw",
   "id": "6db72f80-9eca-4499-a0d1-a4ee8c20ce99",
   "metadata": {},
   "source": [
    "#### Reference \n",
    "https://huggingface.co/CHIH-HUNG/llama-2-13b-FINETUNE3_3.3w-r4-q_k_v_o\n",
    "https://huggingface.co/datasets/huangyt/FINETUNE3\n",
    "https://huggingface.co/erhwenkuo\n",
    "https://huggingface.co/datasets/erhwenkuo/school_math_0.25m-zhtw\n",
    "https://huggingface.co/datasets/erhwenkuo/medical_dialogue-chinese-zhtw\n",
    "https://huggingface.co/datasets/DataAgent/medical-qa-instruction-zhtw\n",
    "https://huggingface.co/datasets/lavita/ChatDoctor-HealthCareMagic-100k\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e44fa40-2323-4693-9fb7-f35a937e1d26",
   "metadata": {},
   "source": [
    "## 初始環境設定"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3285ba4-6ab8-450d-84e2-dbb8e287a93f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "HOME = str(Path.home())\n",
    "Add_Binarry_Path=HOME+'/.local/bin'\n",
    "os.environ['PATH']=os.environ['PATH']+':'+Add_Binarry_Path\n",
    "current_foldr=!pwd\n",
    "current_foldr=current_foldr[0]\n",
    "current_foldr"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9456fd3a-842d-4ac0-9d42-1343c1a7e53b",
   "metadata": {},
   "source": [
    "## 安裝套件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9540a73-2762-4ac4-97ee-1a018dc664dc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "pip install datasets -q"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b8f2282-5719-49f7-bf1c-b51e05050218",
   "metadata": {},
   "source": [
    "### LOAD LIBRARY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9eb9652c-a06a-492b-8545-dd9714b2fcea",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b8c9f7f-10d3-48fc-86d9-a39cd9eafd58",
   "metadata": {},
   "source": [
    "### Loading dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea9beae9-429c-462c-969c-83aee3276724",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 讀取數據集，take可以取得該數據集前n筆資料\n",
    "dataset = load_dataset(\"lavita/ChatDoctor-HealthCareMagic-100k\", split=\"train\", streaming=True)\n",
    "\n",
    "# 提取所需欄位並建立新的字典列表\n",
    "limit=0\n",
    "extracted_data = []\n",
    "for example in dataset:\n",
    "    extracted_example = {\n",
    "        \"instruction\": example[\"instruction\"],\n",
    "        \"input\": example[\"input\"],\n",
    "        \"output\": example[\"output\"]\n",
    "    }\n",
    "    extracted_data.append(extracted_example)\n",
    "    if len(extracted_data) == limit:\n",
    "        break\n",
    "\n",
    "# 指定 JSON 文件名稱\n",
    "json_filename = \"data.json\"\n",
    "\n",
    "# 寫入 JSON 文件\n",
    "with open(json_filename, \"w\") as json_file:\n",
    "    json.dump(extracted_data, json_file, indent=4)\n",
    "\n",
    "print(f\"數據已提取並保存為 {json_filename}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6e6035d-a970-42ee-a1c0-c65df4688e5f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 所有資料內容\n",
    "import pandas as pd\n",
    "df = pd. read_json ( 'data.json' )\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5af944f6-5311-4fce-bde3-0ca9eebc0002",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 所有資料內容狀態\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a89bba4f-abd7-4b69-9416-3e2178944fd2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 取出前一千筆, 並儲存\n",
    "dataset_df_1k = df[:1000]\n",
    "dataset_df_1k.to_json('data_1k.json', orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e73bfd85-53d8-4f8b-9039-8fae6b5e5f41",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 列出資料內容\n",
    "df = pd. read_json ( 'data_1k.json' )\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47e4e718-d72f-4f2e-9179-bee5ace0fa7d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}