a03_Text_Generation_WebUI_hpc_Simple.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "e0PlUF-R2rmx"
   },
   "source": [
    "## 實作案例: 請先切換kernel 於您新安裝的 kernel環境\n",
    "## <span style=\"color:red\">Change to kernel:   Image_XXXXXXXXXXl</span>.\n",
    "\n",
    "https://python.langchain.com.cn/docs/get_started/quickstart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/work/g00cjz00/github/nycu_class2023_v1'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 初始環境設定\n",
    "import os\n",
    "from pathlib import Path\n",
    "HOME = str(Path.home())\n",
    "Add_Binarry_Path=HOME+'/.local/bin:/usr/ubuntu_bin'\n",
    "os.environ['PATH']=os.environ['PATH']+':'+Add_Binarry_Path\n",
    "current_foldr=!pwd\n",
    "current_foldr=current_foldr[0]\n",
    "current_foldr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 主程式 \n",
    "1. 請修改 倒數第五行 password=\"nchc:nchcorgtw\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning into 'text-generation-webui'...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://node01.biobank.org.tw/rstudio/172.17.9.1/46704/\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "kill: (34045): No such process\n",
      "2023-12-12 20:59:05 INFO:\u001b[32mLoading the extension \"gallery\"...\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on local URL:  http://gn0901:46704\n",
      "\n",
      "To create a public link, set `share=True` in `launch()`.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/work/g00cjz00/github/nycu_class2023_v1/text-generation-webui/server_nchc.py\", line 241, in <module>\n",
      "    time.sleep(0.5)\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/IPython/core/magics/script.py:288\u001b[0m, in \u001b[0;36mScriptMagics.shebang\u001b[0;34m(self, line, cell)\u001b[0m\n\u001b[1;32m    287\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 288\u001b[0m     \u001b[43min_thread\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_stream_communicate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcell\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    289\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m:\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/IPython/core/magics/script.py:210\u001b[0m, in \u001b[0;36mScriptMagics.shebang.<locals>.in_thread\u001b[0;34m(coro)\u001b[0m\n\u001b[1;32m    209\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Call a coroutine on the asyncio thread\"\"\"\u001b[39;00m\n\u001b[0;32m--> 210\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43masyncio\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_coroutine_threadsafe\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcoro\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevent_loop\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/concurrent/futures/_base.py:453\u001b[0m, in \u001b[0;36mFuture.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    451\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__get_result()\n\u001b[0;32m--> 453\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_condition\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    455\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;129;01min\u001b[39;00m [CANCELLED, CANCELLED_AND_NOTIFIED]:\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/threading.py:320\u001b[0m, in \u001b[0;36mCondition.wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    319\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 320\u001b[0m     \u001b[43mwaiter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43macquire\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    321\u001b[0m     gotit \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mget_ipython\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_cell_magic\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mbash\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m#  DOWNLOAD text-generation-webui, install requirements.txt and execute this service \u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mREPOSRC=https://github.com/oobabooga/text-generation-webui.git\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mLOCALREPO=text-generation-webui\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m# We do it this way so that we can abstract if from just git later on\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mLOCALREPO_VC_DIR=$LOCALREPO/.git\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mif [ ! -d $LOCALREPO_VC_DIR ]\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mthen\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m    git clone $REPOSRC $LOCALREPO\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m    # It does not need to pip with this IMAGE\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m    #!pip install -r text-generation-webui/temp_requirements.txt -q \u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m    # Prepare\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m    cp ./textgen/config-user.yaml ./text-generation-webui/models/config-user.yaml\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m    cp ./textgen/server_nchc.py ./text-generation-webui/server_nchc.py\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m    cd ./text-generation-webui/models\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m    ln -s /work/u00cjz00/slurm_jobs/github/models/Taiwan-LLM-7B-v2.1-chat-Q8_0.gguf .\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m    ln -s /work/u00cjz00/slurm_jobs/github/models/Llama-2-7b-chat-hf .\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m    ln -s /work/u00cjz00/slurm_jobs/github/models/Llama-2-7B-Chat-GPTQ .\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m    cd ../../\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m    \u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mfi \u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m# GET IP and Free Port\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m# IP\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mnode_ip=$(cat /etc/hosts |grep \u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m$(hostname -a)\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m | awk \u001b[39;49m\u001b[38;5;130;43;01m\\'\u001b[39;49;00m\u001b[38;5;124;43m{\u001b[39;49m\u001b[38;5;124;43mprint $1}\u001b[39;49m\u001b[38;5;130;43;01m\\'\u001b[39;49;00m\u001b[38;5;124;43m)\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m# PORT\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mnoed_port_genai=$(python -c \u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mimport socket; s = socket.socket(socket.AF_INET, socket.SOCK_STREAM); s.bind((\u001b[39;49m\u001b[38;5;130;43;01m\\'\u001b[39;49;00m\u001b[38;5;130;43;01m\\'\u001b[39;49;00m\u001b[38;5;124;43m, 0)); addr = s.getsockname(); s.close(); print(addr[1])\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m)\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m# PROXY\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mproxy_url=/rstudio/$\u001b[39;49m\u001b[38;5;132;43;01m{node_ip}\u001b[39;49;00m\u001b[38;5;124;43m/$\u001b[39;49m\u001b[38;5;132;43;01m{noed_port_genai}\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m# URL\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mhttps_url=https://node01.biobank.org.tw$\u001b[39;49m\u001b[38;5;132;43;01m{proxy_url}\u001b[39;49;00m\u001b[38;5;124;43m/\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mecho $https_url\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m# 強制刪除運行中服務\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mps -ef |grep server.py | awk \u001b[39;49m\u001b[38;5;130;43;01m\\'\u001b[39;49;00m\u001b[38;5;124;43m{\u001b[39;49m\u001b[38;5;124;43mprint $2}\u001b[39;49m\u001b[38;5;130;43;01m\\'\u001b[39;49;00m\u001b[38;5;124;43m | xargs kill -9\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m# EXPORT PATH for ~/.local\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mexport PATH=$PATH:$HOME/.local/bin\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m# 主程式\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mpassword=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mnchc:nchcorgtw\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mcd text-generation-webui\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43mHF_TOKEN=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mhf_RJmXSjaHfLrwNdyALUziURORNkHsZQfwzD\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m python server_nchc.py  --listen --listen-port $\u001b[39;49m\u001b[38;5;132;43;01m{noed_port_genai}\u001b[39;49;00m\u001b[38;5;124;43m --listen-host $(hostname -s) \u001b[39;49m\u001b[38;5;130;43;01m\\\\\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m--chat-buttons --ssl-keyfile $\u001b[39;49m\u001b[38;5;132;43;01m{proxy_url}\u001b[39;49;00m\u001b[38;5;124;43m --gradio-auth $\u001b[39;49m\u001b[38;5;132;43;01m{password}\u001b[39;49;00m\u001b[38;5;124;43m \u001b[39;49m\u001b[38;5;130;43;01m\\\\\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2493\u001b[0m, in \u001b[0;36mInteractiveShell.run_cell_magic\u001b[0;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[1;32m   2491\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuiltin_trap:\n\u001b[1;32m   2492\u001b[0m     args \u001b[38;5;241m=\u001b[39m (magic_arg_s, cell)\n\u001b[0;32m-> 2493\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2495\u001b[0m \u001b[38;5;66;03m# The code below prevents the output from being displayed\u001b[39;00m\n\u001b[1;32m   2496\u001b[0m \u001b[38;5;66;03m# when using magics with decorator @output_can_be_silenced\u001b[39;00m\n\u001b[1;32m   2497\u001b[0m \u001b[38;5;66;03m# when the last Python token in the expression is a ';'.\u001b[39;00m\n\u001b[1;32m   2498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(fn, magic\u001b[38;5;241m.\u001b[39mMAGIC_OUTPUT_CAN_BE_SILENCED, \u001b[38;5;28;01mFalse\u001b[39;00m):\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/IPython/core/magics/script.py:154\u001b[0m, in \u001b[0;36mScriptMagics._make_script_magic.<locals>.named_script_magic\u001b[0;34m(line, cell)\u001b[0m\n\u001b[1;32m    152\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    153\u001b[0m     line \u001b[38;5;241m=\u001b[39m script\n\u001b[0;32m--> 154\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshebang\u001b[49m\u001b[43m(\u001b[49m\u001b[43mline\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcell\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/IPython/core/magics/script.py:291\u001b[0m, in \u001b[0;36mScriptMagics.shebang\u001b[0;34m(self, line, cell)\u001b[0m\n\u001b[1;32m    289\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m:\n\u001b[1;32m    290\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 291\u001b[0m         \u001b[43mp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend_signal\u001b[49m\u001b[43m(\u001b[49m\u001b[43msignal\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mSIGINT\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    292\u001b[0m         in_thread(asyncio\u001b[38;5;241m.\u001b[39mwait_for(p\u001b[38;5;241m.\u001b[39mwait(), timeout\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.1\u001b[39m))\n\u001b[1;32m    293\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m p\u001b[38;5;241m.\u001b[39mreturncode \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/asyncio/subprocess.py:137\u001b[0m, in \u001b[0;36mProcess.send_signal\u001b[0;34m(self, signal)\u001b[0m\n\u001b[1;32m    136\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msend_signal\u001b[39m(\u001b[38;5;28mself\u001b[39m, signal):\n\u001b[0;32m--> 137\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_transport\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend_signal\u001b[49m\u001b[43m(\u001b[49m\u001b[43msignal\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/asyncio/base_subprocess.py:146\u001b[0m, in \u001b[0;36mBaseSubprocessTransport.send_signal\u001b[0;34m(self, signal)\u001b[0m\n\u001b[1;32m    144\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msend_signal\u001b[39m(\u001b[38;5;28mself\u001b[39m, signal):\n\u001b[1;32m    145\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_proc()\n\u001b[0;32m--> 146\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_proc\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend_signal\u001b[49m\u001b[43m(\u001b[49m\u001b[43msignal\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/subprocess.py:2109\u001b[0m, in \u001b[0;36mPopen.send_signal\u001b[0;34m(self, sig)\u001b[0m\n\u001b[1;32m   2105\u001b[0m \u001b[38;5;66;03m# The race condition can still happen if the race condition\u001b[39;00m\n\u001b[1;32m   2106\u001b[0m \u001b[38;5;66;03m# described above happens between the returncode test\u001b[39;00m\n\u001b[1;32m   2107\u001b[0m \u001b[38;5;66;03m# and the kill() call.\u001b[39;00m\n\u001b[1;32m   2108\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 2109\u001b[0m     \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkill\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpid\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msig\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2110\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mProcessLookupError\u001b[39;00m:\n\u001b[1;32m   2111\u001b[0m     \u001b[38;5;66;03m# Supress the race condition error; bpo-40550.\u001b[39;00m\n\u001b[1;32m   2112\u001b[0m     \u001b[38;5;28;01mpass\u001b[39;00m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "#  DOWNLOAD text-generation-webui, install requirements.txt and execute this service \n",
    "\n",
    "REPOSRC=https://github.com/oobabooga/text-generation-webui.git\n",
    "LOCALREPO=text-generation-webui\n",
    "\n",
    "# We do it this way so that we can abstract if from just git later on\n",
    "LOCALREPO_VC_DIR=$LOCALREPO/.git\n",
    "\n",
    "if [ ! -d $LOCALREPO_VC_DIR ]\n",
    "then\n",
    "    git clone $REPOSRC $LOCALREPO\n",
    "\n",
    "    # It does not need to pip with this IMAGE\n",
    "    #!pip install -r text-generation-webui/temp_requirements.txt -q \n",
    "\n",
    "    # Prepare\n",
    "    cp ./textgen/config-user.yaml ./text-generation-webui/models/config-user.yaml\n",
    "    cp ./textgen/server_nchc.py ./text-generation-webui/server_nchc.py\n",
    "    cd ./text-generation-webui/models\n",
    "    ln -s /work/u00cjz00/slurm_jobs/github/models/Taiwan-LLM-7B-v2.1-chat-Q8_0.gguf .\n",
    "    ln -s /work/u00cjz00/slurm_jobs/github/models/Llama-2-7b-chat-hf .\n",
    "    ln -s /work/u00cjz00/slurm_jobs/github/models/Llama-2-7B-Chat-GPTQ .\n",
    "    cd ../../\n",
    "    \n",
    "fi \n",
    "\n",
    "# GET IP and Free Port\n",
    "# IP\n",
    "node_ip=$(cat /etc/hosts |grep \"$(hostname -a)\" | awk '{print $1}')\n",
    "# PORT\n",
    "noed_port_genai=$(python -c \"import socket; s = socket.socket(socket.AF_INET, socket.SOCK_STREAM); s.bind(('', 0)); addr = s.getsockname(); s.close(); print(addr[1])\")\n",
    "# PROXY\n",
    "proxy_url=/rstudio/${node_ip}/${noed_port_genai}\n",
    "# URL\n",
    "https_url=https://node01.biobank.org.tw${proxy_url}/\n",
    "echo $https_url\n",
    "\n",
    "# 強制刪除運行中服務\n",
    "ps -ef |grep server.py | awk '{print $2}' | xargs kill -9\n",
    "\n",
    "# EXPORT PATH for ~/.local\n",
    "export PATH=$PATH:$HOME/.local/bin\n",
    "\n",
    "# 主程式\n",
    "password=\"nchc:nchcorgtw\"\n",
    "cd text-generation-webui\n",
    "HF_TOKEN=\"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\" python server_nchc.py  --listen --listen-port ${noed_port_genai} --listen-host $(hostname -s) \\\n",
    "--chat-buttons --ssl-keyfile ${proxy_url} --gradio-auth ${password} \\\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kill: (33753): No such process\n"
     ]
    }
   ],
   "source": [
    "# 強制刪除服務\n",
    "!ps -ef |grep server.py | awk '{print $2}' | xargs kill -9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Image_S_work-demo01_pytorch_2.1.0-cuda11.8-cudnn8-runtime_textgen",
   "language": "python",
   "name": "s_work-demo01_pytorch_2.1.0-cuda11.8-cudnn8-runtime_textgen"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}