Z01_TextGen-HPC_v4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "e0PlUF-R2rmx"
   },
   "source": [
    "# oobabooga/text-generation-webui"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download package"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'text-generation-webui'...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Note: checking out '705f04a0c9b09870fbe595dbba1d61db8289a07b'.\n",
      "\n",
      "You are in 'detached HEAD' state. You can look around, make experimental\n",
      "changes and commit them, and you can discard any commits you make in this\n",
      "state without impacting any branches by performing another checkout.\n",
      "\n",
      "If you want to create a new branch to retain commits you create, you may\n",
      "do so (now or later) by using -b with the checkout command again. Example:\n",
      "\n",
      "  git checkout -b new_branch_name\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "# Delete old version\n",
    "rm -rf text-generation-webui\n",
    "sleep 3\n",
    "REPOSRC=https://github.com/oobabooga/text-generation-webui.git\n",
    "LOCALREPO=text-generation-webui\n",
    "BRANCH=\"snapshot-2023-12-10\"\n",
    "git clone --branch $BRANCH $REPOSRC $LOCALREPO\n",
    "\n",
    "# Replace \n",
    "cp text-generation-webui/server.py text-generation-webui/server_nchc.py\n",
    "sed -i 's@prevent_thread_lock=True,@root_path=shared.args.public_api_id, prevent_thread_lock=True,@g' text-generation-webui/server_nchc.py            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RUN TEXTGEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://node01.biobank.org.tw/rstudio/172.16.124.152/34226/\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "kill: sending signal to 64428 failed: No such process\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "# IP\n",
    "node_ip=$(cat /etc/hosts |grep \"$(hostname -a)\" | awk '{print $1}')\n",
    "\n",
    "# PORT\n",
    "noed_port_genai=$(python -c \"import socket; s = socket.socket(socket.AF_INET, socket.SOCK_STREAM); s.bind(('', 0)); addr = s.getsockname(); s.close(); print(addr[1])\")\n",
    "noed_port_api=$(python -c \"import socket; s = socket.socket(socket.AF_INET, socket.SOCK_STREAM); s.bind(('', 0)); addr = s.getsockname(); s.close(); print(addr[1])\")\n",
    "\n",
    "# PROXY\n",
    "proxy_url=/rstudio/${node_ip}/${noed_port_genai}\n",
    "\n",
    "# URL\n",
    "https_url=https://node01.biobank.org.tw${proxy_url}/\n",
    "echo $https_url\n",
    "\n",
    "# Password\n",
    "password=\"nchc:nchcorgtw\"\n",
    "\n",
    "# Kill Server\n",
    "ps -ef |grep server.py | awk '{print $2}' | xargs kill -9\n",
    "ps -ef |grep server_nchc.py | awk '{print $2}' | xargs kill -9\n",
    "\n",
    "# CMD\n",
    "CMD=\"HF_TOKEN=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 python server_nchc.py --api --api-port ${noed_port_api} --listen --listen-port ${noed_port_genai} --listen-host $(hostname -s) \\\n",
    "--chat-buttons --public-api-id ${proxy_url} --gradio-auth ${password} --extensions Training_PRO\"\n",
    "\n",
    "# SINGULARITY\n",
    "cat << EOF >  $(pwd)/demo.sh\n",
    "/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec --nv -C \\\n",
    "-B $(pwd)/text-generation-webui:/app \\\n",
    "-B /work \\\n",
    "/work/u00cjz00/nvidia/cuda118/c00cjz00_cuda11.8_text-generation-webui_snapshot-2023-12-10.sif \\\n",
    "bash -c 'cd /app; ${CMD}'\n",
    "EOF\n",
    "\n",
    "chmod 755 $(pwd)/demo.sh\n",
    "\n",
    "nohup $(pwd)/demo.sh  > ./textgen.log 2>&1 &"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "cd ./text-generation-webui/models\n",
    "ln -s /work/u00cjz00/slurm_jobs/github/models/Taiwan-LLM-7B-v2.1-chat-Q8_0.gguf .\n",
    "ln -s /work/u00cjz00/slurm_jobs/github/models/Llama-2-7b-chat-hf .\n",
    "ln -s /work/u00cjz00/slurm_jobs/github/models/Llama-2-7B-Chat-GPTQ ."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 強制刪除服務"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kill: sending signal to 83558 failed: No such process\n",
      "kill: sending signal to 83567 failed: No such process\n"
     ]
    }
   ],
   "source": [
    "# 強制刪除服務\n",
    "!ps -ef |grep server_nchc.py | awk '{print $2}' | xargs kill -9\n",
    "!ps -ef |grep server.py | awk '{print $2}' | xargs kill -9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}