01_create_image_ipykernel_v1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1f8f93ae-3208-4361-a023-d24a5109f951",
   "metadata": {},
   "source": [
    "## 01_create_image_ipykernel \n",
    "#### <span style=\"color:red\">Change to Default kernel:  Pyhon 3 (ipykernel)</span>."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94b2c504-0dbf-4e8c-9fd5-2ae0db8749c0",
   "metadata": {},
   "source": [
    "### 1. 任何版本已經安裝python之sif檔案"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36e2b37a-e84b-4db0-9520-a862829ecc36",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "### 0. pipi install libsray savefolder\n",
    "saveFolder=work\n",
    "label=genai\n",
    "\n",
    "### 1. IMAGE\n",
    "#IMAGE=/work/u00cjz00/nvidia/text-generation-inference_1.1.1.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/cjz_images/pytorch_2.1.0-cuda11.8-cudnn8-runtime_textgen.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv.sif\n",
    "IMAGE_basename=S-${saveFolder}-${label}_$(basename \"$IMAGE\" .sif)\n",
    "\n",
    "### 2. VIRTUAL LIBRARY and BINARY FOLDER\n",
    "libraryFolder=/${saveFolder}/$(whoami)/libraryFolder/${IMAGE_basename}/local/lib\n",
    "libraryFolder_binding=${libraryFolder}:${HOME}/.local/lib\n",
    "binFolder=/${saveFolder}/$(whoami)/libraryFolder/${IMAGE_basename}/local/bin\n",
    "binFolder_binding=${binFolder}:${HOME}/.local/bin\n",
    "rm -rf /${saveFolder}/$(whoami)/libraryFolder/${IMAGE_basename}\n",
    "mkdir -p ${libraryFolder} ${binFolder}\n",
    "\n",
    "# 3. PIP INSTALL SLAVE IPYKERNEL\n",
    "ml libs/singularity/3.10.2\n",
    "singularity exec -B ${libraryFolder_binding} -B ${binFolder_binding} ${IMAGE} pip install -q ipykernel IProgress ipywidgets\n",
    "\n",
    "# 4. IPYKERNEL for IMAGE\n",
    "IPYKERNEL=/work/u00cjz00/slurm_jobs/ipykernel/t2/image_with_ipykernel_local\n",
    "mkdir -p ${HOME}/.local/share/jupyter/kernels/\n",
    "rm -rf ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}\n",
    "cp -rf ${IPYKERNEL} ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}\n",
    "chmod -R 755 ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}\n",
    "IMAGE_desc=$(echo $IMAGE | sed 's_/_\\\\/_g')\n",
    "sed -i \"s/templateSIF/${IMAGE_desc}/g\" ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "sed -i \"s/templateImage/Image_${IMAGE_basename}/g\" ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "sed -i \"s@templateLibrayFolder@${libraryFolder_binding}@g\" ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "sed -i \"s@templateBinFolder@${binFolder_binding}@g\" ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "\n",
    "# 5. check size\n",
    "du -sh ${libraryFolder}\n",
    "echo /home/$(whoami)/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "cat /home/$(whoami)/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "\n",
    "cmd=\"/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec --nv --cleanenv \\\n",
    "-B /work -B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin \\\n",
    "-B ${libraryFolder}:/home/g00cjz00/.local/lib \\\n",
    "-B ${binFolder}:/home/g00cjz00/.local/bin \\\n",
    "${IMAGE} \\\n",
    "bash -c 'export PATH=\\$PATH:\\$HOME/.local/bin; echo \\$PATH;'\n",
    "\"\n",
    "\n",
    "echo ${cmd}\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee333646-f5b1-485b-a18c-58500d405f61",
   "metadata": {},
   "source": [
    "### 2. cuda_11.8.0-cudnn8-runtime-ubuntu22.04.sif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e83512a5-32e4-4a08-9adb-64b611d2316a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "### 0. pipi install libsray savefolder\n",
    "saveFolder=work\n",
    "label=genai\n",
    "python_version=3.10\n",
    "\n",
    "### 1. IMAGE\n",
    "IMAGE=/work/u00cjz00/nvidia/cuda_11.8.0-cudnn8-runtime-ubuntu22.04.sif\n",
    "IMAGE_basename=P-${python_version}-${saveFolder}-${label}_$(basename \"$IMAGE\" .sif)\n",
    "\n",
    "### 2. VIRTUAL LIBRARY and BINARY FOLDER\n",
    "libraryFolder=/${saveFolder}/$(whoami)/libraryFolder/${IMAGE_basename}/local/lib\n",
    "libraryFolder_binding=${libraryFolder}:${HOME}/.local/lib\n",
    "binFolder=/${saveFolder}/$(whoami)/libraryFolder/${IMAGE_basename}/local/bin\n",
    "binFolder_binding=${binFolder}:${HOME}/.local/bin\n",
    "rm -rf /${saveFolder}/$(whoami)/libraryFolder/${IMAGE_basename}\n",
    "mkdir -p ${libraryFolder} ${binFolder}\n",
    "\n",
    "# 3. PIP INSTALL SLAVE IPYKERNEL\n",
    "ml libs/singularity/3.10.2\n",
    "singularity exec -B ${libraryFolder_binding} -B ${binFolder_binding} ${IMAGE} pip${python_version} install -q ipykernel IProgress ipywidgets\n",
    "\n",
    "# 4. IPYKERNEL for IMAGE\n",
    "IPYKERNEL=/work/u00cjz00/slurm_jobs/ipykernel/t3bio/image_with_ipykernel_local_python\n",
    "mkdir -p ${HOME}/.local/share/jupyter/kernels/\n",
    "rm -rf ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}\n",
    "cp -rf ${IPYKERNEL} ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}\n",
    "chmod -R 755 ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}\n",
    "IMAGE_desc=$(echo $IMAGE | sed 's_/_\\\\/_g')\n",
    "sed -i \"s/templateSIF/${IMAGE_desc}/g\" ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "sed -i \"s/templateImage/Image_${IMAGE_basename}/g\" ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "sed -i \"s@templateLibrayFolder@${libraryFolder_binding}@g\" ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "sed -i \"s@templateBinFolder@${binFolder_binding}@g\" ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "sed -i \"s@_PythonVersion@${python_version}@g\" ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "\n",
    "# 5. check size\n",
    "du -sh ${libraryFolder}\n",
    "echo /home/$(whoami)/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "cat /home/$(whoami)/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "\n",
    "cmd=\"/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec --nv --cleanenv \\\n",
    "-B /work -B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin \\\n",
    "-B ${libraryFolder}:/home/g00cjz00/.local/lib \\\n",
    "-B ${binFolder}:/home/g00cjz00/.local/bin \\\n",
    "${IMAGE} \\\n",
    "bash -c 'export PATH=\\$PATH:\\$HOME/.local/bin; echo \\$PATH;'\n",
    "\"\n",
    "\n",
    "echo ${cmd}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d941444-7d62-4c54-8969-d1bb566fd018",
   "metadata": {},
   "source": [
    "### 3. 確認安裝狀況 (1. 任何版本已經安裝python之sif檔案)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b70bb551-62fb-445a-bde8-a3eb1bd99d43",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "### 0. pipi install libsray savefolder\n",
    "saveFolder=work\n",
    "label=genai\n",
    "\n",
    "### 1. IMAGE\n",
    "#IMAGE=/work/u00cjz00/nvidia/text-generation-inference_1.1.1.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/cjz_images/pytorch_2.1.0-cuda11.8-cudnn8-runtime_textgen.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv.sif\n",
    "IMAGE_basename=S-${saveFolder}-${label}_$(basename \"$IMAGE\" .sif)\n",
    "\n",
    "### 2. VIRTUAL LIBRARY and BINARY FOLDER\n",
    "libraryFolder=/${saveFolder}/$(whoami)/libraryFolder/${IMAGE_basename}/local/lib\n",
    "du -sh ${libraryFolder}\n",
    "echo /home/$(whoami)/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "cat /home/$(whoami)/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "\n",
    "cmd=\"/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec --nv --cleanenv \\\n",
    "-B /work -B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin \\\n",
    "-B ${libraryFolder}:/home/g00cjz00/.local/lib \\\n",
    "-B ${binFolder}:/home/g00cjz00/.local/bin \\\n",
    "${IMAGE} \\\n",
    "bash -c 'export PATH=\\$PATH:\\$HOME/.local/bin; echo \\$PATH;'\n",
    "\"\n",
    "\n",
    "echo ${cmd}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1bc99317-cc8a-4cbb-91c8-05940740a9e1",
   "metadata": {},
   "source": [
    "## 4. 確認安裝狀況 (1. 任何版本已經安裝python之sif檔案)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d34471f-fb4f-4854-8dfe-f901e469da98",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "### 0. pipi install libsray savefolder\n",
    "saveFolder=work\n",
    "label=genai\n",
    "python_version=3.10\n",
    "\n",
    "### 1. IMAGE\n",
    "IMAGE=/work/u00cjz00/nvidia/cuda_11.8.0-cudnn8-runtime-ubuntu22.04.sif\n",
    "IMAGE_basename=P-${python_version}-${saveFolder}-${label}_$(basename \"$IMAGE\" .sif)\n",
    "\n",
    "### 2. VIRTUAL LIBRARY and BINARY FOLDER\n",
    "libraryFolder=/${saveFolder}/$(whoami)/libraryFolder/${IMAGE_basename}/local/lib\n",
    "du -sh ${libraryFolder}\n",
    "echo /home/$(whoami)/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "cat /home/$(whoami)/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "\n",
    "cmd=\"/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec --nv --cleanenv \\\n",
    "-B /work -B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin \\\n",
    "-B ${libraryFolder}:/home/g00cjz00/.local/lib \\\n",
    "-B ${binFolder}:/home/g00cjz00/.local/bin \\\n",
    "${IMAGE} \\\n",
    "bash -c 'export PATH=\\$PATH:\\$HOME/.local/bin; echo \\$PATH;'\n",
    "\"\n",
    "\n",
    "echo ${cmd}"
   ]
  },
  {
   "cell_type": "raw",
   "id": "f24fbe3b-f064-4dfe-85d9-74e17c07b8b9",
   "metadata": {},
   "source": [
    "# Link\n",
    "!ln -s /work/$(whoami)/libraryFolder /home/$(whoami)/libraryFolder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbecc406-1c87-45ea-884d-fc4e5dd2ed4e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}