01_create_image_ipykernel.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1f8f93ae-3208-4361-a023-d24a5109f951",
   "metadata": {},
   "source": [
    "## 01_create_image_ipykernel\n",
    "#### <span style=\"color:red\">Change to Default kernel:  Pyhon 3 (ipykernel)</span>."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36e2b37a-e84b-4db0-9520-a862829ecc36",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "### 0. pipi install libsray savefolder\n",
    "saveFolder=work\n",
    "label=genai01\n",
    "### 1. IMAGE\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.7.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.8.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.9.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.10.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.11.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/pytorch_2.1.0-cuda11.8-cudnn8-devel.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/cjz_images/pytorch_2.1.0-cuda11.8-cudnn8-runtime.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/cjz_images/pytorch_2.1.0-cuda11.8-cudnn8-runtime_textgen.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/stable-diffusion-webui_full.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/pytorch_2.1.0-cuda11.8-cudnn8-devel.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/pytorch_22.12-py3.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/text-generation-inference_1.1.1.sif\n",
    "IMAGE_basename=S_${saveFolder}-${label}_$(basename \"$IMAGE\" .sif)\n",
    "\n",
    "### 2. VIRTUAL LIBRARY and BINARY FOLDER\n",
    "libraryFolder=/${saveFolder}/$(whoami)/libraryFolder/${IMAGE_basename}/local/lib\n",
    "libraryFolder_binding=${libraryFolder}:${HOME}/.local/lib\n",
    "binFolder=/${saveFolder}/$(whoami)/libraryFolder/${IMAGE_basename}/local/bin\n",
    "binFolder_binding=${binFolder}:${HOME}/.local/bin\n",
    "rm -rf /${saveFolder}/$(whoami)/libraryFolder/${IMAGE_basename}\n",
    "mkdir -p ${libraryFolder} ${binFolder}\n",
    "\n",
    "# 3. PIP INSTALL SLAVE IPYKERNEL\n",
    "ml libs/singularity/3.10.2\n",
    "singularity exec -B ${libraryFolder_binding} -B ${binFolder_binding} ${IMAGE} pip install -q ipykernel IProgress ipywidgets\n",
    "\n",
    "# 4. IPYKERNEL for IMAGE\n",
    "IPYKERNEL=/work/u00cjz00/slurm_jobs/ipykernel/t2/image_with_ipykernel_local\n",
    "mkdir -p ${HOME}/.local/share/jupyter/kernels/\n",
    "rm -rf ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}\n",
    "cp -rf ${IPYKERNEL} ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}\n",
    "chmod -R 755 ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}\n",
    "IMAGE_desc=$(echo $IMAGE | sed 's_/_\\\\/_g')\n",
    "sed -i \"s/templateSIF/${IMAGE_desc}/g\" ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "sed -i \"s/templateImage/Image_${IMAGE_basename}/g\" ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "sed -i \"s@templateLibrayFolder@${libraryFolder_binding}@g\" ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "sed -i \"s@templateBinFolder@${binFolder_binding}@g\" ${HOME}/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "\n",
    "# 5. check size\n",
    "du -sh ${libraryFolder}\n",
    "echo /home/$(whoami)/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "cat /home/$(whoami)/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "\n",
    "cmd=\"/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec --nv --cleanenv \\\n",
    "-B /work -B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin \\\n",
    "-B ${libraryFolder}:/home/g00cjz00/.local/lib \\\n",
    "-B ${binFolder}:/home/g00cjz00/.local/bin \\\n",
    "${IMAGE} \\\n",
    "bash -c 'export PATH=\\$PATH:\\$HOME/.local/bin; echo \\$PATH;'\n",
    "\"\n",
    "\n",
    "echo ${cmd}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b70bb551-62fb-445a-bde8-a3eb1bd99d43",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "### 0. pipi install libsray savefolder\n",
    "saveFolder=work\n",
    "label=genai01\n",
    "### 1. IMAGE\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.7.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.8.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.9.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.10.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.11.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/pytorch_2.1.0-cuda11.8-cudnn8-devel.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/cjz_images/pytorch_2.1.0-cuda11.8-cudnn8-runtime.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/cjz_images/pytorch_2.1.0-cuda11.8-cudnn8-runtime_textgen.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/stable-diffusion-webui_full.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/pytorch_2.1.0-cuda11.8-cudnn8-devel.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/pytorch_22.12-py3.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/text-generation-inference_1.1.1.sif\n",
    "IMAGE_basename=S_${saveFolder}-${label}_$(basename \"$IMAGE\" .sif)\n",
    "\n",
    "\n",
    "### 2. VIRTUAL LIBRARY and BINARY FOLDER\n",
    "libraryFolder=/${saveFolder}/$(whoami)/libraryFolder/${IMAGE_basename}/local/lib\n",
    "du -sh ${libraryFolder}\n",
    "echo /home/$(whoami)/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json\n",
    "cat /home/$(whoami)/.local/share/jupyter/kernels/${IMAGE_basename}/kernel.json"
   ]
  },
  {
   "cell_type": "raw",
   "id": "82af2721-52e7-4c05-8db3-c7ab8f8148f2",
   "metadata": {
    "tags": []
   },
   "source": [
    "%%bash\n",
    "### 0. pipi install libsray savefolder\n",
    "saveFolder=work\n",
    "label=genai01\n",
    "### 1. IMAGE\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.7.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.8.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.9.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.10.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/python_IMAGE/python_3.11.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/pytorch_2.1.0-cuda11.8-cudnn8-devel.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/cjz_images/pytorch_2.1.0-cuda11.8-cudnn8-runtime.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/cjz_images/pytorch_2.1.0-cuda11.8-cudnn8-runtime_textgen.sif\n",
    "#IMAGE=/work/u00cjz00/nvidia/stable-diffusion-webui_full.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/pytorch_2.1.0-cuda11.8-cudnn8-devel.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/pytorch_22.12-py3.sif\n",
    "IMAGE=/work/u00cjz00/nvidia/text-generation-inference_1.1.1.sif\n",
    "IMAGE_basename=S_${saveFolder}-${label}_$(basename \"$IMAGE\" .sif)\n",
    "\n",
    "### 2. VIRTUAL LIBRARY and BINARY FOLDER\n",
    "libraryFolder=/${saveFolder}/$(whoami)/libraryFolder/${IMAGE_basename}\n",
    "kernelFolder=${HOME}/.local/share/jupyter/kernels/${IMAGE_basename} \n",
    "### 3. REMOVE\n",
    "rm -rf ${libraryFolder}\n",
    "rm -rf ${kernelFolder} \n",
    "echo  ${kernelFolder} "
   ]
  },
  {
   "cell_type": "raw",
   "id": "f24fbe3b-f064-4dfe-85d9-74e17c07b8b9",
   "metadata": {},
   "source": [
    "# Link\n",
    "!ln -s /work/$(whoami)/libraryFolder /home/$(whoami)/libraryFolder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c21e8f86-f847-4a15-945b-1a467e7dacbc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}