Merge pull request #74 from duocang/modulization

bug fix of docker deployment

01_deploy_via_Docker.sh

@@ -3,7 +3,24 @@
 source scripts/colored_print.sh
 source scripts/check_and_set_email_credentials.sh
 source scripts/check_CPU_number.sh
+
+# Function to extract port number from configuration file
+extract_port() {
+    local file=$1
+    grep -Eo "listen\s+[0-9]+" "$file" | grep -Eo "[0-9]+"
+}
+
+# Function to format and display elapsed time
+show_time() {
+    local elapsed_time=$1
+    local hours=$((elapsed_time / 3600))
+    local minutes=$(( (elapsed_time % 3600) / 60))
+    local seconds=$((elapsed_time % 60))
+    printf "\rElapsed time: %02d:%02d:%02d" $hours $minutes $seconds
+}
+
 set -e
+chmod -R 766 data result
 
 #################### 1 email
 print_green "1. Configurations of email to send results"
@@ -13,7 +30,56 @@ check_and_set_email_credentials
 print_green "2. Configuration of CPU number"
 check_cpu_number
 
-#################### 3. Docker build
-print_green "3. Docker building"
-chmod  777 logs result
-docker-compose up -d
+
+#################### 3. Update shiny and nginx configuration files
+print_green "3. Update shiny and nginx configuration files"
+
+SHINY_CONF="conf/shiny-server.conf"
+NGINX_CONF="conf/nginx.conf"
+
+rm -rf $SHINY_CONF
+rm -rf $NGINX_CONF
+
+print_green "Generating shiny-server.conf..."
+bash scripts/create_shiny_server_conf.sh start
+print_green "\nGenerating nginx.conf..."
+bash scripts/create_nginx_conf.sh start
+
+# remove line with run_as
+sed -i '/^run_as/d' $SHINY_CONF
+
+# Extract port numbers
+NGINX_PORT=$(extract_port "$NGINX_CONF")
+SHINY_PORT=$(extract_port "$SHINY_CONF")
+
+# update nginx link
+echo "http://localhost:$NGINX_PORT/result/" > data/nginx_link.txt
+
+#################### 4. Update dockerfiles and docker-compose.yml
+print_green "4. Update dockerfiles and docker-compose.yml"
+
+# update dockerfiles
+sed -i "s|EXPOSE [0-9]*|EXPOSE $NGINX_PORT|" dockerfiles/Dockerfile.nginx
+sed -i "s|EXPOSE [0-9]*|EXPOSE $SHINY_PORT|" dockerfiles/Dockerfile.shiny
+
+# update docker-compose.yml
+bash scripts/create_docker_compose.sh $SHINY_PORT $NGINX_PORT
+
+#################### 5. Docker build
+print_green "5. Docker building"
+
+start_time=$(date +%s)
+
+docker-compose up -d > logs/docker_img_build.log 2>&1 &
+
+build_pid=$!
+# While the build process is running, display the elapsed time
+while kill -0 $build_pid 2> /dev/null; do
+    current_time=$(date +%s)
+    elapsed_time=$((current_time - start_time))
+    show_time $elapsed_time
+    sleep 1
+done
+
+print_green "\n\nYou are safe to colse this console."
+wait

R/app.R

@@ -1,5 +1,5 @@
-Sys.setenv(R_LIBS_USER = "/usr/local/lib/R/library")
-.libPaths(new = "/usr/local/lib/R/library")
+# Sys.setenv(R_LIBS_USER = "/usr/local/lib/R/library")
+# .libPaths(new = "/usr/local/lib/R/library")
 
 source("R/global.R")
 

R/global.R

@@ -1,5 +1,5 @@
-Sys.setenv(R_LIBS_USER = "/usr/local/lib/R/library")
-.libPaths(new = "/usr/local/lib/R/library")
+# Sys.setenv(R_LIBS_USER = "/usr/local/lib/R/library")
+# .libPaths(new = "/usr/local/lib/R/library")
 
 options(shiny.maxRequestSize = 30000 * 1024^2)
 

R/utils/send_mail.R

@@ -39,6 +39,9 @@
 
 # SendResultMail(recipient = recipient, result_link = result_link)
 
+library(dplyr)
+library(emayili)
+
 SendResultMail <- function(recipient = NULL, result_link = NULL) {
 
   email_credential <- readLines("data/email_credential.txt")

app.R

@@ -1,5 +1,5 @@
-Sys.setenv(R_LIBS_USER = "/usr/local/lib/R/library")
-.libPaths(new = "/usr/local/lib/R/library")
+# Sys.setenv(R_LIBS_USER = "/usr/local/lib/R/library")
+# .libPaths(new = "/usr/local/lib/R/library")
 
 source("R/app.R")
 

archive/test_nginx.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+source scripts/colored_print.sh
+
+rm -rf conf/nginx_singularity.conf
+
+print_green "生成nginx_singularity.conf..."
+bash scripts/create_nginx_conf_singularity.sh start
+
+# 定义文件和镜像名称
+DEF_FILE="Singularities/Singularity_nginx.def"
+SIMG_FILE="Singularities/nginx.simg"
+
+rm -f "$SIMG_FILE"
+print_orange "已删除 $SIMG_FILE 文件，并重现构建镜像。"
+singularity build --fakeroot $SIMG_FILE $DEF_FILE
+
+#################################### 运行镜像 ####################################
+singularity run                           \
+    --bind ./result:/etc/nginx/html \
+    $SIMG_FILE

conf/nginx.conf

@@ -1,12 +1,12 @@
 server {
-    listen       84;
-    listen  [::]:84;
+    listen       19258;
+    listen  [::]:19258;
     server_name  localhost_pmet;
 
-    # access_log  /var/log/nginx/localhost.access.log;
+    # access_log  /var/log/nginx/localhost_pmet.access.log;
 
     location /result {
         # autoindex on;
-        alias /usr/share/nginx/pmet;
+        alias /etc/nginx/html;
     }
 }

conf/shiny-server.conf

@@ -1,13 +1,10 @@
-# Instruct Shiny Server to run applications as the user "shiny"
-run_as shiny;
-
-# Define a server that listens on port 3838
+run_as wang;
 server {
-  listen 3838;
+  listen 4615;
 
   # Define a location at the base URL
   location / {
-    app_idle_timeout 0;
+
     # Host the directory of Shiny Apps stored in this directory
     site_dir /srv/shiny-server;
 
@@ -17,5 +14,8 @@ server {
     # When a user visits the base URL rather than a particular application,
     # an index of the applications available in this directory will be shown.
     directory_index on;
+
+    # keep shiny app alive
+    app_idle_timeout 0;
   }
 }

data/nginx_link.txt

@@ -1 +1 @@
-https://bar.utoronto.ca/pmet_result/
+http://localhost:84/result/

docker-compose.yml

@@ -5,28 +5,31 @@ services:
     build:
       context: .
       dockerfile: dockerfiles/Dockerfile.shiny
+    container_name: shiny
+    user: "1000:1000"
     volumes:
-      - ./result:/home/shiny/PMET_docker/result
-      - ./logs:/var/log/shiny-server
+      # - ./data/indexing:/home/shiny/PMET_docker/data/indexing
+      # - ./result:/home/shiny/PMET_docker/result
+      # - ./logs:/var/log/shiny-server
+      - ./result:/srv/shiny-server/pmet/result
     ports:
-      - "3838:3838"
+      - "4615:4615"
     networks:
       - app-network
 
   nginx-server:
     build:
       context: .
       dockerfile: dockerfiles/Dockerfile.nginx
-    container_name: test_nginx
+    container_name: nginx
     volumes:
-      - ./result:/usr/share/nginx/pmet
+      - ./result:/etc/nginx/html
     ports:
-      - "84:84"
+      - "19258:19258"
     restart: always
     networks:
       - app-network
 
-
 volumes:
   logs:
   result:

dockerfiles/Dockerfile.nginx

@@ -2,13 +2,13 @@
 FROM nginx:latest
 
 # 创建结果目录
-RUN mkdir -p /usr/share/nginx/pmet && \
-    chmod -R 777 /usr/share/nginx/pmet
+RUN mkdir -p /etc/nginx/html && \
+    chmod -R 777 /etc/nginx/html
 
 COPY ./conf/nginx.conf /etc/nginx/conf.d/pmet.conf
 
 # Expose Nginx's default port
-EXPOSE 84
+EXPOSE 19258
 
 # Start Nginx service
 CMD ["nginx", "-g", "daemon off;"]

dockerfiles/Dockerfile.shiny

@@ -1,11 +1,17 @@
 FROM rocker/shiny:4.3.3
 
 ############################## 1. shiny setting ###############################
+# 添加 shiny 用户到 sudo 组
+RUN usermod -aG sudo shiny
+
 COPY ./conf/shiny-server.conf /etc/shiny-server/shiny-server.conf
 
 ENV work_dir="/home/shiny/PMET_docker"
+WORKDIR $work_dir
 COPY . $work_dir
 
+RUN chown -R shiny:shiny $work_dir  && \
+    chmod -R 777         $work_dir
 RUN ln -sf $work_dir /srv/shiny-server/pmet
 
 # Use the find command to find all .sh, .r, .py, .pl files and grant them 755 permissions
@@ -17,28 +23,31 @@ RUN $work_dir/scripts/install_apt_tools.sh
 
 
 # ############################## 3. R libs settings ###############################
-ENV R_LIBS_USER=/usr/local/lib/R/library
-# Set the directory owner and permissions
-RUN mkdir -p     $R_LIBS_USER      && \
-    chmod -R 777 $R_LIBS_USER
-# Set environment variables in respective user profiles
-RUN echo "export R_LIBS_USER='$R_LIBS_USER'" >> /root/.bashrc         && \
-    echo "export R_LIBS_USER='$R_LIBS_USER'" >> /home/shiny/.bashrc   && \
-    echo "R_LIBS_USER='$R_LIBS_USER'"        >> /root/.Renviron       && \
-    echo "R_LIBS_USER='$R_LIBS_USER'"        >> /home/shiny/.Renviron
+# ENV R_LIBS_USER=/usr/local/lib/R/library
+
+# RUN chown -R shiny:shiny $R_LIBS_USER && \
+#     chmod -R 755 $R_LIBS_USER
+# # # Set the directory owner and permissions
+# # RUN mkdir -p     $R_LIBS_USER      && \
+# #     chmod -R 777 $R_LIBS_USER
+# # # Set environment variables in respective user profiles
+# # RUN echo "export R_LIBS_USER='$R_LIBS_USER'" >> /root/.bashrc         && \
+# #     echo "export R_LIBS_USER='$R_LIBS_USER'" >> /home/shiny/.bashrc   && \
+# #     echo "R_LIBS_USER='$R_LIBS_USER'"        >> /root/.Renviron       && \
+# #     echo "R_LIBS_USER='$R_LIBS_USER'"        >> /home/shiny/.Renviron
 
 
 ############################## 4. install R libs ###############################
-# # Configure Java using R CMD javareconf
+# Configure Java using R CMD javareconf
 RUN sudo R CMD javareconf
 
 # # renv for package management
 # RUN Rscript -e "install.packages(c('remotes', 'littler', 'docopt', 'renv', 'jsonlite'))"
-# RUN Rscript -e "renv::restore()"
+# RUN Rscript -e "Sys.setenv(R_INSTALL_STAGED = FALSE);renv::restore();renv::install('jsonlite')"
 RUN Rscript  $work_dir/R/utils/install_packages.R
 
-RUN rsync -av --ignore-existing /usr/local/lib/R/library/ /usr/local/lib/R/site-library/ && \
-    rsync -av --ignore-existing /usr/local/lib/R/site-library/ /usr/local/lib/R/library/
+# RUN rsync -av --ignore-existing /usr/local/lib/R/library/ /usr/local/lib/R/site-library/ && \
+#     rsync -av --ignore-existing /usr/local/lib/R/site-library/ /usr/local/lib/R/library/
 
 
 ############################## 5. install python libs ###############################
@@ -51,18 +60,18 @@ RUN $work_dir/scripts/install_python_libs.sh
 RUN apt -y install bedtools
 
 # Samtools 1.3.1 #
-ENV SAMTOOLS_INSTALL_DIR=/opt/samtools
-ENV HTSLIB_INSTALL_DIR=/opt/htslib
-
 WORKDIR /tmp
 RUN wget https://github.com/samtools/samtools/releases/download/1.17/samtools-1.17.tar.bz2 && \
     tar -xjf samtools-1.17.tar.bz2 && \
     cd samtools-1.17 && \
-    ./configure --disable-lzma --prefix=$SAMTOOLS_INSTALL_DIR && \
+    ./configure --disable-lzma --prefix=/opt/samtools && \
     make && \
     make install && \
     cd / && \
     rm -rf /tmp/samtools-1.17 /tmp/samtools-1.17.tar.bz2
+
+RUN echo "export PATH=/opt/samtools/bin:$PATH" >> /root/.bashrc        && \
+    echo "export PATH=/opt/samtools/bin:$PATH" >> /home/shiny/.bashrc
 # Set the working directory back to the root directory
 WORKDIR $work_dir
 
@@ -104,6 +113,11 @@ RUN cd $work_dir/PMETdev/src/pmet \
 ############################# 8. download homotypic data ##############################
 RUN $work_dir/scripts/download_pmet_indexing.sh
 
+# RUN chmod -R 777  $work_dir/data $work_dir/result
+
+
+# Clean apt cache to reduce image size
+RUN apt-get clean && rm -rf /var/lib/apt/lists/*
 
-EXPOSE 3838
+EXPOSE 4615
 CMD ["/usr/bin/shiny-server"]

scripts/create_docker_compose.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# 检查是否提供了两个参数
+if [ "$#" -ne 2 ]; then
+    echo "Usage: $0 <SHINY_CONF> <NGINX_CONF>"
+    exit 1
+fi
+
+SHINY_CONF=$1
+NGINX_CONF=$2
+
+# 生成 docker-compose.yml 文件
+cat <<EOF > docker-compose.yml
+x-yml-version: '3.8'
+
+services:
+  shiny-app:
+    build:
+      context: .
+      dockerfile: dockerfiles/Dockerfile.shiny
+    container_name: shiny
+    user: "$(id -u):$(id -g)"
+    volumes:
+      # - ./data/indexing:/home/shiny/PMET_docker/data/indexing
+      # - ./result:/home/shiny/PMET_docker/result
+      # - ./logs:/var/log/shiny-server
+      - ./result:/srv/shiny-server/pmet/result
+    ports:
+      - "$SHINY_CONF:$SHINY_CONF"
+    networks:
+      - app-network
+
+  nginx-server:
+    build:
+      context: .
+      dockerfile: dockerfiles/Dockerfile.nginx
+    container_name: nginx
+    volumes:
+      - ./result:/etc/nginx/html
+    ports:
+      - "$NGINX_CONF:$NGINX_CONF"
+    restart: always
+    networks:
+      - app-network
+
+volumes:
+  logs:
+  result:
+
+networks:
+  app-network:
+EOF