From c124909dc827cdcbf3fa4d31b09c9a67d5ab11cd Mon Sep 17 00:00:00 2001 From: Bai juyi Date: Tue, 4 Jun 2024 11:49:03 +0200 Subject: [PATCH] deployment via singularity --- 01_deploy_via_singularity.sh | 215 ++++++++++++++++++++++++++++ Singularities/Singularity_nginx.def | 13 ++ Singularities/Singularity_shiny.def | 62 ++++++++ Singularities/lib/.gitkeep | 0 test.R | 110 ++++++++++++++ 5 files changed, 400 insertions(+) create mode 100644 01_deploy_via_singularity.sh create mode 100644 Singularities/Singularity_nginx.def create mode 100644 Singularities/Singularity_shiny.def create mode 100644 Singularities/lib/.gitkeep create mode 100644 test.R diff --git a/01_deploy_via_singularity.sh b/01_deploy_via_singularity.sh new file mode 100644 index 0000000..f4f6638 --- /dev/null +++ b/01_deploy_via_singularity.sh @@ -0,0 +1,215 @@ +#!/bin/bash +source scripts/colored_print.sh +source scripts/check_and_set_email_credentials.sh +source scripts/check_CPU_number.sh + +# Function to extract port number from configuration file +extract_port() { + local file=$1 + grep -Eo "listen\s+[0-9]+" "$file" | grep -Eo "[0-9]+" +} + +# Function to format and display elapsed time +show_time() { + local elapsed_time=$1 + local hours=$((elapsed_time / 3600)) + local minutes=$(( (elapsed_time % 3600) / 60)) + local seconds=$((elapsed_time % 60)) + printf "\rElapsed time: %02d:%02d:%02d" $hours $minutes $seconds +} + +set -e +chmod -R 755 scripts/ +chmod -R 755 PMETdev/ +chmod -R 766 result/ + + +##################################### email ################################################### +print_green "Configurations of email to send results" +check_and_set_email_credentials + +################################## CPU ##################################################### +print_green "Configuration of CPU number" +check_cpu_number + + +# Definition files and image names +DEF_FILE_SHINY="Singularities/Singularity_shiny.def" +SIMG_FILE_SHINY="Singularities/shiny_server.simg" + +DEF_FILE_NGINX="Singularities/Singularity_nginx.def" +SIMG_FILE_NGINX="Singularities/nginx.simg" + +NGINX_CONF="conf/nginx.conf" +SHINY_CONF="conf/shiny-server.conf" + +echo "" + +#################################### Check definition files #################################### +# Check if Singularity definition files exist +if [ ! -f "$DEF_FILE_SHINY" ] && [ ! -f "$DEF_FILE_NGINX" ]; then + echo "Singularity definition files $DEF_FILE_SHINY and $DEF_FILE_NGINX do not exist." + exit 1 +elif [ ! -f "$DEF_FILE_SHINY" ]; then + echo "Singularity definition file $DEF_FILE_SHINY does not exist." + exit 1 +elif [ ! -f "$DEF_FILE_NGINX" ]; then + echo "Singularity definition file $DEF_FILE_NGINX does not exist." + exit 1 +fi +echo "" + +#################################### Check if simg files need to be rebuilt #################################### +FLAG_IMG_REBUILD=false + +# Check if simg files exist +if [ -f "$SIMG_FILE_SHINY" ] || [ -f "$SIMG_FILE_NGINX" ]; then + # simg files exist, ask user if they want to delete and rebuild + print_green "Existing $SIMG_FILE_SHINY and $SIMG_FILE_NGINX found." + print_orange_no_br "Do you want to delete and rebuild? (y/N): " + read -p "" -n 1 -r REPLY + if [[ $REPLY =~ ^[Yy]$ ]]; then + # Set flag to rebuild images + FLAG_IMG_REBUILD=true + fi +else + # simg files do not exist, set flag to rebuild + print_orange "simg files do not exist, rebuild images." + FLAG_IMG_REBUILD=true +fi + + +#################################### Build images #################################### +if [ "$FLAG_IMG_REBUILD" = true ]; then + #################################### + # shiny and nginx configuration files + #################################### + rm -rf conf/shiny-server.conf + rm -rf conf/nginx.conf + + print_green "\nGenerating shiny-server.conf..." + bash scripts/create_shiny_server_conf.sh start + print_green "\nGenerating nginx.conf..." + bash scripts/create_nginx_conf.sh start + + # Extract port numbers + NGINX_PORT=$(extract_port "$NGINX_CONF") + SHINY_PORT=$(extract_port "$SHINY_CONF") + + # update nginx link + echo "http://localhost:$NGINX_PORT/result/" > data/nginx_link.txt + + #################################### + # build images + #################################### + + rm -rf $SIMG_FILE_SHINY $SIMG_FILE_NGINX + + print_green "\nBuilding shiny-server image (50 min)" + # Run the singularity build command in the background + start_time=$(date +%s) + singularity build --fakeroot $SIMG_FILE_SHINY $DEF_FILE_SHINY > logs/singularity_img_build_shiny.log 2>&1 & + build_pid=$! + # While the build process is running, display the elapsed time + while kill -0 $build_pid 2> /dev/null; do + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + show_time $elapsed_time + sleep 1 + done + echo "" + + print_green "Building nginx images..." + start_time=$(date +%s) + singularity build --fakeroot $SIMG_FILE_NGINX $DEF_FILE_NGINX > logs/singularity_img_build_nginx.log 2>&1 & + build_pid=$! + + while kill -0 $build_pid 2> /dev/null; do + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + show_time $elapsed_time + sleep 1 + done + echo "" +fi + + +#################################### Check images #################################### +if [ $? -ne 0 ]; then + print_red "Singularity image build failed." + exit 1 +fi + +#################################### Check required software #################################### +REQUIRED_SOFTWARE=("fimo" "pmet" "pmetindex" "pmetParallel_linux") +MISSING_SOFTWARE=() + +for software in "${REQUIRED_SOFTWARE[@]}"; do + if [ ! -f "PMETdev/scripts/$software" ]; then + MISSING_SOFTWARE+=("$software") + fi +done + +if [ ${#MISSING_SOFTWARE[@]} -ne 0 ]; then + print_red "The following required software is missing in PMETdev/scripts:" + for software in "${MISSING_SOFTWARE[@]}"; do + echo " $software" + done + + print_orange " Compiling PMET tools..." + bash scripts/pmet_binary_compile.sh > /dev/null 2>&1 || print_red "Failed to run scripts/pmet_binary_compile.sh" + exit 1 +fi + + +#################################### Download PMET indexing data #################################### +INDEXING_DIR="data/indexing" + +# Check if there are subdirectories in the data/indexing directory +if [ -d "$INDEXING_DIR" ] && [ "$(ls -l "$INDEXING_DIR" 2>/dev/null | grep '^d' | wc -l)" -gt 0 ]; then + # print_orange "There are subdirectories in $INDEXING_DIR." + # # Directory is not empty (has subdirectories), skip download script + # echo "Skipping download script." + echo "" +else + print_red "No subdirectories in $INDEXING_DIR." + # Directory is empty (no subdirectories), run download script + print_green "Running download script..." + print_green "Downloading PMET indexing data." + bash scripts/download_pmet_indexing.sh + if [ $? -eq 0 ]; then + echo "Download script ran successfully." + else + echo "Download script failed." + exit 1 + fi +fi + + +#################################### Run images #################################### +print_green "Running shiny-server and Nginx..." +# Extract port numbers +NGINX_PORT=$(extract_port "$NGINX_CONF") +SHINY_PORT=$(extract_port "$SHINY_CONF") + +# Print the extracted port numbers +print_fluorescent_yellow "PMET link : localhost:$SHINY_PORT/pmet" +print_fluorescent_yellow "Result link: localhost:$NGINX_PORT/result" + +singularity run \ + --bind ./:/srv/shiny-server/pmet \ + --bind ./data:/srv/shiny-server/pmet/data \ + --bind ./logs:/var/log/shiny-server \ + --bind ./Singularities/lib:/var/lib/shiny-server \ + --bind ./conf/shiny-server.conf:/etc/shiny-server/shiny-server.conf \ + --bind ./test.R:/srv/shiny-server/test/app.R \ + $SIMG_FILE_SHINY > logs/singularity_running_shiny_server.log 2>&1 & + +singularity run \ + --bind ./result:/etc/nginx/html \ + $SIMG_FILE_NGINX > logs/singularity_running_nginx.log 2>&1 & + +# Wait for all background processes to complete + +print_green "\n\nYou are safe to colse this console." +wait diff --git a/Singularities/Singularity_nginx.def b/Singularities/Singularity_nginx.def new file mode 100644 index 0000000..a7c1277 --- /dev/null +++ b/Singularities/Singularity_nginx.def @@ -0,0 +1,13 @@ +Bootstrap: docker +# From:nginx:latest +From:nginxinc/nginx-unprivileged + +%files + ./conf/nginx.conf /etc/nginx/conf.d/default.conf + ./result /etc/nginx/html + +%post + mkdir -p /etc/nginx/html + +%runscript + exec nginx -g 'daemon off;' diff --git a/Singularities/Singularity_shiny.def b/Singularities/Singularity_shiny.def new file mode 100644 index 0000000..4ed43d4 --- /dev/null +++ b/Singularities/Singularity_shiny.def @@ -0,0 +1,62 @@ +Bootstrap: docker +From: rocker/shiny + +%labels +maintainer wangxuesong29@gmail.com + +%files + ./scripts/install_apt_tools.sh /opt/pmet/scripts/install_apt_tools.sh + ./scripts/install_python_libs.sh /opt/pmet/scripts/install_python_libs.sh + ./R/utils/install_packages.R /opt/pmet/R/utils/install_packages.R + +%environment + export PATH=$PATH:/opt/pmet/ + + +%post + ############################## 2. install linux libs ############################### + /opt/pmet/scripts/install_apt_tools.sh + + # ############################## 4. install R libs ############################### + # # Configure Java using R CMD javareconf + R CMD javareconf + + # # renv for package management + # Rscript -e "install.packages(c('remotes', 'littler', 'docopt', 'renv', 'jsonlite'))" + # Rscript -e "setwd('/opt/pmet');renv::restore()" + Rscript /opt/pmet/R/utils/install_packages.R + + # R -e "install.packages(c('dplyr', 'ggplot2', 'gapminder'))" + + + ############################## 5. install python libs ############################### + apt-get update && \ + apt -y install python3-pip + /opt/pmet/scripts/install_python_libs.sh + + + ########################## 6. install bedtool and samtools ########################### + apt update && apt upgrade -y + apt -y install bedtools + + # Samtools 1.3.1 # + wget https://github.com/samtools/samtools/releases/download/1.17/samtools-1.17.tar.bz2 && \ + tar -xjf samtools-1.17.tar.bz2 && \ + cd samtools-1.17 && \ + ./configure --disable-lzma --prefix=/opt/samtools && \ + make && \ + make install && \ + cd / && \ + rm -rf /tmp/samtools-1.17 /tmp/samtools-1.17.tar.bz2 + + echo "export PATH=/opt/samtools/bin:$PATH" >> /root/.bashrc + echo "export PATH=/opt/samtools/bin:$PATH" >> /home/shiny/.bashrc + + # Clean apt cache to reduce image size + apt-get clean && rm -rf /var/lib/apt/lists/* + + mkdir -p /var/log/shiny-server + chown shiny.shiny /var/log/shiny-server + +%runscript + exec shiny-server 2>&1 diff --git a/Singularities/lib/.gitkeep b/Singularities/lib/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/test.R b/test.R new file mode 100644 index 0000000..ec3692e --- /dev/null +++ b/test.R @@ -0,0 +1,110 @@ +library(shiny) +library(dplyr) +library(ggplot2) +library(gapminder) + +# Specify the application port +options(shiny.host = "0.0.0.0") +options(shiny.port = 8180) + +# ui <- fluidPage( +# sidebarLayout( +# sidebarPanel( +# tags$h4("Gapminder Dashboard"), +# tags$hr(), +# selectInput(inputId = "inContinent", label = "Continent", choices = unique(gapminder$continent), selected = "Europe"), +# tags$p(id = "currentDir", "Current Shiny Directory: "), +# tags$p(id = "libPaths", "Library Paths: ") +# ), +# mainPanel( +# plotOutput(outputId = "outChartLifeExp"), +# plotOutput(outputId = "outChartGDP"), +# actionButton(inputId = "printInfo", label = "Print Info") +# ) +# ) +# ) +ui <- fluidPage( + sidebarLayout( + sidebarPanel( + tags$h4("Gapminder Dashboard"), + tags$hr(), + selectInput(inputId = "inContinent", label = "Continent", choices = unique(gapminder$continent), selected = "Europe"), + tags$p(id = "currentDirLabel", "Current Shiny Directory: "), + tags$p(id = "currentDir", "", class = "infoText"), + tags$p(id = "libPathsLabel", "Library Paths: "), + tags$p(id = "libPaths", "", class = "infoText") + ), + mainPanel( + plotOutput(outputId = "outChartLifeExp"), + plotOutput(outputId = "outChartGDP"), + actionButton(inputId = "printInfo", label = "Print Info") + ) + ) +) + + + +server <- function(input, output, session) { + # Filter data and store as reactive value + data <- reactive({ + gapminder %>% + filter(continent == input$inContinent) %>% + group_by(year) %>% + summarise( + AvgLifeExp = round(mean(lifeExp)), + AvgGdpPercap = round(mean(gdpPercap), digits = 2) + ) + }) + + # Common properties for charts + chart_theme <- ggplot2::theme( + plot.title = element_text(hjust = 0.5, size = 20, face = "bold"), + axis.title.x = element_text(size = 15), + axis.title.y = element_text(size = 15), + axis.text.x = element_text(size = 12), + axis.text.y = element_text(size = 12) + ) + + # Render Life Exp chart + output$outChartLifeExp <- renderPlot({ + ggplot(data(), aes(x = year, y = AvgLifeExp)) + + geom_col(fill = "#0099f9") + + geom_text(aes(label = AvgLifeExp), vjust = 2, size = 6, color = "#ffffff") + + labs(title = paste("Average life expectancy in", input$inContinent)) + + theme_classic() + + chart_theme + }) + + # Render GDP chart + output$outChartGDP <- renderPlot({ + ggplot(data(), aes(x = year, y = AvgGdpPercap)) + + geom_line(color = "#f96000", size = 2) + + geom_point(color = "#f96000", size = 5) + + geom_label( + aes(label = AvgGdpPercap), + nudge_x = 0.25, + nudge_y = 0.25 + ) + + labs(title = paste("Average GDP per capita in", input$inContinent)) + + theme_classic() + + chart_theme + }) + + # Function to print current directory and library paths + printInfo <- function() { + currentDir <- getwd() + libPaths <- .libPaths() + updateTextInput(session, "currentDir", value = paste("Current Shiny Directory: ", currentDir)) + updateTextInput(session, "libPaths", value = paste("Library Paths: ", paste(libPaths, collapse = ", "))) + } + + # Observe the action button and update text when clicked + observeEvent(input$printInfo, { + printInfo() + }) +} + +# shinyApp(ui = ui, server = server) + + +shinyApp(ui = ui, server = server) \ No newline at end of file