OMPP: Moved copy file directives from start-custom.sh to start-oms.sh (…

…#553)

* Moved copy file directives from start-custom.sh to start-oms.sh

* Ran make to update dockerfiles and outputs.

* Changes to start-oms.sh. Create and set default model log directory. Download scripts and templates needed to run mpi jobs via kubeflow.

* Added commands in start-oms.sh to persist some environment variables in /etc/openmpp for use with parseCommand.py

* Forgot to run make on previous changes.

* Removed mkdir and cd commands and added chmod +x command in start-oms.s

* Forgot to run make again.

* fix(oms): fix references in shell script

* fix(start-custom): force removal of tmp rprofile

* fix(start-oms): fix git commands, allow support for localhost

* Move mpi enabled model binary of RiskPaths into model binaries directory.

* Forgot to run make.

* Adding several options to the command used to start the openm web service. This should make the web service write to its own log file. Should help me diagnose the hung server errors that I was getting in the UI.

* I keep forgetting to run make.

* Changed openmpp repo reference in start-oms.sh to get the main branch instead of openmpp-13.

* Forgot to run make.

* Added a remove file command to start-oms.sh to remove incorrect golang template in openm etc directory.

---------

Co-authored-by: jacekzbigniewdudek <[email protected]>
Co-authored-by: souheil-yazji <[email protected]>

output/docker-stacks-datascience-notebook/start-custom.sh

@@ -16,7 +16,7 @@ fi
 test -z "$GIT_EXAMPLE_NOTEBOOKS" || git clone "$GIT_EXAMPLE_NOTEBOOKS"
 
 if [ ! -e /home/$NB_USER/.Rprofile ]; then
-    cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm /tmp/.Rprofile
+    cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm -rf /tmp/.Rprofile
 fi
 
 # Configure the shell! If not already configured.
@@ -151,14 +151,6 @@ if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then
   conda config --add channels https://jfrog.aaw.cloud.statcan.ca/artifactory/api/conda/conda-pytorch-remote 
 fi
 
-# Copy default ompp models on first start up
-export OMS_MODELS_DIR="/home/jovyan/models"
-if [ ! -d "$OMS_MODELS_DIR" ]; then
-  echo "Creating ompp default model directory"
-  mkdir -p "$OMS_MODELS_DIR"
-  cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODELS_DIR"
-fi
-
 echo "--------------------starting jupyter--------------------"
 
 /opt/conda/bin/jupyter server --notebook-dir=/home/${NB_USER} \
@@ -172,4 +164,4 @@ echo "--------------------starting jupyter--------------------"
                  --ServerApp.base_url=${NB_PREFIX} \
                  --ServerApp.default_url=${DEFAULT_JUPYTER_URL:-/tree}
 
-echo "--------------------shutting down, persisting VS_CODE settings--------------------"
+echo "--------------------shutting down, persisting VS_CODE settings--------------------"

output/docker-stacks-datascience-notebook/start-oms.sh

@@ -33,47 +33,94 @@ OM_ROOT="$OMPP_INSTALL_DIR"
 [ "$OM_ROOT" != "$PWD" ] && pushd $OM_ROOT
 
 # allow to use $MODEL_NAME.ini file in UI for model run
-#
 export OM_CFG_INI_ALLOW=true
 export OM_CFG_INI_ANY_KEY=true
 export OMS_URL=${JUPYTER_SERVER_URL}ompp
 
 # OpenM++ default configuraton
-if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then
-  #DEV
+if [ "$KUBERNETES_SERVICE_HOST" =~ ".131." ] || [ -z $KUBERNETES_SERVICE_HOST ]; then
+  #DEV or Localhost
   export OMS_MODEL_DIR=/home/jovyan/models
+  export OMS_LOG_DIR=/home/jovyan/logs
   export OMS_HOME_DIR=/home/jovyan/
 else
   if [ -d "/etc/protb" ]; then
     export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/models
+    export OMS_LOG_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/logs
     export OMS_HOME_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/
   else
     export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/models
+    export OMS_LOG_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/logs
     export OMS_HOME_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/
   fi
 fi
 
-# start oms web-service
-#
+# Create models directory if it doesn't exist:
+if [ ! -d "$OMS_MODEL_DIR" ]; then
+  mkdir -p "$OMS_MODEL_DIR"
+fi
+
+# Create model log directory if it doesn't exist:
+if [ ! -d "$OMS_LOG_DIR" ]; then
+  mkdir -p "$OMS_LOG_DIR"
+fi
+
+# Copy sample models from openmpp installation archive into models directory:
+cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODEL_DIR"
+
+# These three environment variables don't persist so let's try using a file:
+echo "$OMS_HOME_DIR" > $OM_ROOT/etc/oms_home_dir 
+echo "$OMS_MODEL_DIR" > $OM_ROOT/etc/oms_model_dir 
+echo "$OMS_LOG_DIR" > $OM_ROOT/etc/oms_log_dir
+
+
+# Import openmpp repo to get scripts and templates needed to run mpi jobs via kubeflow:
+if [ ! -d /openmpp ]
+ then
+  git clone https://github.com/StatCan/openmpp.git
+fi
+cd openmpp
+branch="main"
+state=$(git symbolic-ref --short HEAD 2>&1)
+if [ $state != $branch ]
+ then
+  git checkout $branch
+fi 
+git pull
+cd mpi-job-files
+
+# Copy scripts and templates into openmpp installation bin and etc folders:
+cp dispatchMPIJob.sh parseCommand.py "$OM_ROOT/bin/"
+cp mpi.kubeflow.template.txt MPIJobTemplate.yaml "$OM_ROOT/etc/"
+
+# Delete the default mpi golang template that does not work in our context:
+rm -f "$OM_ROOT/etc/mpi.ModelRun.template.txt"
+
+# Copy the mpi enabled model binary to the model binaries directory:
+cp RiskPaths_mpi "$OMS_MODEL_DIR/bin"
+
+# Making sure these can execute:
+chmod +x dispatchMPIJob.sh parseCommand.py
+
+# Remove repo as it's not needed anymore:
+cd "$OM_ROOT" && rm -rf openmpp
+
+# Output various oms settings to console:
 [ -z "$OMS_PORT" ] && OMS_PORT=4040
 
 echo "OM_ROOT=$OM_ROOT"
 echo "OMS_PORT=$OMS_PORT"
 echo "OMS_URL=$OMS_URL"
 
 echo "OMS_MODEL_DIR=$OMS_MODEL_DIR"
-if [ ! -d $OMS_MODEL_DIR ]; then
-  mkdir -p $OMS_MODEL_DIR
-fi
-
 echo "OMS_HOME_DIR=$OMS_HOME_DIR"
-if [ ! -d $OMS_HOME_DIR ]; then
-  mkdir -p $OMS_HOME_DIR
-fi
+echo "OMS_LOG_DIR=$OMS_LOG_DIR"
 
-OM_ROOT=$OM_ROOT ./bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest
-status=$?
 
+# start oms web-service:
+OM_ROOT=$OM_ROOT ${OM_ROOT}/bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.ModelLogDir ${OMS_LOG_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -OpenM.LogToFile -OpenM.LogUseDailyStamp -OpenM.LogFilePath ${OM_ROOT}/log/oms.log
+
+status=$?
 if [ $status -ne 0 ] ;
 then
   [ $status -eq 130 ] && echo " oms web-service terminated by Ctrl+C"

output/jupyterlab-cpu/start-custom.sh

@@ -16,7 +16,7 @@ fi
 test -z "$GIT_EXAMPLE_NOTEBOOKS" || git clone "$GIT_EXAMPLE_NOTEBOOKS"
 
 if [ ! -e /home/$NB_USER/.Rprofile ]; then
-    cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm /tmp/.Rprofile
+    cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm -rf /tmp/.Rprofile
 fi
 
 # Configure the shell! If not already configured.
@@ -151,14 +151,6 @@ if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then
   conda config --add channels https://jfrog.aaw.cloud.statcan.ca/artifactory/api/conda/conda-pytorch-remote 
 fi
 
-# Copy default ompp models on first start up
-export OMS_MODELS_DIR="/home/jovyan/models"
-if [ ! -d "$OMS_MODELS_DIR" ]; then
-  echo "Creating ompp default model directory"
-  mkdir -p "$OMS_MODELS_DIR"
-  cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODELS_DIR"
-fi
-
 echo "--------------------starting jupyter--------------------"
 
 /opt/conda/bin/jupyter server --notebook-dir=/home/${NB_USER} \
@@ -172,4 +164,4 @@ echo "--------------------starting jupyter--------------------"
                  --ServerApp.base_url=${NB_PREFIX} \
                  --ServerApp.default_url=${DEFAULT_JUPYTER_URL:-/tree}
 
-echo "--------------------shutting down, persisting VS_CODE settings--------------------"
+echo "--------------------shutting down, persisting VS_CODE settings--------------------"

output/jupyterlab-cpu/start-oms.sh

@@ -33,47 +33,94 @@ OM_ROOT="$OMPP_INSTALL_DIR"
 [ "$OM_ROOT" != "$PWD" ] && pushd $OM_ROOT
 
 # allow to use $MODEL_NAME.ini file in UI for model run
-#
 export OM_CFG_INI_ALLOW=true
 export OM_CFG_INI_ANY_KEY=true
 export OMS_URL=${JUPYTER_SERVER_URL}ompp
 
 # OpenM++ default configuraton
-if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then
-  #DEV
+if [ "$KUBERNETES_SERVICE_HOST" =~ ".131." ] || [ -z $KUBERNETES_SERVICE_HOST ]; then
+  #DEV or Localhost
   export OMS_MODEL_DIR=/home/jovyan/models
+  export OMS_LOG_DIR=/home/jovyan/logs
   export OMS_HOME_DIR=/home/jovyan/
 else
   if [ -d "/etc/protb" ]; then
     export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/models
+    export OMS_LOG_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/logs
     export OMS_HOME_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/
   else
     export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/models
+    export OMS_LOG_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/logs
     export OMS_HOME_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/
   fi
 fi
 
-# start oms web-service
-#
+# Create models directory if it doesn't exist:
+if [ ! -d "$OMS_MODEL_DIR" ]; then
+  mkdir -p "$OMS_MODEL_DIR"
+fi
+
+# Create model log directory if it doesn't exist:
+if [ ! -d "$OMS_LOG_DIR" ]; then
+  mkdir -p "$OMS_LOG_DIR"
+fi
+
+# Copy sample models from openmpp installation archive into models directory:
+cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODEL_DIR"
+
+# These three environment variables don't persist so let's try using a file:
+echo "$OMS_HOME_DIR" > $OM_ROOT/etc/oms_home_dir 
+echo "$OMS_MODEL_DIR" > $OM_ROOT/etc/oms_model_dir 
+echo "$OMS_LOG_DIR" > $OM_ROOT/etc/oms_log_dir
+
+
+# Import openmpp repo to get scripts and templates needed to run mpi jobs via kubeflow:
+if [ ! -d /openmpp ]
+ then
+  git clone https://github.com/StatCan/openmpp.git
+fi
+cd openmpp
+branch="main"
+state=$(git symbolic-ref --short HEAD 2>&1)
+if [ $state != $branch ]
+ then
+  git checkout $branch
+fi 
+git pull
+cd mpi-job-files
+
+# Copy scripts and templates into openmpp installation bin and etc folders:
+cp dispatchMPIJob.sh parseCommand.py "$OM_ROOT/bin/"
+cp mpi.kubeflow.template.txt MPIJobTemplate.yaml "$OM_ROOT/etc/"
+
+# Delete the default mpi golang template that does not work in our context:
+rm -f "$OM_ROOT/etc/mpi.ModelRun.template.txt"
+
+# Copy the mpi enabled model binary to the model binaries directory:
+cp RiskPaths_mpi "$OMS_MODEL_DIR/bin"
+
+# Making sure these can execute:
+chmod +x dispatchMPIJob.sh parseCommand.py
+
+# Remove repo as it's not needed anymore:
+cd "$OM_ROOT" && rm -rf openmpp
+
+# Output various oms settings to console:
 [ -z "$OMS_PORT" ] && OMS_PORT=4040
 
 echo "OM_ROOT=$OM_ROOT"
 echo "OMS_PORT=$OMS_PORT"
 echo "OMS_URL=$OMS_URL"
 
 echo "OMS_MODEL_DIR=$OMS_MODEL_DIR"
-if [ ! -d $OMS_MODEL_DIR ]; then
-  mkdir -p $OMS_MODEL_DIR
-fi
-
 echo "OMS_HOME_DIR=$OMS_HOME_DIR"
-if [ ! -d $OMS_HOME_DIR ]; then
-  mkdir -p $OMS_HOME_DIR
-fi
+echo "OMS_LOG_DIR=$OMS_LOG_DIR"
 
-OM_ROOT=$OM_ROOT ./bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest
-status=$?
 
+# start oms web-service:
+OM_ROOT=$OM_ROOT ${OM_ROOT}/bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.ModelLogDir ${OMS_LOG_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -OpenM.LogToFile -OpenM.LogUseDailyStamp -OpenM.LogFilePath ${OM_ROOT}/log/oms.log
+
+status=$?
 if [ $status -ne 0 ] ;
 then
   [ $status -eq 130 ] && echo " oms web-service terminated by Ctrl+C"

output/jupyterlab-pytorch/start-custom.sh

@@ -16,7 +16,7 @@ fi
 test -z "$GIT_EXAMPLE_NOTEBOOKS" || git clone "$GIT_EXAMPLE_NOTEBOOKS"
 
 if [ ! -e /home/$NB_USER/.Rprofile ]; then
-    cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm /tmp/.Rprofile
+    cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm -rf /tmp/.Rprofile
 fi
 
 # Configure the shell! If not already configured.
@@ -151,14 +151,6 @@ if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then
   conda config --add channels https://jfrog.aaw.cloud.statcan.ca/artifactory/api/conda/conda-pytorch-remote 
 fi
 
-# Copy default ompp models on first start up
-export OMS_MODELS_DIR="/home/jovyan/models"
-if [ ! -d "$OMS_MODELS_DIR" ]; then
-  echo "Creating ompp default model directory"
-  mkdir -p "$OMS_MODELS_DIR"
-  cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODELS_DIR"
-fi
-
 echo "--------------------starting jupyter--------------------"
 
 /opt/conda/bin/jupyter server --notebook-dir=/home/${NB_USER} \
@@ -172,4 +164,4 @@ echo "--------------------starting jupyter--------------------"
                  --ServerApp.base_url=${NB_PREFIX} \
                  --ServerApp.default_url=${DEFAULT_JUPYTER_URL:-/tree}
 
-echo "--------------------shutting down, persisting VS_CODE settings--------------------"
+echo "--------------------shutting down, persisting VS_CODE settings--------------------"