diff --git a/output/docker-stacks-datascience-notebook/start-custom.sh b/output/docker-stacks-datascience-notebook/start-custom.sh index 3bdce8eb5..d9c859ebe 100755 --- a/output/docker-stacks-datascience-notebook/start-custom.sh +++ b/output/docker-stacks-datascience-notebook/start-custom.sh @@ -16,7 +16,7 @@ fi test -z "$GIT_EXAMPLE_NOTEBOOKS" || git clone "$GIT_EXAMPLE_NOTEBOOKS" if [ ! -e /home/$NB_USER/.Rprofile ]; then - cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm /tmp/.Rprofile + cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm -rf /tmp/.Rprofile fi # Configure the shell! If not already configured. @@ -151,14 +151,6 @@ if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then conda config --add channels https://jfrog.aaw.cloud.statcan.ca/artifactory/api/conda/conda-pytorch-remote fi -# Copy default ompp models on first start up -export OMS_MODELS_DIR="/home/jovyan/models" -if [ ! -d "$OMS_MODELS_DIR" ]; then - echo "Creating ompp default model directory" - mkdir -p "$OMS_MODELS_DIR" - cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODELS_DIR" -fi - echo "--------------------starting jupyter--------------------" /opt/conda/bin/jupyter server --notebook-dir=/home/${NB_USER} \ @@ -172,4 +164,4 @@ echo "--------------------starting jupyter--------------------" --ServerApp.base_url=${NB_PREFIX} \ --ServerApp.default_url=${DEFAULT_JUPYTER_URL:-/tree} -echo "--------------------shutting down, persisting VS_CODE settings--------------------" \ No newline at end of file +echo "--------------------shutting down, persisting VS_CODE settings--------------------" diff --git a/output/docker-stacks-datascience-notebook/start-oms.sh b/output/docker-stacks-datascience-notebook/start-oms.sh index f6632b8b1..e825a466e 100644 --- a/output/docker-stacks-datascience-notebook/start-oms.sh +++ b/output/docker-stacks-datascience-notebook/start-oms.sh @@ -33,28 +33,79 @@ OM_ROOT="$OMPP_INSTALL_DIR" [ "$OM_ROOT" != "$PWD" ] && pushd $OM_ROOT # allow to use $MODEL_NAME.ini file in UI for model run -# export OM_CFG_INI_ALLOW=true export OM_CFG_INI_ANY_KEY=true export OMS_URL=${JUPYTER_SERVER_URL}ompp # OpenM++ default configuraton -if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then - #DEV +if [ "$KUBERNETES_SERVICE_HOST" =~ ".131." ] || [ -z $KUBERNETES_SERVICE_HOST ]; then + #DEV or Localhost export OMS_MODEL_DIR=/home/jovyan/models + export OMS_LOG_DIR=/home/jovyan/logs export OMS_HOME_DIR=/home/jovyan/ else if [ -d "/etc/protb" ]; then export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/ else export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/ fi fi -# start oms web-service -# +# Create models directory if it doesn't exist: +if [ ! -d "$OMS_MODEL_DIR" ]; then + mkdir -p "$OMS_MODEL_DIR" +fi + +# Create model log directory if it doesn't exist: +if [ ! -d "$OMS_LOG_DIR" ]; then + mkdir -p "$OMS_LOG_DIR" +fi + +# Copy sample models from openmpp installation archive into models directory: +cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODEL_DIR" + +# These three environment variables don't persist so let's try using a file: +echo "$OMS_HOME_DIR" > $OM_ROOT/etc/oms_home_dir +echo "$OMS_MODEL_DIR" > $OM_ROOT/etc/oms_model_dir +echo "$OMS_LOG_DIR" > $OM_ROOT/etc/oms_log_dir + + +# Import openmpp repo to get scripts and templates needed to run mpi jobs via kubeflow: +if [ ! -d /openmpp ] + then + git clone https://github.com/StatCan/openmpp.git +fi +cd openmpp +branch="main" +state=$(git symbolic-ref --short HEAD 2>&1) +if [ $state != $branch ] + then + git checkout $branch +fi +git pull +cd mpi-job-files + +# Copy scripts and templates into openmpp installation bin and etc folders: +cp dispatchMPIJob.sh parseCommand.py "$OM_ROOT/bin/" +cp mpi.kubeflow.template.txt MPIJobTemplate.yaml "$OM_ROOT/etc/" + +# Delete the default mpi golang template that does not work in our context: +rm -f "$OM_ROOT/etc/mpi.ModelRun.template.txt" + +# Copy the mpi enabled model binary to the model binaries directory: +cp RiskPaths_mpi "$OMS_MODEL_DIR/bin" + +# Making sure these can execute: +chmod +x dispatchMPIJob.sh parseCommand.py + +# Remove repo as it's not needed anymore: +cd "$OM_ROOT" && rm -rf openmpp + +# Output various oms settings to console: [ -z "$OMS_PORT" ] && OMS_PORT=4040 echo "OM_ROOT=$OM_ROOT" @@ -62,18 +113,14 @@ echo "OMS_PORT=$OMS_PORT" echo "OMS_URL=$OMS_URL" echo "OMS_MODEL_DIR=$OMS_MODEL_DIR" -if [ ! -d $OMS_MODEL_DIR ]; then - mkdir -p $OMS_MODEL_DIR -fi - echo "OMS_HOME_DIR=$OMS_HOME_DIR" -if [ ! -d $OMS_HOME_DIR ]; then - mkdir -p $OMS_HOME_DIR -fi +echo "OMS_LOG_DIR=$OMS_LOG_DIR" -OM_ROOT=$OM_ROOT ./bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -status=$? +# start oms web-service: +OM_ROOT=$OM_ROOT ${OM_ROOT}/bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.ModelLogDir ${OMS_LOG_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -OpenM.LogToFile -OpenM.LogUseDailyStamp -OpenM.LogFilePath ${OM_ROOT}/log/oms.log + +status=$? if [ $status -ne 0 ] ; then [ $status -eq 130 ] && echo " oms web-service terminated by Ctrl+C" diff --git a/output/jupyterlab-cpu/start-custom.sh b/output/jupyterlab-cpu/start-custom.sh index 3bdce8eb5..d9c859ebe 100755 --- a/output/jupyterlab-cpu/start-custom.sh +++ b/output/jupyterlab-cpu/start-custom.sh @@ -16,7 +16,7 @@ fi test -z "$GIT_EXAMPLE_NOTEBOOKS" || git clone "$GIT_EXAMPLE_NOTEBOOKS" if [ ! -e /home/$NB_USER/.Rprofile ]; then - cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm /tmp/.Rprofile + cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm -rf /tmp/.Rprofile fi # Configure the shell! If not already configured. @@ -151,14 +151,6 @@ if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then conda config --add channels https://jfrog.aaw.cloud.statcan.ca/artifactory/api/conda/conda-pytorch-remote fi -# Copy default ompp models on first start up -export OMS_MODELS_DIR="/home/jovyan/models" -if [ ! -d "$OMS_MODELS_DIR" ]; then - echo "Creating ompp default model directory" - mkdir -p "$OMS_MODELS_DIR" - cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODELS_DIR" -fi - echo "--------------------starting jupyter--------------------" /opt/conda/bin/jupyter server --notebook-dir=/home/${NB_USER} \ @@ -172,4 +164,4 @@ echo "--------------------starting jupyter--------------------" --ServerApp.base_url=${NB_PREFIX} \ --ServerApp.default_url=${DEFAULT_JUPYTER_URL:-/tree} -echo "--------------------shutting down, persisting VS_CODE settings--------------------" \ No newline at end of file +echo "--------------------shutting down, persisting VS_CODE settings--------------------" diff --git a/output/jupyterlab-cpu/start-oms.sh b/output/jupyterlab-cpu/start-oms.sh index f6632b8b1..e825a466e 100644 --- a/output/jupyterlab-cpu/start-oms.sh +++ b/output/jupyterlab-cpu/start-oms.sh @@ -33,28 +33,79 @@ OM_ROOT="$OMPP_INSTALL_DIR" [ "$OM_ROOT" != "$PWD" ] && pushd $OM_ROOT # allow to use $MODEL_NAME.ini file in UI for model run -# export OM_CFG_INI_ALLOW=true export OM_CFG_INI_ANY_KEY=true export OMS_URL=${JUPYTER_SERVER_URL}ompp # OpenM++ default configuraton -if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then - #DEV +if [ "$KUBERNETES_SERVICE_HOST" =~ ".131." ] || [ -z $KUBERNETES_SERVICE_HOST ]; then + #DEV or Localhost export OMS_MODEL_DIR=/home/jovyan/models + export OMS_LOG_DIR=/home/jovyan/logs export OMS_HOME_DIR=/home/jovyan/ else if [ -d "/etc/protb" ]; then export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/ else export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/ fi fi -# start oms web-service -# +# Create models directory if it doesn't exist: +if [ ! -d "$OMS_MODEL_DIR" ]; then + mkdir -p "$OMS_MODEL_DIR" +fi + +# Create model log directory if it doesn't exist: +if [ ! -d "$OMS_LOG_DIR" ]; then + mkdir -p "$OMS_LOG_DIR" +fi + +# Copy sample models from openmpp installation archive into models directory: +cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODEL_DIR" + +# These three environment variables don't persist so let's try using a file: +echo "$OMS_HOME_DIR" > $OM_ROOT/etc/oms_home_dir +echo "$OMS_MODEL_DIR" > $OM_ROOT/etc/oms_model_dir +echo "$OMS_LOG_DIR" > $OM_ROOT/etc/oms_log_dir + + +# Import openmpp repo to get scripts and templates needed to run mpi jobs via kubeflow: +if [ ! -d /openmpp ] + then + git clone https://github.com/StatCan/openmpp.git +fi +cd openmpp +branch="main" +state=$(git symbolic-ref --short HEAD 2>&1) +if [ $state != $branch ] + then + git checkout $branch +fi +git pull +cd mpi-job-files + +# Copy scripts and templates into openmpp installation bin and etc folders: +cp dispatchMPIJob.sh parseCommand.py "$OM_ROOT/bin/" +cp mpi.kubeflow.template.txt MPIJobTemplate.yaml "$OM_ROOT/etc/" + +# Delete the default mpi golang template that does not work in our context: +rm -f "$OM_ROOT/etc/mpi.ModelRun.template.txt" + +# Copy the mpi enabled model binary to the model binaries directory: +cp RiskPaths_mpi "$OMS_MODEL_DIR/bin" + +# Making sure these can execute: +chmod +x dispatchMPIJob.sh parseCommand.py + +# Remove repo as it's not needed anymore: +cd "$OM_ROOT" && rm -rf openmpp + +# Output various oms settings to console: [ -z "$OMS_PORT" ] && OMS_PORT=4040 echo "OM_ROOT=$OM_ROOT" @@ -62,18 +113,14 @@ echo "OMS_PORT=$OMS_PORT" echo "OMS_URL=$OMS_URL" echo "OMS_MODEL_DIR=$OMS_MODEL_DIR" -if [ ! -d $OMS_MODEL_DIR ]; then - mkdir -p $OMS_MODEL_DIR -fi - echo "OMS_HOME_DIR=$OMS_HOME_DIR" -if [ ! -d $OMS_HOME_DIR ]; then - mkdir -p $OMS_HOME_DIR -fi +echo "OMS_LOG_DIR=$OMS_LOG_DIR" -OM_ROOT=$OM_ROOT ./bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -status=$? +# start oms web-service: +OM_ROOT=$OM_ROOT ${OM_ROOT}/bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.ModelLogDir ${OMS_LOG_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -OpenM.LogToFile -OpenM.LogUseDailyStamp -OpenM.LogFilePath ${OM_ROOT}/log/oms.log + +status=$? if [ $status -ne 0 ] ; then [ $status -eq 130 ] && echo " oms web-service terminated by Ctrl+C" diff --git a/output/jupyterlab-pytorch/start-custom.sh b/output/jupyterlab-pytorch/start-custom.sh index 3bdce8eb5..d9c859ebe 100755 --- a/output/jupyterlab-pytorch/start-custom.sh +++ b/output/jupyterlab-pytorch/start-custom.sh @@ -16,7 +16,7 @@ fi test -z "$GIT_EXAMPLE_NOTEBOOKS" || git clone "$GIT_EXAMPLE_NOTEBOOKS" if [ ! -e /home/$NB_USER/.Rprofile ]; then - cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm /tmp/.Rprofile + cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm -rf /tmp/.Rprofile fi # Configure the shell! If not already configured. @@ -151,14 +151,6 @@ if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then conda config --add channels https://jfrog.aaw.cloud.statcan.ca/artifactory/api/conda/conda-pytorch-remote fi -# Copy default ompp models on first start up -export OMS_MODELS_DIR="/home/jovyan/models" -if [ ! -d "$OMS_MODELS_DIR" ]; then - echo "Creating ompp default model directory" - mkdir -p "$OMS_MODELS_DIR" - cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODELS_DIR" -fi - echo "--------------------starting jupyter--------------------" /opt/conda/bin/jupyter server --notebook-dir=/home/${NB_USER} \ @@ -172,4 +164,4 @@ echo "--------------------starting jupyter--------------------" --ServerApp.base_url=${NB_PREFIX} \ --ServerApp.default_url=${DEFAULT_JUPYTER_URL:-/tree} -echo "--------------------shutting down, persisting VS_CODE settings--------------------" \ No newline at end of file +echo "--------------------shutting down, persisting VS_CODE settings--------------------" diff --git a/output/jupyterlab-pytorch/start-oms.sh b/output/jupyterlab-pytorch/start-oms.sh index f6632b8b1..e825a466e 100644 --- a/output/jupyterlab-pytorch/start-oms.sh +++ b/output/jupyterlab-pytorch/start-oms.sh @@ -33,28 +33,79 @@ OM_ROOT="$OMPP_INSTALL_DIR" [ "$OM_ROOT" != "$PWD" ] && pushd $OM_ROOT # allow to use $MODEL_NAME.ini file in UI for model run -# export OM_CFG_INI_ALLOW=true export OM_CFG_INI_ANY_KEY=true export OMS_URL=${JUPYTER_SERVER_URL}ompp # OpenM++ default configuraton -if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then - #DEV +if [ "$KUBERNETES_SERVICE_HOST" =~ ".131." ] || [ -z $KUBERNETES_SERVICE_HOST ]; then + #DEV or Localhost export OMS_MODEL_DIR=/home/jovyan/models + export OMS_LOG_DIR=/home/jovyan/logs export OMS_HOME_DIR=/home/jovyan/ else if [ -d "/etc/protb" ]; then export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/ else export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/ fi fi -# start oms web-service -# +# Create models directory if it doesn't exist: +if [ ! -d "$OMS_MODEL_DIR" ]; then + mkdir -p "$OMS_MODEL_DIR" +fi + +# Create model log directory if it doesn't exist: +if [ ! -d "$OMS_LOG_DIR" ]; then + mkdir -p "$OMS_LOG_DIR" +fi + +# Copy sample models from openmpp installation archive into models directory: +cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODEL_DIR" + +# These three environment variables don't persist so let's try using a file: +echo "$OMS_HOME_DIR" > $OM_ROOT/etc/oms_home_dir +echo "$OMS_MODEL_DIR" > $OM_ROOT/etc/oms_model_dir +echo "$OMS_LOG_DIR" > $OM_ROOT/etc/oms_log_dir + + +# Import openmpp repo to get scripts and templates needed to run mpi jobs via kubeflow: +if [ ! -d /openmpp ] + then + git clone https://github.com/StatCan/openmpp.git +fi +cd openmpp +branch="main" +state=$(git symbolic-ref --short HEAD 2>&1) +if [ $state != $branch ] + then + git checkout $branch +fi +git pull +cd mpi-job-files + +# Copy scripts and templates into openmpp installation bin and etc folders: +cp dispatchMPIJob.sh parseCommand.py "$OM_ROOT/bin/" +cp mpi.kubeflow.template.txt MPIJobTemplate.yaml "$OM_ROOT/etc/" + +# Delete the default mpi golang template that does not work in our context: +rm -f "$OM_ROOT/etc/mpi.ModelRun.template.txt" + +# Copy the mpi enabled model binary to the model binaries directory: +cp RiskPaths_mpi "$OMS_MODEL_DIR/bin" + +# Making sure these can execute: +chmod +x dispatchMPIJob.sh parseCommand.py + +# Remove repo as it's not needed anymore: +cd "$OM_ROOT" && rm -rf openmpp + +# Output various oms settings to console: [ -z "$OMS_PORT" ] && OMS_PORT=4040 echo "OM_ROOT=$OM_ROOT" @@ -62,18 +113,14 @@ echo "OMS_PORT=$OMS_PORT" echo "OMS_URL=$OMS_URL" echo "OMS_MODEL_DIR=$OMS_MODEL_DIR" -if [ ! -d $OMS_MODEL_DIR ]; then - mkdir -p $OMS_MODEL_DIR -fi - echo "OMS_HOME_DIR=$OMS_HOME_DIR" -if [ ! -d $OMS_HOME_DIR ]; then - mkdir -p $OMS_HOME_DIR -fi +echo "OMS_LOG_DIR=$OMS_LOG_DIR" -OM_ROOT=$OM_ROOT ./bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -status=$? +# start oms web-service: +OM_ROOT=$OM_ROOT ${OM_ROOT}/bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.ModelLogDir ${OMS_LOG_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -OpenM.LogToFile -OpenM.LogUseDailyStamp -OpenM.LogFilePath ${OM_ROOT}/log/oms.log + +status=$? if [ $status -ne 0 ] ; then [ $status -eq 130 ] && echo " oms web-service terminated by Ctrl+C" diff --git a/output/jupyterlab-tensorflow/start-custom.sh b/output/jupyterlab-tensorflow/start-custom.sh index 3bdce8eb5..d9c859ebe 100755 --- a/output/jupyterlab-tensorflow/start-custom.sh +++ b/output/jupyterlab-tensorflow/start-custom.sh @@ -16,7 +16,7 @@ fi test -z "$GIT_EXAMPLE_NOTEBOOKS" || git clone "$GIT_EXAMPLE_NOTEBOOKS" if [ ! -e /home/$NB_USER/.Rprofile ]; then - cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm /tmp/.Rprofile + cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm -rf /tmp/.Rprofile fi # Configure the shell! If not already configured. @@ -151,14 +151,6 @@ if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then conda config --add channels https://jfrog.aaw.cloud.statcan.ca/artifactory/api/conda/conda-pytorch-remote fi -# Copy default ompp models on first start up -export OMS_MODELS_DIR="/home/jovyan/models" -if [ ! -d "$OMS_MODELS_DIR" ]; then - echo "Creating ompp default model directory" - mkdir -p "$OMS_MODELS_DIR" - cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODELS_DIR" -fi - echo "--------------------starting jupyter--------------------" /opt/conda/bin/jupyter server --notebook-dir=/home/${NB_USER} \ @@ -172,4 +164,4 @@ echo "--------------------starting jupyter--------------------" --ServerApp.base_url=${NB_PREFIX} \ --ServerApp.default_url=${DEFAULT_JUPYTER_URL:-/tree} -echo "--------------------shutting down, persisting VS_CODE settings--------------------" \ No newline at end of file +echo "--------------------shutting down, persisting VS_CODE settings--------------------" diff --git a/output/jupyterlab-tensorflow/start-oms.sh b/output/jupyterlab-tensorflow/start-oms.sh index f6632b8b1..e825a466e 100644 --- a/output/jupyterlab-tensorflow/start-oms.sh +++ b/output/jupyterlab-tensorflow/start-oms.sh @@ -33,28 +33,79 @@ OM_ROOT="$OMPP_INSTALL_DIR" [ "$OM_ROOT" != "$PWD" ] && pushd $OM_ROOT # allow to use $MODEL_NAME.ini file in UI for model run -# export OM_CFG_INI_ALLOW=true export OM_CFG_INI_ANY_KEY=true export OMS_URL=${JUPYTER_SERVER_URL}ompp # OpenM++ default configuraton -if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then - #DEV +if [ "$KUBERNETES_SERVICE_HOST" =~ ".131." ] || [ -z $KUBERNETES_SERVICE_HOST ]; then + #DEV or Localhost export OMS_MODEL_DIR=/home/jovyan/models + export OMS_LOG_DIR=/home/jovyan/logs export OMS_HOME_DIR=/home/jovyan/ else if [ -d "/etc/protb" ]; then export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/ else export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/ fi fi -# start oms web-service -# +# Create models directory if it doesn't exist: +if [ ! -d "$OMS_MODEL_DIR" ]; then + mkdir -p "$OMS_MODEL_DIR" +fi + +# Create model log directory if it doesn't exist: +if [ ! -d "$OMS_LOG_DIR" ]; then + mkdir -p "$OMS_LOG_DIR" +fi + +# Copy sample models from openmpp installation archive into models directory: +cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODEL_DIR" + +# These three environment variables don't persist so let's try using a file: +echo "$OMS_HOME_DIR" > $OM_ROOT/etc/oms_home_dir +echo "$OMS_MODEL_DIR" > $OM_ROOT/etc/oms_model_dir +echo "$OMS_LOG_DIR" > $OM_ROOT/etc/oms_log_dir + + +# Import openmpp repo to get scripts and templates needed to run mpi jobs via kubeflow: +if [ ! -d /openmpp ] + then + git clone https://github.com/StatCan/openmpp.git +fi +cd openmpp +branch="main" +state=$(git symbolic-ref --short HEAD 2>&1) +if [ $state != $branch ] + then + git checkout $branch +fi +git pull +cd mpi-job-files + +# Copy scripts and templates into openmpp installation bin and etc folders: +cp dispatchMPIJob.sh parseCommand.py "$OM_ROOT/bin/" +cp mpi.kubeflow.template.txt MPIJobTemplate.yaml "$OM_ROOT/etc/" + +# Delete the default mpi golang template that does not work in our context: +rm -f "$OM_ROOT/etc/mpi.ModelRun.template.txt" + +# Copy the mpi enabled model binary to the model binaries directory: +cp RiskPaths_mpi "$OMS_MODEL_DIR/bin" + +# Making sure these can execute: +chmod +x dispatchMPIJob.sh parseCommand.py + +# Remove repo as it's not needed anymore: +cd "$OM_ROOT" && rm -rf openmpp + +# Output various oms settings to console: [ -z "$OMS_PORT" ] && OMS_PORT=4040 echo "OM_ROOT=$OM_ROOT" @@ -62,18 +113,14 @@ echo "OMS_PORT=$OMS_PORT" echo "OMS_URL=$OMS_URL" echo "OMS_MODEL_DIR=$OMS_MODEL_DIR" -if [ ! -d $OMS_MODEL_DIR ]; then - mkdir -p $OMS_MODEL_DIR -fi - echo "OMS_HOME_DIR=$OMS_HOME_DIR" -if [ ! -d $OMS_HOME_DIR ]; then - mkdir -p $OMS_HOME_DIR -fi +echo "OMS_LOG_DIR=$OMS_LOG_DIR" -OM_ROOT=$OM_ROOT ./bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -status=$? +# start oms web-service: +OM_ROOT=$OM_ROOT ${OM_ROOT}/bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.ModelLogDir ${OMS_LOG_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -OpenM.LogToFile -OpenM.LogUseDailyStamp -OpenM.LogFilePath ${OM_ROOT}/log/oms.log + +status=$? if [ $status -ne 0 ] ; then [ $status -eq 130 ] && echo " oms web-service terminated by Ctrl+C" diff --git a/output/remote-desktop/start-custom.sh b/output/remote-desktop/start-custom.sh index 3bdce8eb5..d9c859ebe 100755 --- a/output/remote-desktop/start-custom.sh +++ b/output/remote-desktop/start-custom.sh @@ -16,7 +16,7 @@ fi test -z "$GIT_EXAMPLE_NOTEBOOKS" || git clone "$GIT_EXAMPLE_NOTEBOOKS" if [ ! -e /home/$NB_USER/.Rprofile ]; then - cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm /tmp/.Rprofile + cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm -rf /tmp/.Rprofile fi # Configure the shell! If not already configured. @@ -151,14 +151,6 @@ if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then conda config --add channels https://jfrog.aaw.cloud.statcan.ca/artifactory/api/conda/conda-pytorch-remote fi -# Copy default ompp models on first start up -export OMS_MODELS_DIR="/home/jovyan/models" -if [ ! -d "$OMS_MODELS_DIR" ]; then - echo "Creating ompp default model directory" - mkdir -p "$OMS_MODELS_DIR" - cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODELS_DIR" -fi - echo "--------------------starting jupyter--------------------" /opt/conda/bin/jupyter server --notebook-dir=/home/${NB_USER} \ @@ -172,4 +164,4 @@ echo "--------------------starting jupyter--------------------" --ServerApp.base_url=${NB_PREFIX} \ --ServerApp.default_url=${DEFAULT_JUPYTER_URL:-/tree} -echo "--------------------shutting down, persisting VS_CODE settings--------------------" \ No newline at end of file +echo "--------------------shutting down, persisting VS_CODE settings--------------------" diff --git a/output/remote-desktop/start-oms.sh b/output/remote-desktop/start-oms.sh index f6632b8b1..e825a466e 100644 --- a/output/remote-desktop/start-oms.sh +++ b/output/remote-desktop/start-oms.sh @@ -33,28 +33,79 @@ OM_ROOT="$OMPP_INSTALL_DIR" [ "$OM_ROOT" != "$PWD" ] && pushd $OM_ROOT # allow to use $MODEL_NAME.ini file in UI for model run -# export OM_CFG_INI_ALLOW=true export OM_CFG_INI_ANY_KEY=true export OMS_URL=${JUPYTER_SERVER_URL}ompp # OpenM++ default configuraton -if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then - #DEV +if [ "$KUBERNETES_SERVICE_HOST" =~ ".131." ] || [ -z $KUBERNETES_SERVICE_HOST ]; then + #DEV or Localhost export OMS_MODEL_DIR=/home/jovyan/models + export OMS_LOG_DIR=/home/jovyan/logs export OMS_HOME_DIR=/home/jovyan/ else if [ -d "/etc/protb" ]; then export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/ else export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/ fi fi -# start oms web-service -# +# Create models directory if it doesn't exist: +if [ ! -d "$OMS_MODEL_DIR" ]; then + mkdir -p "$OMS_MODEL_DIR" +fi + +# Create model log directory if it doesn't exist: +if [ ! -d "$OMS_LOG_DIR" ]; then + mkdir -p "$OMS_LOG_DIR" +fi + +# Copy sample models from openmpp installation archive into models directory: +cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODEL_DIR" + +# These three environment variables don't persist so let's try using a file: +echo "$OMS_HOME_DIR" > $OM_ROOT/etc/oms_home_dir +echo "$OMS_MODEL_DIR" > $OM_ROOT/etc/oms_model_dir +echo "$OMS_LOG_DIR" > $OM_ROOT/etc/oms_log_dir + + +# Import openmpp repo to get scripts and templates needed to run mpi jobs via kubeflow: +if [ ! -d /openmpp ] + then + git clone https://github.com/StatCan/openmpp.git +fi +cd openmpp +branch="main" +state=$(git symbolic-ref --short HEAD 2>&1) +if [ $state != $branch ] + then + git checkout $branch +fi +git pull +cd mpi-job-files + +# Copy scripts and templates into openmpp installation bin and etc folders: +cp dispatchMPIJob.sh parseCommand.py "$OM_ROOT/bin/" +cp mpi.kubeflow.template.txt MPIJobTemplate.yaml "$OM_ROOT/etc/" + +# Delete the default mpi golang template that does not work in our context: +rm -f "$OM_ROOT/etc/mpi.ModelRun.template.txt" + +# Copy the mpi enabled model binary to the model binaries directory: +cp RiskPaths_mpi "$OMS_MODEL_DIR/bin" + +# Making sure these can execute: +chmod +x dispatchMPIJob.sh parseCommand.py + +# Remove repo as it's not needed anymore: +cd "$OM_ROOT" && rm -rf openmpp + +# Output various oms settings to console: [ -z "$OMS_PORT" ] && OMS_PORT=4040 echo "OM_ROOT=$OM_ROOT" @@ -62,18 +113,14 @@ echo "OMS_PORT=$OMS_PORT" echo "OMS_URL=$OMS_URL" echo "OMS_MODEL_DIR=$OMS_MODEL_DIR" -if [ ! -d $OMS_MODEL_DIR ]; then - mkdir -p $OMS_MODEL_DIR -fi - echo "OMS_HOME_DIR=$OMS_HOME_DIR" -if [ ! -d $OMS_HOME_DIR ]; then - mkdir -p $OMS_HOME_DIR -fi +echo "OMS_LOG_DIR=$OMS_LOG_DIR" -OM_ROOT=$OM_ROOT ./bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -status=$? +# start oms web-service: +OM_ROOT=$OM_ROOT ${OM_ROOT}/bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.ModelLogDir ${OMS_LOG_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -OpenM.LogToFile -OpenM.LogUseDailyStamp -OpenM.LogFilePath ${OM_ROOT}/log/oms.log + +status=$? if [ $status -ne 0 ] ; then [ $status -eq 130 ] && echo " oms web-service terminated by Ctrl+C" diff --git a/output/rstudio/start-custom.sh b/output/rstudio/start-custom.sh index 3bdce8eb5..d9c859ebe 100755 --- a/output/rstudio/start-custom.sh +++ b/output/rstudio/start-custom.sh @@ -16,7 +16,7 @@ fi test -z "$GIT_EXAMPLE_NOTEBOOKS" || git clone "$GIT_EXAMPLE_NOTEBOOKS" if [ ! -e /home/$NB_USER/.Rprofile ]; then - cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm /tmp/.Rprofile + cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm -rf /tmp/.Rprofile fi # Configure the shell! If not already configured. @@ -151,14 +151,6 @@ if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then conda config --add channels https://jfrog.aaw.cloud.statcan.ca/artifactory/api/conda/conda-pytorch-remote fi -# Copy default ompp models on first start up -export OMS_MODELS_DIR="/home/jovyan/models" -if [ ! -d "$OMS_MODELS_DIR" ]; then - echo "Creating ompp default model directory" - mkdir -p "$OMS_MODELS_DIR" - cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODELS_DIR" -fi - echo "--------------------starting jupyter--------------------" /opt/conda/bin/jupyter server --notebook-dir=/home/${NB_USER} \ @@ -172,4 +164,4 @@ echo "--------------------starting jupyter--------------------" --ServerApp.base_url=${NB_PREFIX} \ --ServerApp.default_url=${DEFAULT_JUPYTER_URL:-/tree} -echo "--------------------shutting down, persisting VS_CODE settings--------------------" \ No newline at end of file +echo "--------------------shutting down, persisting VS_CODE settings--------------------" diff --git a/output/rstudio/start-oms.sh b/output/rstudio/start-oms.sh index f6632b8b1..e825a466e 100644 --- a/output/rstudio/start-oms.sh +++ b/output/rstudio/start-oms.sh @@ -33,28 +33,79 @@ OM_ROOT="$OMPP_INSTALL_DIR" [ "$OM_ROOT" != "$PWD" ] && pushd $OM_ROOT # allow to use $MODEL_NAME.ini file in UI for model run -# export OM_CFG_INI_ALLOW=true export OM_CFG_INI_ANY_KEY=true export OMS_URL=${JUPYTER_SERVER_URL}ompp # OpenM++ default configuraton -if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then - #DEV +if [ "$KUBERNETES_SERVICE_HOST" =~ ".131." ] || [ -z $KUBERNETES_SERVICE_HOST ]; then + #DEV or Localhost export OMS_MODEL_DIR=/home/jovyan/models + export OMS_LOG_DIR=/home/jovyan/logs export OMS_HOME_DIR=/home/jovyan/ else if [ -d "/etc/protb" ]; then export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/ else export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/ fi fi -# start oms web-service -# +# Create models directory if it doesn't exist: +if [ ! -d "$OMS_MODEL_DIR" ]; then + mkdir -p "$OMS_MODEL_DIR" +fi + +# Create model log directory if it doesn't exist: +if [ ! -d "$OMS_LOG_DIR" ]; then + mkdir -p "$OMS_LOG_DIR" +fi + +# Copy sample models from openmpp installation archive into models directory: +cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODEL_DIR" + +# These three environment variables don't persist so let's try using a file: +echo "$OMS_HOME_DIR" > $OM_ROOT/etc/oms_home_dir +echo "$OMS_MODEL_DIR" > $OM_ROOT/etc/oms_model_dir +echo "$OMS_LOG_DIR" > $OM_ROOT/etc/oms_log_dir + + +# Import openmpp repo to get scripts and templates needed to run mpi jobs via kubeflow: +if [ ! -d /openmpp ] + then + git clone https://github.com/StatCan/openmpp.git +fi +cd openmpp +branch="main" +state=$(git symbolic-ref --short HEAD 2>&1) +if [ $state != $branch ] + then + git checkout $branch +fi +git pull +cd mpi-job-files + +# Copy scripts and templates into openmpp installation bin and etc folders: +cp dispatchMPIJob.sh parseCommand.py "$OM_ROOT/bin/" +cp mpi.kubeflow.template.txt MPIJobTemplate.yaml "$OM_ROOT/etc/" + +# Delete the default mpi golang template that does not work in our context: +rm -f "$OM_ROOT/etc/mpi.ModelRun.template.txt" + +# Copy the mpi enabled model binary to the model binaries directory: +cp RiskPaths_mpi "$OMS_MODEL_DIR/bin" + +# Making sure these can execute: +chmod +x dispatchMPIJob.sh parseCommand.py + +# Remove repo as it's not needed anymore: +cd "$OM_ROOT" && rm -rf openmpp + +# Output various oms settings to console: [ -z "$OMS_PORT" ] && OMS_PORT=4040 echo "OM_ROOT=$OM_ROOT" @@ -62,18 +113,14 @@ echo "OMS_PORT=$OMS_PORT" echo "OMS_URL=$OMS_URL" echo "OMS_MODEL_DIR=$OMS_MODEL_DIR" -if [ ! -d $OMS_MODEL_DIR ]; then - mkdir -p $OMS_MODEL_DIR -fi - echo "OMS_HOME_DIR=$OMS_HOME_DIR" -if [ ! -d $OMS_HOME_DIR ]; then - mkdir -p $OMS_HOME_DIR -fi +echo "OMS_LOG_DIR=$OMS_LOG_DIR" -OM_ROOT=$OM_ROOT ./bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -status=$? +# start oms web-service: +OM_ROOT=$OM_ROOT ${OM_ROOT}/bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.ModelLogDir ${OMS_LOG_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -OpenM.LogToFile -OpenM.LogUseDailyStamp -OpenM.LogFilePath ${OM_ROOT}/log/oms.log + +status=$? if [ $status -ne 0 ] ; then [ $status -eq 130 ] && echo " oms web-service terminated by Ctrl+C" diff --git a/output/sas/start-custom.sh b/output/sas/start-custom.sh index 3bdce8eb5..d9c859ebe 100755 --- a/output/sas/start-custom.sh +++ b/output/sas/start-custom.sh @@ -16,7 +16,7 @@ fi test -z "$GIT_EXAMPLE_NOTEBOOKS" || git clone "$GIT_EXAMPLE_NOTEBOOKS" if [ ! -e /home/$NB_USER/.Rprofile ]; then - cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm /tmp/.Rprofile + cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm -rf /tmp/.Rprofile fi # Configure the shell! If not already configured. @@ -151,14 +151,6 @@ if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then conda config --add channels https://jfrog.aaw.cloud.statcan.ca/artifactory/api/conda/conda-pytorch-remote fi -# Copy default ompp models on first start up -export OMS_MODELS_DIR="/home/jovyan/models" -if [ ! -d "$OMS_MODELS_DIR" ]; then - echo "Creating ompp default model directory" - mkdir -p "$OMS_MODELS_DIR" - cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODELS_DIR" -fi - echo "--------------------starting jupyter--------------------" /opt/conda/bin/jupyter server --notebook-dir=/home/${NB_USER} \ @@ -172,4 +164,4 @@ echo "--------------------starting jupyter--------------------" --ServerApp.base_url=${NB_PREFIX} \ --ServerApp.default_url=${DEFAULT_JUPYTER_URL:-/tree} -echo "--------------------shutting down, persisting VS_CODE settings--------------------" \ No newline at end of file +echo "--------------------shutting down, persisting VS_CODE settings--------------------" diff --git a/output/sas/start-oms.sh b/output/sas/start-oms.sh index f6632b8b1..e825a466e 100644 --- a/output/sas/start-oms.sh +++ b/output/sas/start-oms.sh @@ -33,28 +33,79 @@ OM_ROOT="$OMPP_INSTALL_DIR" [ "$OM_ROOT" != "$PWD" ] && pushd $OM_ROOT # allow to use $MODEL_NAME.ini file in UI for model run -# export OM_CFG_INI_ALLOW=true export OM_CFG_INI_ANY_KEY=true export OMS_URL=${JUPYTER_SERVER_URL}ompp # OpenM++ default configuraton -if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then - #DEV +if [ "$KUBERNETES_SERVICE_HOST" =~ ".131." ] || [ -z $KUBERNETES_SERVICE_HOST ]; then + #DEV or Localhost export OMS_MODEL_DIR=/home/jovyan/models + export OMS_LOG_DIR=/home/jovyan/logs export OMS_HOME_DIR=/home/jovyan/ else if [ -d "/etc/protb" ]; then export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/ else export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/ fi fi -# start oms web-service -# +# Create models directory if it doesn't exist: +if [ ! -d "$OMS_MODEL_DIR" ]; then + mkdir -p "$OMS_MODEL_DIR" +fi + +# Create model log directory if it doesn't exist: +if [ ! -d "$OMS_LOG_DIR" ]; then + mkdir -p "$OMS_LOG_DIR" +fi + +# Copy sample models from openmpp installation archive into models directory: +cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODEL_DIR" + +# These three environment variables don't persist so let's try using a file: +echo "$OMS_HOME_DIR" > $OM_ROOT/etc/oms_home_dir +echo "$OMS_MODEL_DIR" > $OM_ROOT/etc/oms_model_dir +echo "$OMS_LOG_DIR" > $OM_ROOT/etc/oms_log_dir + + +# Import openmpp repo to get scripts and templates needed to run mpi jobs via kubeflow: +if [ ! -d /openmpp ] + then + git clone https://github.com/StatCan/openmpp.git +fi +cd openmpp +branch="main" +state=$(git symbolic-ref --short HEAD 2>&1) +if [ $state != $branch ] + then + git checkout $branch +fi +git pull +cd mpi-job-files + +# Copy scripts and templates into openmpp installation bin and etc folders: +cp dispatchMPIJob.sh parseCommand.py "$OM_ROOT/bin/" +cp mpi.kubeflow.template.txt MPIJobTemplate.yaml "$OM_ROOT/etc/" + +# Delete the default mpi golang template that does not work in our context: +rm -f "$OM_ROOT/etc/mpi.ModelRun.template.txt" + +# Copy the mpi enabled model binary to the model binaries directory: +cp RiskPaths_mpi "$OMS_MODEL_DIR/bin" + +# Making sure these can execute: +chmod +x dispatchMPIJob.sh parseCommand.py + +# Remove repo as it's not needed anymore: +cd "$OM_ROOT" && rm -rf openmpp + +# Output various oms settings to console: [ -z "$OMS_PORT" ] && OMS_PORT=4040 echo "OM_ROOT=$OM_ROOT" @@ -62,18 +113,14 @@ echo "OMS_PORT=$OMS_PORT" echo "OMS_URL=$OMS_URL" echo "OMS_MODEL_DIR=$OMS_MODEL_DIR" -if [ ! -d $OMS_MODEL_DIR ]; then - mkdir -p $OMS_MODEL_DIR -fi - echo "OMS_HOME_DIR=$OMS_HOME_DIR" -if [ ! -d $OMS_HOME_DIR ]; then - mkdir -p $OMS_HOME_DIR -fi +echo "OMS_LOG_DIR=$OMS_LOG_DIR" -OM_ROOT=$OM_ROOT ./bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -status=$? +# start oms web-service: +OM_ROOT=$OM_ROOT ${OM_ROOT}/bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.ModelLogDir ${OMS_LOG_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -OpenM.LogToFile -OpenM.LogUseDailyStamp -OpenM.LogFilePath ${OM_ROOT}/log/oms.log + +status=$? if [ $status -ne 0 ] ; then [ $status -eq 130 ] && echo " oms web-service terminated by Ctrl+C" diff --git a/resources/common/start-custom.sh b/resources/common/start-custom.sh index 3bdce8eb5..d9c859ebe 100755 --- a/resources/common/start-custom.sh +++ b/resources/common/start-custom.sh @@ -16,7 +16,7 @@ fi test -z "$GIT_EXAMPLE_NOTEBOOKS" || git clone "$GIT_EXAMPLE_NOTEBOOKS" if [ ! -e /home/$NB_USER/.Rprofile ]; then - cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm /tmp/.Rprofile + cat /tmp/.Rprofile >> /home/$NB_USER/.Rprofile && rm -rf /tmp/.Rprofile fi # Configure the shell! If not already configured. @@ -151,14 +151,6 @@ if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then conda config --add channels https://jfrog.aaw.cloud.statcan.ca/artifactory/api/conda/conda-pytorch-remote fi -# Copy default ompp models on first start up -export OMS_MODELS_DIR="/home/jovyan/models" -if [ ! -d "$OMS_MODELS_DIR" ]; then - echo "Creating ompp default model directory" - mkdir -p "$OMS_MODELS_DIR" - cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODELS_DIR" -fi - echo "--------------------starting jupyter--------------------" /opt/conda/bin/jupyter server --notebook-dir=/home/${NB_USER} \ @@ -172,4 +164,4 @@ echo "--------------------starting jupyter--------------------" --ServerApp.base_url=${NB_PREFIX} \ --ServerApp.default_url=${DEFAULT_JUPYTER_URL:-/tree} -echo "--------------------shutting down, persisting VS_CODE settings--------------------" \ No newline at end of file +echo "--------------------shutting down, persisting VS_CODE settings--------------------" diff --git a/resources/common/start-oms.sh b/resources/common/start-oms.sh index f6632b8b1..e825a466e 100644 --- a/resources/common/start-oms.sh +++ b/resources/common/start-oms.sh @@ -33,28 +33,79 @@ OM_ROOT="$OMPP_INSTALL_DIR" [ "$OM_ROOT" != "$PWD" ] && pushd $OM_ROOT # allow to use $MODEL_NAME.ini file in UI for model run -# export OM_CFG_INI_ALLOW=true export OM_CFG_INI_ANY_KEY=true export OMS_URL=${JUPYTER_SERVER_URL}ompp # OpenM++ default configuraton -if [[ "$KUBERNETES_SERVICE_HOST" =~ ".131." ]]; then - #DEV +if [ "$KUBERNETES_SERVICE_HOST" =~ ".131." ] || [ -z $KUBERNETES_SERVICE_HOST ]; then + #DEV or Localhost export OMS_MODEL_DIR=/home/jovyan/models + export OMS_LOG_DIR=/home/jovyan/logs export OMS_HOME_DIR=/home/jovyan/ else if [ -d "/etc/protb" ]; then export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-protected-b/microsim/ else export OMS_MODEL_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/models + export OMS_LOG_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/logs export OMS_HOME_DIR=/home/jovyan/buckets/aaw-unclassified/microsim/ fi fi -# start oms web-service -# +# Create models directory if it doesn't exist: +if [ ! -d "$OMS_MODEL_DIR" ]; then + mkdir -p "$OMS_MODEL_DIR" +fi + +# Create model log directory if it doesn't exist: +if [ ! -d "$OMS_LOG_DIR" ]; then + mkdir -p "$OMS_LOG_DIR" +fi + +# Copy sample models from openmpp installation archive into models directory: +cp -r "$OMPP_INSTALL_DIR/models/." "$OMS_MODEL_DIR" + +# These three environment variables don't persist so let's try using a file: +echo "$OMS_HOME_DIR" > $OM_ROOT/etc/oms_home_dir +echo "$OMS_MODEL_DIR" > $OM_ROOT/etc/oms_model_dir +echo "$OMS_LOG_DIR" > $OM_ROOT/etc/oms_log_dir + + +# Import openmpp repo to get scripts and templates needed to run mpi jobs via kubeflow: +if [ ! -d /openmpp ] + then + git clone https://github.com/StatCan/openmpp.git +fi +cd openmpp +branch="main" +state=$(git symbolic-ref --short HEAD 2>&1) +if [ $state != $branch ] + then + git checkout $branch +fi +git pull +cd mpi-job-files + +# Copy scripts and templates into openmpp installation bin and etc folders: +cp dispatchMPIJob.sh parseCommand.py "$OM_ROOT/bin/" +cp mpi.kubeflow.template.txt MPIJobTemplate.yaml "$OM_ROOT/etc/" + +# Delete the default mpi golang template that does not work in our context: +rm -f "$OM_ROOT/etc/mpi.ModelRun.template.txt" + +# Copy the mpi enabled model binary to the model binaries directory: +cp RiskPaths_mpi "$OMS_MODEL_DIR/bin" + +# Making sure these can execute: +chmod +x dispatchMPIJob.sh parseCommand.py + +# Remove repo as it's not needed anymore: +cd "$OM_ROOT" && rm -rf openmpp + +# Output various oms settings to console: [ -z "$OMS_PORT" ] && OMS_PORT=4040 echo "OM_ROOT=$OM_ROOT" @@ -62,18 +113,14 @@ echo "OMS_PORT=$OMS_PORT" echo "OMS_URL=$OMS_URL" echo "OMS_MODEL_DIR=$OMS_MODEL_DIR" -if [ ! -d $OMS_MODEL_DIR ]; then - mkdir -p $OMS_MODEL_DIR -fi - echo "OMS_HOME_DIR=$OMS_HOME_DIR" -if [ ! -d $OMS_HOME_DIR ]; then - mkdir -p $OMS_HOME_DIR -fi +echo "OMS_LOG_DIR=$OMS_LOG_DIR" -OM_ROOT=$OM_ROOT ./bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -status=$? +# start oms web-service: +OM_ROOT=$OM_ROOT ${OM_ROOT}/bin/oms -l localhost:${OMS_PORT} -oms.ModelDir ${OMS_MODEL_DIR} -oms.HomeDir ${OMS_HOME_DIR} -oms.ModelLogDir ${OMS_LOG_DIR} -oms.AllowDownload -oms.AllowUpload -oms.AllowMicrodata -oms.LogRequest -OpenM.LogToFile -OpenM.LogUseDailyStamp -OpenM.LogFilePath ${OM_ROOT}/log/oms.log + +status=$? if [ $status -ne 0 ] ; then [ $status -eq 130 ] && echo " oms web-service terminated by Ctrl+C"