feat: Add debug comments to bash scripts

EBISPOT · Nov 30, 2023 · 9556784 · 9556784
1 parent 3ddade4
commit 9556784
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 1 deletion.
diff --git a/celery_worker_scripts/dev/START_CELERY_WORKERS_SLURM.sh b/celery_worker_scripts/dev/START_CELERY_WORKERS_SLURM.sh
@@ -16,15 +16,19 @@ lmod_cmd="module load singularity-3.6.4-gcc-9.3.0-yvkwp5n; module load openjdk-1
 
 if [ ! -z "${COMMIT_SHA}" ];
 then
+    echo "START pulling Singularity Image"
     sed -i "s/SINGULARITY_TAG=.*/SINGULARITY_TAG=\"${COMMIT_SHA}\"/g" $ENV_FILE
     singularity pull --dir $SINGULARITY_CACHEDIR docker://${SINGULARITY_REPO}/${SINGULARITY_IMAGE}:${COMMIT_SHA}
     SINGULARITY_TAG=$COMMIT_SHA
+    echo "DONE pulling Singularity Image"
 else
-    echo "not set"
+    echo "COMMIT_SHA not set"
 fi
 
 # Set Singularity cmd
+echo "START setting Singularity cmd"
 singularity_cmd="singularity exec --env-file $ENV_FILE $SINGULARITY_CACHEDIR/gwas-sumstats-service_${SINGULARITY_TAG}.sif"
+echo "DONE setting Singularity cmd"
 
 # Set celery worker cmd
 celery_cmd="celery -A sumstats_service.app.celery worker --loglevel=${LOG_LEVEL} --queues=${CELERY_QUEUE1},${CELERY_QUEUE2}"
@@ -36,7 +40,10 @@ celery_cmd="celery -A sumstats_service.app.celery worker --loglevel=${LOG_LEVEL}
 # See https://slurm.schedmd.com/scancel.html#OPT_full
 scancel --name=sumstats_service_celery_worker --signal=TERM --full
 
+echo "START spinning up dev celery workers:"
 # Submit new SLURM jobs for celery workers
 for WORKER_ID in {1..2}; do
+	echo $WORKER_ID
     sbatch --parsable --output="cel_${WORKER_ID}.o" --error="cel_${WORKER_ID}.e" --mem=${MEM} --time=7-00:00:00 --job-name=sumstats_service_celery_worker --wrap="${lmod_cmd}; ${singularity_cmd} ${celery_cmd}"
 done
+echo "DONE spinning up dev celery workers"
diff --git a/celery_worker_scripts/prod/START_CELERY_WORKERS_SLURM.sh b/celery_worker_scripts/prod/START_CELERY_WORKERS_SLURM.sh
@@ -11,7 +11,9 @@ source $ENV_FILE
 lmod_cmd="module load singularity-3.6.4-gcc-9.3.0-yvkwp5n; module load openjdk-16.0.2-gcc-9.3.0-xyn6nf5; module load nextflow-21.10.6-gcc-9.3.0-tkuemwd"
 
 # pull Singularity image
+echo "START pulling Singularity Image"
 singularity pull --dir $SINGULARITY_CACHEDIR --force  docker://${SINGULARITY_REPO}/${SINGULARITY_IMAGE}:latest
+echo "DONE pulling Singularity Image"
 
 # Set Singularity cmd
 singularity_cmd="singularity exec --env-file $ENV_FILE $SINGULARITY_CACHEDIR/gwas-sumstats-service_latest.sif"
@@ -24,18 +26,25 @@ celery_cmd="celery -A sumstats_service.app.celery worker --loglevel=${LOG_LEVEL}
 # are not sent to the batch step (the shell script). With this 
 # option scancel also signals the batch script and its children processes."
 # See https://slurm.schedmd.com/scancel.html#OPT_full
+echo "sending SIGTERM signal to prod celery workers"
 scancel --name=sumstats_service_celery_worker_prod --signal=TERM --full
 
 # Submit new SLURM jobs for HX celery workers
+echo "START spinning up HX celery workers:"
 for WORKER_ID in {1..3}; do
+	echo $WORKER_ID
     sbatch --parsable --output="cel_${WORKER_ID}.o" --error="cel_${WORKER_ID}.e" --mem=${MEM} --time=7-00:00:00 --job-name=sumstats_service_celery_worker_prod --wrap="${lmod_cmd}; ${singularity_cmd} ${celery_cmd}"
 done
+echo "DONE spinning up HX celery workers"
 
 # spin up the workers for hl rabbitmq
 ENV_FILE="/hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/cel_envs_hh"
 singularity_cmd="singularity exec --env-file $ENV_FILE $SINGULARITY_CACHEDIR/gwas-sumstats-service_latest.sif"
 
 # Submit new SLURM jobs for HH celery workers
+echo "spinning up HH celery workers:"
 for WORKER_ID in {4..6}; do
+	echo $WORKER_ID
     sbatch --parsable --output="cel_${WORKER_ID}.o" --error="cel_${WORKER_ID}.e" --mem=${MEM} --time=7-00:00:00 --job-name=sumstats_service_celery_worker_prod --wrap="${lmod_cmd}; ${singularity_cmd} ${celery_cmd}"
 done
+echo "DONE spinning up HH celery workers"