diff --git a/bin/SLpipeline.sh b/bin/SLpipeline.sh index 54c3cd4..837fb4b 100755 --- a/bin/SLpipeline.sh +++ b/bin/SLpipeline.sh @@ -10,7 +10,7 @@ # @todo optional PI parameters # option to have a data+time ID in the name, by default it will be blank? -_version="SLpipeline: 4-sep-2024" +_version="SLpipeline: 17-sep-2024" echo "" echo "LMTOY>> VERSION $(cat $LMTOY/VERSION)" @@ -36,6 +36,7 @@ chunk=10g # chunksize for zipping up what used to be a tar file (use 0 to grun=1 # save the script generator? admit=0 # run ADMIT ? meta=1 # 1 or 2: 1=activate update for frontend db (for dataverse) +dataverse=0 # ingest an existing obsnum (assumes pipeline had been RUN) sleep=2 # add few seconds before running, allowing quick interrupt nproc=1 # number of processors to use (keep it at 1) rsync="" # rsync address for the TAP file (used at LMT/malt) @@ -45,6 +46,7 @@ webrun="" # optional directive for webrun to do parameter checking (SEQ/ma qagrade=0 # the final grade recorded for the archive (QAFAIL enforces -1; 0 by default; -2,1,2,3 when DA graded) public="" # if given, the public data for archiving. Default is 1 year after today. Example: 2020-12-31 + # Optional instrument specific pipeline can be added as well but are not known here # A few examples: # rsr_pipeline.sh --help @@ -122,7 +124,7 @@ fi rc0=$WORK_LMT/tmp/lmtoy_${obsnum}.rc lmtinfo.py $obsnum > $rc0 if [ $? != 0 ]; then - # some error (typically OBSNUM did not exist) + # some error (typically OBSNUM did not exist) - @todo give better error cat $rc0 rm -f $rc0 exit 1 @@ -149,6 +151,17 @@ if [ $obsnums = 0 ]; then pdir=$pidir/${obsnum} else pdir=$pidir/${on0}_${on1} + obsnum=${on0}_${on1} +fi +if [ $dataverse == 1 ]; then + if [ ! -d ${pdir} ]; then + echo "LMTOY>> $pdir does not exist yet" + exit 1 + fi + echo "LMTOY>> dataverse ingestion $obsnum from $pidir" + lmtoy_dataverse $obsnum $pidir + echo "LMTOY>> dataverse ingestion $obsnum from $pidir done." + exit 0 fi if [ $exist == 1 ] && [ -d $pidir/$obsnum ]; then echo "LMTOY>> Skipping work for $pidir/$obsnum, it already exists" diff --git a/bin/lmtoy_functions.sh b/bin/lmtoy_functions.sh index 9aafb59..d47fd75 100755 --- a/bin/lmtoy_functions.sh +++ b/bin/lmtoy_functions.sh @@ -3,7 +3,7 @@ # some functions to share for lmtoy pipeline operations # beware, in bash shell variables are common variables between this and the caller -lmtoy_version="29-aug-2024" +lmtoy_version="17-sep-2024" echo "LMTOY>> lmtoy_functions $lmtoy_version via $0" @@ -176,6 +176,33 @@ function qac_select { } +function lmtoy_dataverse { + # input: obsnum pidir + # 123456 $WORK_LMT/$PID + echo "lmtoy_dataverse: $1 $2" + obsnum=$1 + pidir=$2 + db=$WORK_LMT/example_lmt.db + pid=$(basename $pidir) + dir4dv=${pidir}/dir4dv/${pid}/${obsnum} + if [ ! -d $dir4dv ]; then + echo "LMTOY>> no $dir4dv, return" + return + fi + # @todo make it less verbose + pushd $WORK_LMT/$pid/dir4dv + mk_metadata.py -y ${dir4dv}/${obsnum}_lmtmetadata.yaml $dir4dv + upload_project.sh in=. out=/tmp/dvout publish=1 verbose=0 overwrite=1 + mk_metadata.py -z /tmp/dvout/${pid}_${obsnum}_output.yaml -f $db $dir4dv + # we don't need flock anymore, since we run it serially and it's on NFS + # flock --verbose $db.flock mk_metadata.py ... + # + scp $db toltec3:lmtsearch_web + # cleanup $pid ??? + echo "$(lmtoy_date -u) $pid $obsnum" >> $WORK_LMT/dataverse.log + popd +} + function lmtoy_rsr1 { # input: first, obsnum, badlags, blanking, rfile, .... @@ -1540,3 +1567,5 @@ function lmtoy_ps1 { # mv index.html README.html } # lmtoy_ps1 + +