Tickets/dm 45476

.github/workflows/test_and_build_cache.yml

@@ -10,7 +10,7 @@ on:
 
 env:
   PROGRAM_NAME: transfer-embargo
-  EUPS_DISTRIB_VERSION: "w_2024_16"
+  EUPS_DISTRIB_VERSION: "w_2024_32"
 
 jobs:
   test:
@@ -21,9 +21,8 @@ jobs:
       - name: Cache dependencies
         uses: actions/cache@v2
         with:
-          path: ${{ github.workspace }}/eups/
+          path: ${{ github.workspace }}/conda/
           key: ${{ runner.os }}-eups-${{ env.EUPS_DISTRIB_VERSION }}-${{ hashFiles('**/*.txt') }}
-          restore-keys: ${{ runner.os }}-eups-
       - name: Install dependencies
         run: |
           curl -OL https://ls.st/lsstinstall
@@ -39,7 +38,12 @@ jobs:
           setup obs_lsst
           cd ./tests
           pytest test_move_embargo_args.py
-
+  build:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v3
       - name: docker build
         run: |
           docker build . \

Dockerfile

@@ -1,35 +1,45 @@
-# Dockerfile
-FROM python:3.11
+## Dockerfile
+ARG RUBINENV_VERSION=9.0.0
+FROM ghcr.io/lsst-dm/docker-newinstall:9-latest-${RUBINENV_VERSION}
+## workdir is /opt/lsst/software/stack
 
-# Copy source code and test files
+## Copy source code and test files
 COPY requirements.txt /opt/lsst/transfer_embargo/
 COPY src/ /opt/lsst/transfer_embargo/src/
 COPY tests_docker/ /opt/lsst/transfer_embargo/tests_docker/
 
-# Set the working directory
+## Set the working directory
 WORKDIR /opt/lsst/transfer_embargo
 
-# List files for debugging
+## Optionally list files for debugging
 # RUN ls -la /opt/lsst/transfer_embargo/
-# RUN ls -R /opt/lsst/transfer_embargo/src/
-# RUN ls -R /opt/lsst/transfer_embargo/tests_docker/
-# RUN ls -R /opt/lsst/transfer_embargo/tests/data/test_from/
 
-RUN pip install -r requirements.txt
+ARG OBS_LSST_VERSION
+ENV OBS_LSST_VERSION=${OBS_LSST_VERSION:-w_2024_32}
 
-# Define the environment variables
-# These are written over if they are re-defined
-# by the cronjob or on the command line deploy
-# of the pod
+## Optionally debug eups
+# RUN command -v eups || echo "eups not found in PATH"
+
+## Optionally check that bash is available and loadLSST.bash works
+# RUN ls -la /opt/lsst/software/stack/loadLSST.bash
+# RUN command -v bash || echo "bash not found, installing bash"
+
+## Run in a bash shell
+RUN <<EOF
+set -e
+source /opt/lsst/software/stack/loadLSST.bash
+command -v eups || echo "eups not found in PATH"
+eups distrib install -t "${OBS_LSST_VERSION}" obs_lsst
+EOF
+
+## Define the environment variables
+## These are written over if they are re-defined
+## by the pod deployment yaml or cli
 ENV FROMREPO "tests_docker/temp_from"
 ENV TOREPO "tests_docker/temp_to"
 ENV INSTRUMENT "LATISS"
-ENV NOW "2020-03-01 23:59:59.999999"
-ENV EMBARGO_HRS "7200"
-ENV DATAQUERIES '{ "datasettype": "raw", "collections": "LATISS/raw/all"}'
-ENV LOG "True"
+ENV DATAQUERIES "--dataqueries '{ \"datasettype\": \"raw\", \"collections\": \"LATISS/raw/all\"}'"
 ENV PASTEMBARGO "1.0"
+ENV OTHER_ARGUMENTS "--embargohours 80 --nowtime now"
 
-#CMD ["/bin/sh", "-c", "python src/move_embargo_args.py \"$FROMREPO\" \"$TOREPO\" \"$INSTRUMENT\" --log \"$LOG\""]
-
-CMD ["/bin/sh", "-c", "python src/move_embargo_args.py \"$FROMREPO\" \"$TOREPO\" \"$INSTRUMENT\" --nowtime \"$NOW\" --embargohours \"$EMBARGO_HRS\" --log \"$LOG\" --pastembargohours \"$PASTEMBARGO\" --dataqueries \"$DATAQUERIES\""]
+ENTRYPOINT [ "bash", "-c", "source /opt/lsst/software/stack/loadLSST.bash; setup lsst_obs; python src/move_embargo_args.py \"$FROMREPO\" \"$TOREPO\" \"$INSTRUMENT\" --log \"$LOG\" --pastembargohours \"$PASTEMBARGO\" $DATAQUERIES $OTHER_ARGUMENTS" ]

commands.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+bash -c source /opt/lsst/software/stack/loadLSST.bash
+setup lsst_obs
+python src/move_embargo_args.py $FROMREPO $TOREPO $INSTRUMENT --log $LOG --pastembargohours $PASTEMBARGO $DATAQUERIES $OTHER_ARGUMENTS

src/move_embargo_args.py

@@ -454,6 +454,7 @@ def parse_args():
                         butler, datasetRefs_exposure
                     )
                     dest_butler.ingest(*filedataset_list, transfer="direct")
+                    logger.info("did the ingest for raws")
                 except IndexError:
                     # this will be thrown if nothing is being moved
                     logger.info("nothing in datasetRefs_exposure")