Work in progress: CircleCI inline orbs

.circleci/config.yml

@@ -1,4 +1,68 @@
-version: 2
+version: 2.1
+
+orbs:
+  lcdb-wf:
+    commands:
+      deploy-clone:
+        parameters:
+          flavor:
+            default: "full"
+            type: string
+        steps:
+          - run:
+              name: Deploy to a new directory, cloning to a temp dir
+              command: |
+                source /opt/mambaforge/etc/profile.d/conda.sh
+                conda activate $LCDBWF_ENV
+                conda info --envs
+                conda config --show
+
+                # Copy the deploy script to a different location to simulate the
+                # suggested deployment method of downloading just the script.
+                cp deploy.py /tmp/deploy.py
+                cd /tmp/
+
+                # Deploy to the new directory, so we are testing the real-world case of post-deployment.
+                # Note that $DEPLOY is set in the "set-paths" step configured above.
+                python deploy.py --flavor <<parameters.flavor>> --dest $DEPLOY --branch $CIRCLE_BRANCH --clone
+      get-data:
+        parameters:
+          kind:
+            default: "all"
+            type: enum
+            enum: ["all", "rnaseq", "chipseq"]
+          workflow:
+            type: string
+        steps:
+          - run:
+              name: Download example data
+              command: |
+                set -x
+                tree $DEPLOY
+                tree $ORIG
+                set +x
+
+                # Separately copy over some test-specific files
+
+                cp $ORIG/workflows/<<parameters.workflow>>/run_test.sh $DEPLOY/workflows/<<parameters.workflow>>/run_test.sh
+
+                if [[ "<<parameters.workflow>>" = "rnaseq" ]]; then
+                  cp $ORIG/workflows/rnaseq/run_downstream_test.sh $DEPLOY/workflows/rnaseq/run_downstream_test.sh
+                fi
+
+                mkdir $DEPLOY/ci
+                mkdir $DEPLOY/test
+                cp $ORIG/test/lcdb-wf-test $DEPLOY/test/lcdb-wf-test
+                cp $ORIG/test/workflow_test_params.yaml $DEPLOY/test/workflow_test_params.yaml
+                cp $ORIG/ci/get-data.py $DEPLOY/ci/get-data.py
+
+                # the ./run_test.sh scripts run this
+                cp $ORIG/ci/preprocessor.py $DEPLOY/ci/preprocessor.py
+
+                # download example data
+                cd $DEPLOY
+                test/lcdb-wf-test data --kind=<<parameters.kind>> --verbose
+
 
 variables:
 
@@ -121,51 +185,6 @@ variables:
             time mamba env create -n $LCDBWF_ENV_R --file env-r.yml
         fi
 
-  # --------------------------------------------------------------------------
-  # Deploy into a new directory and get the test data.
-  get-data: &get-data
-    run:
-      name: Download example data
-      command: |
-        source /opt/mambaforge/etc/profile.d/conda.sh
-        conda activate $LCDBWF_ENV
-        conda info --envs
-        conda config --show
-
-        # Copy the deploy script to a different location to simulate the
-        # suggested deployment method of downloading just the script.
-        cp deploy.py /tmp/deploy.py
-        cd /tmp/
-
-        # Deploy to the new directory, so we are testing the real-world case of post-deployment.
-        # Note that $DEPLOY is set in the "set-paths" step configured above.
-        python deploy.py --flavor full --dest $DEPLOY --branch $CIRCLE_BRANCH --clone
-
-        set -x
-        tree $DEPLOY
-        tree $ORIG
-        set +x
-
-        # Separately copy over some test-specific files
-        cp $ORIG/workflows/chipseq/run_test.sh $DEPLOY/workflows/chipseq/run_test.sh
-        cp $ORIG/workflows/rnaseq/run_test.sh $DEPLOY/workflows/rnaseq/run_test.sh
-        cp $ORIG/workflows/rnaseq/run_downstream_test.sh $DEPLOY/workflows/rnaseq/run_downstream_test.sh
-        cp $ORIG/workflows/colocalization/run_test.sh $DEPLOY/workflows/references/run_test.sh
-        cp $ORIG/workflows/colocalization/run_test.sh $DEPLOY/workflows/colocalization/run_test.sh
-
-        mkdir $DEPLOY/ci
-        mkdir $DEPLOY/test
-        cp $ORIG/test/lcdb-wf-test $DEPLOY/test/lcdb-wf-test
-        cp $ORIG/test/workflow_test_params.yaml $DEPLOY/test/workflow_test_params.yaml
-        cp $ORIG/ci/get-data.py $DEPLOY/ci/get-data.py
-
-        # the ./run_test.sh scripts run this
-        cp $ORIG/ci/preprocessor.py $DEPLOY/ci/preprocessor.py
-
-        # download example data
-        cd $DEPLOY
-        test/lcdb-wf-test data --kind=all --verbose
-
   # --------------------------------------------------------------------------
   # Run the doctests across the included modules
   pytest-step: &pytest-step
@@ -362,7 +381,11 @@ jobs:
       - checkout
       - *restore_cache
       - *set-path
-      - *get-data
+      - lcdb-wf/deploy-clone:
+          flavor: chipseq
+      - lcdb-wf/get-data:
+          kind: chipseq
+          workflow: chipseq
       - *chipseq-step
       - store_artifacts:
           path: /tmp/lcdb-wf-test/workflows/chipseq/data/chipseq_aggregation/multiqc.html
@@ -373,7 +396,11 @@ jobs:
       - checkout
       - *restore_cache
       - *set-path
-      - *get-data
+      - lcdb-wf/deploy-clone:
+          flavor: chipseq
+      - lcdb-wf/get-data:
+          kind: chipseq
+          workflow: chipseq
       - *chipseq-misc-step
 
   rnaseq:
@@ -382,7 +409,11 @@ jobs:
       - checkout
       - *restore_cache
       - *set-path
-      - *get-data
+      - lcdb-wf/deploy-clone:
+          flavor: rnaseq
+      - lcdb-wf/get-data:
+          kind: rnaseq
+          workflow: rnaseq
       - *rnaseq-step
       - store_artifacts:
           path: /tmp/downstream.tar.gz
@@ -407,7 +438,11 @@ jobs:
       - checkout
       - *restore_cache
       - *set-path
-      - *get-data
+      - lcdb-wf/deploy-clone:
+          flavor: rnaseq
+      - lcdb-wf/get-data:
+          kind: rnaseq
+          workflow: rnaseq
       - *rnaseq-misc-step
 
   colocalization:
@@ -416,7 +451,9 @@ jobs:
       - checkout
       - *restore_cache
       - *set-path
-      - *get-data
+      - lcdb-wf/deploy-clone
+      - lcdb-wf/get-data:
+          workflow: colocalization
       - *colocalization-step
 
   references:
@@ -425,7 +462,9 @@ jobs:
       - checkout
       - *restore_cache
       - *set-path
-      - *get-data
+      - lcdb-wf/deploy-clone
+      - lcdb-wf/get-data:
+          workflow: references
       - *references-step
 
   build-docs: