From 6e5f326688e29b00ca888e80606cde2c21a8b627 Mon Sep 17 00:00:00 2001
From: aliciaaevans <alicia.evans@nih.gov>
Date: Fri, 28 Jul 2023 13:50:18 -0400
Subject: [PATCH 1/7] move deploy to orb
---
.circleci/config.yml | 51 ++++++++++++++++++++++++++++++++------------
1 file changed, 37 insertions(+), 14 deletions(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 81245109..56a4dc13 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,5 +1,32 @@
version: 2
+orbs:
+ lcdb-wf:
+ commands:
+ deploy-clone:
+ parameters:
+ flavor:
+ default: "full"
+ type: string
+ steps:
+ - run:
+ name: Deploy to a new directory, cloning to a temp dir
+ command: |
+ source /opt/mambaforge/etc/profile.d/conda.sh
+ conda activate $LCDBWF_ENV
+ conda info --envs
+ conda config --show
+
+ # Copy the deploy script to a different location to simulate the
+ # suggested deployment method of downloading just the script.
+ cp deploy.py /tmp/deploy.py
+ cd /tmp/
+
+ # Deploy to the new directory, so we are testing the real-world case of post-deployment.
+ # Note that $DEPLOY is set in the "set-paths" step configured above.
+ python deploy.py --flavor <<parameters.flavor>> --dest $DEPLOY --branch $CIRCLE_BRANCH --clone
+
+
variables:
# default settings for all steps
@@ -127,20 +154,6 @@ variables:
run:
name: Download example data
command: |
- source /opt/mambaforge/etc/profile.d/conda.sh
- conda activate $LCDBWF_ENV
- conda info --envs
- conda config --show
-
- # Copy the deploy script to a different location to simulate the
- # suggested deployment method of downloading just the script.
- cp deploy.py /tmp/deploy.py
- cd /tmp/
-
- # Deploy to the new directory, so we are testing the real-world case of post-deployment.
- # Note that $DEPLOY is set in the "set-paths" step configured above.
- python deploy.py --flavor full --dest $DEPLOY --branch $CIRCLE_BRANCH --clone
-
set -x
tree $DEPLOY
tree $ORIG
@@ -362,6 +375,8 @@ jobs:
- checkout
- *restore_cache
- *set-path
+ - deploy-lcdb-wf/deploy-clone:
+ flavor: chipseq
- *get-data
- *chipseq-step
- store_artifacts:
@@ -373,6 +388,8 @@ jobs:
- checkout
- *restore_cache
- *set-path
+ - deploy-lcdb-wf/deploy-clone:
+ flavor: chipseq
- *get-data
- *chipseq-misc-step
@@ -382,6 +399,8 @@ jobs:
- checkout
- *restore_cache
- *set-path
+ - deploy-lcdb-wf/deploy-clone:
+ flavor: rnaseq
- *get-data
- *rnaseq-step
- store_artifacts:
@@ -407,6 +426,8 @@ jobs:
- checkout
- *restore_cache
- *set-path
+ - deploy-lcdb-wf/deploy-clone:
+ flavor: rnaseq
- *get-data
- *rnaseq-misc-step
@@ -416,6 +437,7 @@ jobs:
- checkout
- *restore_cache
- *set-path
+ - deploy-lcdb-wf/deploy-clone
- *get-data
- *colocalization-step
@@ -425,6 +447,7 @@ jobs:
- checkout
- *restore_cache
- *set-path
+ - deploy-lcdb-wf/deploy-clone
- *get-data
- *references-step
From d41dd48c3faa18782ca7ec44db8c8902f3df0b71 Mon Sep 17 00:00:00 2001
From: aliciaaevans <alicia.evans@nih.gov>
Date: Fri, 28 Jul 2023 14:08:05 -0400
Subject: [PATCH 2/7] orbs require version 2.1
---
.circleci/config.yml | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 56a4dc13..59dde9ea 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,4 +1,4 @@
-version: 2
+version: 2.1
orbs:
lcdb-wf:
@@ -375,8 +375,8 @@ jobs:
- checkout
- *restore_cache
- *set-path
- - deploy-lcdb-wf/deploy-clone:
- flavor: chipseq
+ - lcdb-wf/deploy-clone:
+ flavor: chipseq
- *get-data
- *chipseq-step
- store_artifacts:
@@ -388,8 +388,8 @@ jobs:
- checkout
- *restore_cache
- *set-path
- - deploy-lcdb-wf/deploy-clone:
- flavor: chipseq
+ - lcdb-wf/deploy-clone:
+ flavor: chipseq
- *get-data
- *chipseq-misc-step
@@ -399,8 +399,8 @@ jobs:
- checkout
- *restore_cache
- *set-path
- - deploy-lcdb-wf/deploy-clone:
- flavor: rnaseq
+ - lcdb-wf/deploy-clone:
+ flavor: rnaseq
- *get-data
- *rnaseq-step
- store_artifacts:
@@ -426,8 +426,8 @@ jobs:
- checkout
- *restore_cache
- *set-path
- - deploy-lcdb-wf/deploy-clone:
- flavor: rnaseq
+ - lcdb-wf/deploy-clone:
+ flavor: rnaseq
- *get-data
- *rnaseq-misc-step
@@ -437,7 +437,7 @@ jobs:
- checkout
- *restore_cache
- *set-path
- - deploy-lcdb-wf/deploy-clone
+ - lcdb-wf/deploy-clone
- *get-data
- *colocalization-step
@@ -447,7 +447,7 @@ jobs:
- checkout
- *restore_cache
- *set-path
- - deploy-lcdb-wf/deploy-clone
+ - lcdb-wf/deploy-clone
- *get-data
- *references-step
From dd5481ee3d8887ede49ef427640e489432a2c0d6 Mon Sep 17 00:00:00 2001
From: aliciaaevans <alicia.evans@nih.gov>
Date: Fri, 28 Jul 2023 15:50:46 -0400
Subject: [PATCH 3/7] move get-data to orb
---
.circleci/config.yml | 80 ++++++++++++++++++++++++--------------------
1 file changed, 43 insertions(+), 37 deletions(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 59dde9ea..fdefcf1e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -25,6 +25,39 @@ orbs:
# Deploy to the new directory, so we are testing the real-world case of post-deployment.
# Note that $DEPLOY is set in the "set-paths" step configured above.
python deploy.py --flavor <<parameters.flavor>> --dest $DEPLOY --branch $CIRCLE_BRANCH --clone
+ get-data:
+ parameters:
+ kind:
+ default: "all"
+ type: string
+ steps:
+ - run:
+ name: Download example data
+ command: |
+ set -x
+ tree $DEPLOY
+ tree $ORIG
+ set +x
+
+ # Separately copy over some test-specific files
+ cp $ORIG/workflows/chipseq/run_test.sh $DEPLOY/workflows/chipseq/run_test.sh
+ cp $ORIG/workflows/rnaseq/run_test.sh $DEPLOY/workflows/rnaseq/run_test.sh
+ cp $ORIG/workflows/rnaseq/run_downstream_test.sh $DEPLOY/workflows/rnaseq/run_downstream_test.sh
+ cp $ORIG/workflows/colocalization/run_test.sh $DEPLOY/workflows/references/run_test.sh
+ cp $ORIG/workflows/colocalization/run_test.sh $DEPLOY/workflows/colocalization/run_test.sh
+
+ mkdir $DEPLOY/ci
+ mkdir $DEPLOY/test
+ cp $ORIG/test/lcdb-wf-test $DEPLOY/test/lcdb-wf-test
+ cp $ORIG/test/workflow_test_params.yaml $DEPLOY/test/workflow_test_params.yaml
+ cp $ORIG/ci/get-data.py $DEPLOY/ci/get-data.py
+
+ # the ./run_test.sh scripts run this
+ cp $ORIG/ci/preprocessor.py $DEPLOY/ci/preprocessor.py
+
+ # download example data
+ cd $DEPLOY
+ test/lcdb-wf-test data --kind=<<parameters.kind>> --verbose
variables:
@@ -148,37 +181,6 @@ variables:
time mamba env create -n $LCDBWF_ENV_R --file env-r.yml
fi
- # --------------------------------------------------------------------------
- # Deploy into a new directory and get the test data.
- get-data: &get-data
- run:
- name: Download example data
- command: |
- set -x
- tree $DEPLOY
- tree $ORIG
- set +x
-
- # Separately copy over some test-specific files
- cp $ORIG/workflows/chipseq/run_test.sh $DEPLOY/workflows/chipseq/run_test.sh
- cp $ORIG/workflows/rnaseq/run_test.sh $DEPLOY/workflows/rnaseq/run_test.sh
- cp $ORIG/workflows/rnaseq/run_downstream_test.sh $DEPLOY/workflows/rnaseq/run_downstream_test.sh
- cp $ORIG/workflows/colocalization/run_test.sh $DEPLOY/workflows/references/run_test.sh
- cp $ORIG/workflows/colocalization/run_test.sh $DEPLOY/workflows/colocalization/run_test.sh
-
- mkdir $DEPLOY/ci
- mkdir $DEPLOY/test
- cp $ORIG/test/lcdb-wf-test $DEPLOY/test/lcdb-wf-test
- cp $ORIG/test/workflow_test_params.yaml $DEPLOY/test/workflow_test_params.yaml
- cp $ORIG/ci/get-data.py $DEPLOY/ci/get-data.py
-
- # the ./run_test.sh scripts run this
- cp $ORIG/ci/preprocessor.py $DEPLOY/ci/preprocessor.py
-
- # download example data
- cd $DEPLOY
- test/lcdb-wf-test data --kind=all --verbose
-
# --------------------------------------------------------------------------
# Run the doctests across the included modules
pytest-step: &pytest-step
@@ -377,7 +379,8 @@ jobs:
- *set-path
- lcdb-wf/deploy-clone:
flavor: chipseq
- - *get-data
+ - lcdb-wf/get-data:
+ kind: chipseq
- *chipseq-step
- store_artifacts:
path: /tmp/lcdb-wf-test/workflows/chipseq/data/chipseq_aggregation/multiqc.html
@@ -390,7 +393,8 @@ jobs:
- *set-path
- lcdb-wf/deploy-clone:
flavor: chipseq
- - *get-data
+ - lcdb-wf/get-data:
+ kind: chipseq
- *chipseq-misc-step
rnaseq:
@@ -401,7 +405,8 @@ jobs:
- *set-path
- lcdb-wf/deploy-clone:
flavor: rnaseq
- - *get-data
+ - lcdb-wf/get-data:
+ kind: rnaseq
- *rnaseq-step
- store_artifacts:
path: /tmp/downstream.tar.gz
@@ -428,7 +433,8 @@ jobs:
- *set-path
- lcdb-wf/deploy-clone:
flavor: rnaseq
- - *get-data
+ - lcdb-wf/get-data:
+ kind: rnaseq
- *rnaseq-misc-step
colocalization:
@@ -438,7 +444,7 @@ jobs:
- *restore_cache
- *set-path
- lcdb-wf/deploy-clone
- - *get-data
+ - lcdb-wf/get-data
- *colocalization-step
references:
@@ -448,7 +454,7 @@ jobs:
- *restore_cache
- *set-path
- lcdb-wf/deploy-clone
- - *get-data
+ - lcdb-wf/get-data
- *references-step
build-docs:
From c47e477af045ffd4e25cf32cbe508084d8625e4a Mon Sep 17 00:00:00 2001
From: aliciaaevans <alicia.evans@nih.gov>
Date: Fri, 28 Jul 2023 17:12:46 -0400
Subject: [PATCH 4/7] params for workflow folder
---
.circleci/config.yml | 24 +++++++++++++++++-------
1 file changed, 17 insertions(+), 7 deletions(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
index fdefcf1e..4e6c2eac 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -29,6 +29,9 @@ orbs:
parameters:
kind:
default: "all"
+ type: enum
+ enum: ["all", "rnaseq", "chipseq"]
+ workflow:
type: string
steps:
- run:
@@ -40,11 +43,12 @@ orbs:
set +x
# Separately copy over some test-specific files
- cp $ORIG/workflows/chipseq/run_test.sh $DEPLOY/workflows/chipseq/run_test.sh
- cp $ORIG/workflows/rnaseq/run_test.sh $DEPLOY/workflows/rnaseq/run_test.sh
- cp $ORIG/workflows/rnaseq/run_downstream_test.sh $DEPLOY/workflows/rnaseq/run_downstream_test.sh
- cp $ORIG/workflows/colocalization/run_test.sh $DEPLOY/workflows/references/run_test.sh
- cp $ORIG/workflows/colocalization/run_test.sh $DEPLOY/workflows/colocalization/run_test.sh
+
+ cp $ORIG/workflows/<<parameters.workflow>>/run_test.sh $DEPLOY/workflows/<<parameters.workflow>>/run_test.sh
+
+ if ["<<parameters.workflow>>" -eq "rnaseq"]; then
+ cp $ORIG/workflows/rnaseq/run_downstream_test.sh $DEPLOY/workflows/rnaseq/run_downstream_test.sh
+ fi
mkdir $DEPLOY/ci
mkdir $DEPLOY/test
@@ -381,6 +385,7 @@ jobs:
flavor: chipseq
- lcdb-wf/get-data:
kind: chipseq
+ workflow: chipseq
- *chipseq-step
- store_artifacts:
path: /tmp/lcdb-wf-test/workflows/chipseq/data/chipseq_aggregation/multiqc.html
@@ -395,6 +400,7 @@ jobs:
flavor: chipseq
- lcdb-wf/get-data:
kind: chipseq
+ workflow: chipseq
- *chipseq-misc-step
rnaseq:
@@ -407,6 +413,7 @@ jobs:
flavor: rnaseq
- lcdb-wf/get-data:
kind: rnaseq
+ workflow: rnaseq
- *rnaseq-step
- store_artifacts:
path: /tmp/downstream.tar.gz
@@ -435,6 +442,7 @@ jobs:
flavor: rnaseq
- lcdb-wf/get-data:
kind: rnaseq
+ workflow: rnaseq
- *rnaseq-misc-step
colocalization:
@@ -444,7 +452,8 @@ jobs:
- *restore_cache
- *set-path
- lcdb-wf/deploy-clone
- - lcdb-wf/get-data
+ - lcdb-wf/get-data:
+ workflow: colocalization
- *colocalization-step
references:
@@ -454,7 +463,8 @@ jobs:
- *restore_cache
- *set-path
- lcdb-wf/deploy-clone
- - lcdb-wf/get-data
+ - lcdb-wf/get-data:
+ workflow: references
- *references-step
build-docs:
From aeb804a582f513784c31b2c4d1428a6b5dcc0d73 Mon Sep 17 00:00:00 2001
From: aliciaaevans <alicia.evans@nih.gov>
Date: Tue, 1 Aug 2023 15:35:54 -0400
Subject: [PATCH 5/7] use correct operator
---
.circleci/config.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 4e6c2eac..dd80173d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -46,7 +46,7 @@ orbs:
cp $ORIG/workflows/<<parameters.workflow>>/run_test.sh $DEPLOY/workflows/<<parameters.workflow>>/run_test.sh
- if ["<<parameters.workflow>>" -eq "rnaseq"]; then
+ if ["<<parameters.workflow>>" = "rnaseq"]; then
cp $ORIG/workflows/rnaseq/run_downstream_test.sh $DEPLOY/workflows/rnaseq/run_downstream_test.sh
fi
From 515b848b12c40dbdb0e98b0af4e2506614152ff6 Mon Sep 17 00:00:00 2001
From: aliciaaevans <alicia.evans@nih.gov>
Date: Tue, 1 Aug 2023 16:07:49 -0400
Subject: [PATCH 6/7] fix if statement
---
.circleci/config.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
index dd80173d..422550ca 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -46,7 +46,7 @@ orbs:
cp $ORIG/workflows/<<parameters.workflow>>/run_test.sh $DEPLOY/workflows/<<parameters.workflow>>/run_test.sh
- if ["<<parameters.workflow>>" = "rnaseq"]; then
+ if [["<<parameters.workflow>>" = "rnaseq"]]; then
cp $ORIG/workflows/rnaseq/run_downstream_test.sh $DEPLOY/workflows/rnaseq/run_downstream_test.sh
fi
From 16663cb5d7215e2969fe1878886716abc4f00839 Mon Sep 17 00:00:00 2001
From: aliciaaevans <alicia.evans@nih.gov>
Date: Tue, 1 Aug 2023 16:27:54 -0400
Subject: [PATCH 7/7] fix if statement
---
.circleci/config.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 422550ca..119e2eff 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -46,7 +46,7 @@ orbs:
cp $ORIG/workflows/<<parameters.workflow>>/run_test.sh $DEPLOY/workflows/<<parameters.workflow>>/run_test.sh
- if [["<<parameters.workflow>>" = "rnaseq"]]; then
+ if [[ "<<parameters.workflow>>" = "rnaseq" ]]; then
cp $ORIG/workflows/rnaseq/run_downstream_test.sh $DEPLOY/workflows/rnaseq/run_downstream_test.sh
fi