Merge pull request #41 from broadinstitute/dp-trs-build

continuous deploy to GCS

.travis.yml

@@ -40,13 +40,13 @@ jobs:
       script:
         - set -e
         - travis/version-wdl-runtimes.sh
-        - travis/validate-wdl-miniwdl.sh
+        - miniwdl check pipes/WDL/workflows/*.wdl
 
     - language: java
       jdk: openjdk11
-      stage: validate
+      stage: build
       env:
-        - TRAVIS_JOB=validate_cromwell
+        - TRAVIS_JOB=validate_wdl_womtool
       install:
         - travis/install-wdl.sh
       script:
@@ -101,16 +101,44 @@ jobs:
     - language: python
       stage: build
       env:
-        - TRAVIS_JOB=deploy_dockstore
-      install:
+        - TRAVIS_JOB=deploy_gcs
+      script: skip
+      before_deploy:
         - set -e
+        - openssl aes-256-cbc -K $encrypted_6def3e87f286_key -iv $encrypted_6def3e87f286_iv -in travis/viral-ngs-wdl.json.enc -out travis/viral-ngs-wdl.json -d
+        - export DEST_DIR=`travis/list-docker-tags.sh | tail -1 | sed 's/:/\//'`
         - source travis/install-conda.sh
-        - conda install -y miniwdl shellcheck
-      script:
-        - set -e
+        - conda install -y miniwdl
         - travis/version-wdl-runtimes.sh
         - travis/flatten-wdls.sh
-        - echo "actual dockstore deploy not implemented yet"
+      deploy:
+          provider:   gcs
+          edge:       true
+          key_file:   travis/viral-ngs-wdl.json
+          bucket:     viral-ngs-wdl
+          local_dir:  pipes/WDL/flattened
+          upload_dir: "$DEST_DIR"
+          acl:        public-read
+          on:
+              all_branches: true
+
+    #- language: python
+    #  stage: build
+    #  env:
+    #    - TRAVIS_JOB=deploy_github
+    #  before_deploy:
+    #    - set -e
+    #    - source travis/install-conda.sh
+    #    - conda install -y miniwdl
+    #    - travis/version-wdl-runtimes.sh
+    #    - travis/flatten-wdls.sh
+    #  deploy:
+    #      provider:  releases
+    #      edge:      true
+    #      token:     "$GITHUB_OAUTH_TOKEN"
+    #      file: pipes/WDL/flattened/*.wdl
+    #      on:
+    #        tags: true
 
     - language: python
       stage:    build

README.md

@@ -1,13 +1,32 @@
 [![Build Status](https://travis-ci.com/broadinstitute/viral-pipelines.svg?branch=master)](https://travis-ci.com/broadinstitute/viral-pipelines)
 [![Documentation Status](https://readthedocs.org/projects/viral-pipelines/badge/?version=latest)](http://viral-pipelines.readthedocs.io/en/latest/?badge=latest)
-<!--
-[![broad-viral-badge](https://img.shields.io/badge/install%20from-broad--viral-green.svg?style=flat-square)](https://anaconda.org/broad-viral/viral-ngs)
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.252549.svg)](https://doi.org/10.5281/zenodo.252549)
--->
 
 viral-pipelines
-=========
+===============
 
 A set of scripts and tools for the analysis of viral NGS data.
 
-More detailed documentation can be found at http://viral-pipelines.readthedocs.org/
+Workflows are written in [WDL](https://github.com/openwdl/wdl) format, currently in a mix of WDL "draft-2" and "1.0". This is a portable workflow language that allows for easy execution on a wide variety of platforms:
+ - on individual machines (using miniWDL or Cromwell to execute)
+ - on commercial cloud platforms like GCP, AWS, or Azure (using Cromwell or CromwellOnAzure)
+ - on institutional HPC systems (using Cromwell)
+ - on commercial platform as a service vendors (like DNAnexus)
+ - on academic cloud platforms (like Terra)
+
+Currently, all workflows are regularly deployed to a GCS bucket: [gs://viral-ngs-wdl](https://console.cloud.google.com/storage/browser/viral-ngs-wdl?forceOnBucketsSortingFiltering=false&organizationId=548622027621&project=gcid-viral-seq). 
+
+Workflows are also available in the [Terra featured workspace](https://app.terra.bio/#workspaces/pathogen-genomic-surveillance/COVID-19).
+
+Workflows are continuously deployed to a [DNAnexus CI project](https://platform.dnanexus.com/projects/F8PQ6380xf5bK0Qk0YPjB17P).
+
+Continuous deploy to [Dockstore](https://dockstore.org/) is pending.
+
+Basic execution
+---------------
+
+The easiest way to get started is on a single, Docker-capable machine (your laptop, shared workstation, or virtual machine) using [miniWDL](https://github.com/chanzuckerberg/miniwdl). MiniWDL can be installed via `pip` or `conda` (via conda-forge). After confirming that it works (`miniwdl run_self_test`, you can use [miniwdl run](https://github.com/chanzuckerberg/miniwdl#miniwdl-run) to invoke WDL workflows from this repository. For example: `miniwdl run https://storage.googleapis.com/viral-ngs-wdl/quay.io/broadinstitute/viral-pipelines/2.0.20.3/assemble_refbased.wdl` will execute the reference-based assembly pipeline, when provided with the appropriate inputs.
+
+Available workflows
+-------------------
+
+Descriptions pending

travis/flatten-wdls.sh

@@ -9,7 +9,4 @@ for workflow in pipes/WDL/workflows/*.wdl; do
 	out_fn="$FLAT_DIR/$wf_base"
 	echo "flattening $workflow to $out_fn"
 	travis/paste_wdl_imports.py -o $out_fn $workflow
-
-	### DEBUG
-	cat $out_fn
 done