diff --git a/.github/workflows/warp_test_workflow.yml b/.github/workflows/warp_test_workflow.yml index 8821240785..de4de52698 100644 --- a/.github/workflows/warp_test_workflow.yml +++ b/.github/workflows/warp_test_workflow.yml @@ -211,6 +211,7 @@ jobs: done echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!" + exit 1 env: GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }} @@ -249,19 +250,18 @@ jobs: TEST_TYPE="${{ env.testType }}" INPUTS_DIR="${{ inputs.pipeline_dir }}/test_inputs/$TEST_TYPE" echo "Running tests with test type: $TEST_TYPE" - - TRUTH_PATH="gs://broad-gotc-test-storage/${{ inputs.dockstore_pipeline_name }}/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH" - echo "Truth path: $TRUTH_PATH" RESULTS_PATH="gs://pd-test-results/${{ inputs.dockstore_pipeline_name }}/results/$CURRENT_TIME" + # Submit all jobs first and store their submission IDs for input_file in "$INPUTS_DIR"/*.json; do - test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \ + test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py \ --results_path "$RESULTS_PATH" \ --inputs_json "$input_file" \ --update_truth "$UPDATE_TRUTH_BOOL" \ - --branch_name "$BRANCH_NAME" ) + --branch_name "$TRUTH_BRANCH" \ + --dockstore_pipeline_name "${{ inputs.dockstore_pipeline_name }}" ) echo "Uploading the test input file: $test_input_file" # Create the submission_data.json file for this input_file diff --git a/pipelines/broad/arrays/imputation/test_inputs/Plumbing/NA12878_trio.json b/pipelines/broad/arrays/imputation/test_inputs/Plumbing/NA12878_trio.json index 2857a6f576..6830a61534 100644 --- a/pipelines/broad/arrays/imputation/test_inputs/Plumbing/NA12878_trio.json +++ b/pipelines/broad/arrays/imputation/test_inputs/Plumbing/NA12878_trio.json @@ -1,13 +1,13 @@ { "Imputation.single_sample_vcfs": [ - "gs://broad-gotc-test-storage/imputation/plumbing/vcfs/101342370027_R02C01.vcf.gz", - "gs://broad-gotc-test-storage/imputation/plumbing/vcfs/101342370027_R12C02.vcf.gz", - "gs://broad-gotc-test-storage/imputation/plumbing/vcfs/101342370134_R12C02.vcf.gz" + "gs://pd-test-storage-private/Imputation/input/plumbing/NA12878_trio/101342370027_R02C01.vcf.gz", + "gs://pd-test-storage-private/Imputation/input/plumbing/NA12878_trio/101342370027_R12C02.vcf.gz", + "gs://pd-test-storage-private/Imputation/input/plumbing/NA12878_trio/101342370134_R12C02.vcf.gz" ], "Imputation.single_sample_vcf_indices": [ - "gs://broad-gotc-test-storage/imputation/plumbing/vcfs/101342370027_R02C01.vcf.gz.tbi", - "gs://broad-gotc-test-storage/imputation/plumbing/vcfs/101342370027_R12C02.vcf.gz.tbi", - "gs://broad-gotc-test-storage/imputation/plumbing/vcfs/101342370134_R12C02.vcf.gz.tbi" + "gs://pd-test-storage-private/Imputation/input/plumbing/NA12878_trio/101342370027_R02C01.vcf.gz.tbi", + "gs://pd-test-storage-private/Imputation/input/plumbing/NA12878_trio/101342370027_R12C02.vcf.gz.tbi", + "gs://pd-test-storage-private/Imputation/input/plumbing/NA12878_trio/101342370134_R12C02.vcf.gz.tbi" ], "Imputation.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict", "Imputation.reference_panel_path": "gs://gcp-public-data--broad-references/hg19/v0/1000G_reference_panel/", diff --git a/pipelines/broad/arrays/imputation/test_inputs/Scientific/NA12878_trio_scientific.json b/pipelines/broad/arrays/imputation/test_inputs/Scientific/NA12878_trio_scientific.json index 0596ac2577..1c844afce5 100644 --- a/pipelines/broad/arrays/imputation/test_inputs/Scientific/NA12878_trio_scientific.json +++ b/pipelines/broad/arrays/imputation/test_inputs/Scientific/NA12878_trio_scientific.json @@ -1,18 +1,18 @@ { "Imputation.single_sample_vcfs": [ - "gs://broad-gotc-test-storage/imputation/scientific/vcfs/101342370027_R02C01.vcf.gz", - "gs://broad-gotc-test-storage/imputation/scientific/vcfs/101342370027_R12C02.vcf.gz", - "gs://broad-gotc-test-storage/imputation/scientific/vcfs/101342370134_R12C02.vcf.gz" + "gs://pd-test-storage-private/Imputation/input/scientific/NA12878_trio_scientific/101342370027_R02C01.vcf.gz", + "gs://pd-test-storage-private/Imputation/input/scientific/NA12878_trio_scientific/101342370027_R12C02.vcf.gz", + "gs://pd-test-storage-private/Imputation/input/scientific/NA12878_trio_scientific/101342370134_R12C02.vcf.gz" ], "Imputation.single_sample_vcf_indices": [ - "gs://broad-gotc-test-storage/imputation/scientific/vcfs/101342370027_R02C01.vcf.gz.tbi", - "gs://broad-gotc-test-storage/imputation/scientific/vcfs/101342370027_R12C02.vcf.gz.tbi", - "gs://broad-gotc-test-storage/imputation/scientific/vcfs/101342370134_R12C02.vcf.gz.tbi" + "gs://pd-test-storage-private/Imputation/input/scientific/NA12878_trio_scientific/101342370027_R02C01.vcf.gz.tbi", + "gs://pd-test-storage-private/Imputation/input/scientific/NA12878_trio_scientific/101342370027_R12C02.vcf.gz.tbi", + "gs://pd-test-storage-private/Imputation/input/scientific/NA12878_trio_scientific/101342370134_R12C02.vcf.gz.tbi" ], "Imputation.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict", "Imputation.reference_panel_path": "gs://gcp-public-data--broad-references/hg19/v0/1000G_reference_panel/", "Imputation.contigs": [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22" ], - "Imputation.genetic_maps_eagle": "gs://broad-gotc-test-storage/imputation/eagle_genetic_map/genetic_map_hg19_withX.txt.gz", + "Imputation.genetic_maps_eagle": "gs://pd-test-storage-private/Imputation/input/scientific/NA12878_trio_scientific/genetic_map_hg19_withX.txt.gz", "Imputation.output_callset_name": "scientific_test", "Imputation.split_output_to_single_sample": false, "Imputation.perform_extra_qc_steps": false, diff --git a/pipelines/broad/arrays/imputation/test_inputs/Scientific/multi_sample_split_test_data_NOT_PUBLIC.json b/pipelines/broad/arrays/imputation/test_inputs/Scientific/multi_sample_split_test_data_NOT_PUBLIC.json index 08f855bfd1..1c3dea1836 100644 --- a/pipelines/broad/arrays/imputation/test_inputs/Scientific/multi_sample_split_test_data_NOT_PUBLIC.json +++ b/pipelines/broad/arrays/imputation/test_inputs/Scientific/multi_sample_split_test_data_NOT_PUBLIC.json @@ -1,6 +1,6 @@ { - "Imputation.multi_sample_vcf": "gs://broad-gotc-test-storage/imputation/scientific/vcfs/merged_input_samples.vcf.gz", - "Imputation.multi_sample_vcf_index": "gs://broad-gotc-test-storage/imputation/scientific/vcfs/merged_input_samples.vcf.gz.tbi", + "Imputation.multi_sample_vcf": "gs://pd-test-storage-private/Imputation/input/scientific/multi_sample_split_test_data_NOT_PUBLIC/merged_input_samples.vcf.gz", + "Imputation.multi_sample_vcf_index": "gs://pd-test-storage-private/Imputation/input/scientific/multi_sample_split_test_data_NOT_PUBLIC/merged_input_samples.vcf.gz.tbi", "Imputation.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict", "Imputation.reference_panel_path": "gs://gcp-public-data--broad-references/hg19/v0/1000G_reference_panel/", "Imputation.contigs": [ "21", "22" ], diff --git a/pipelines/broad/arrays/imputation/test_inputs/Scientific/multi_sample_test_data_NOT_PUBLIC.json b/pipelines/broad/arrays/imputation/test_inputs/Scientific/multi_sample_test_data_NOT_PUBLIC.json index ce7aa3e1af..64f6776810 100644 --- a/pipelines/broad/arrays/imputation/test_inputs/Scientific/multi_sample_test_data_NOT_PUBLIC.json +++ b/pipelines/broad/arrays/imputation/test_inputs/Scientific/multi_sample_test_data_NOT_PUBLIC.json @@ -1,6 +1,6 @@ { - "Imputation.multi_sample_vcf": "gs://broad-gotc-test-storage/imputation/scientific/vcfs/merged_input_samples.vcf.gz", - "Imputation.multi_sample_vcf_index": "gs://broad-gotc-test-storage/imputation/scientific/vcfs/merged_input_samples.vcf.gz.tbi", + "Imputation.multi_sample_vcf": "gs://pd-test-storage-private/Imputation/input/scientific/multi_sample_test_data_NOT_PUBLIC/merged_input_samples.vcf.gz", + "Imputation.multi_sample_vcf_index": "gs://pd-test-storage-private/Imputation/input/scientific/multi_sample_test_data_NOT_PUBLIC/merged_input_samples.vcf.gz.tbi", "Imputation.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict", "Imputation.reference_panel_path": "gs://gcp-public-data--broad-references/hg19/v0/1000G_reference_panel/", "Imputation.contigs": [ "21", "22" ], diff --git a/pipelines/broad/arrays/imputation_beagle/test_inputs/Plumbing/NA12878_x10_hg38_arrays.json b/pipelines/broad/arrays/imputation_beagle/test_inputs/Plumbing/NA12878_x10_hg38_arrays.json index bdf5a00597..fa9d0eef73 100644 --- a/pipelines/broad/arrays/imputation_beagle/test_inputs/Plumbing/NA12878_x10_hg38_arrays.json +++ b/pipelines/broad/arrays/imputation_beagle/test_inputs/Plumbing/NA12878_x10_hg38_arrays.json @@ -1,8 +1,8 @@ { - "ImputationBeagle.multi_sample_vcf": "gs://broad-gotc-test-storage/imputation_beagle/scientific/vcfs/NA12878_10_duplicate.merged.cleaned.vcf.gz", + "ImputationBeagle.multi_sample_vcf": "gs://pd-test-storage-public/ImputationBeagle/input/plumbing/NA12878_x10_hg38_arrays/NA12878_10_duplicate.merged.cleaned.vcf.gz", "ImputationBeagle.ref_dict": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dict", - "ImputationBeagle.reference_panel_path_prefix": "gs://broad-gotc-test-storage/imputation_beagle/scientific/1000G_HGDP_no_singletons_reference_panel/hgdp.tgp.gwaspy.AN_added.bcf.ac2", + "ImputationBeagle.reference_panel_path_prefix": "gs://pd-test-storage-public/ImputationBeagle/input/plumbing/NA12878_x10_hg38_arrays/1000G_HGDP_no_singletons_reference_panel/hgdp.tgp.gwaspy.AN_added.bcf.ac2", "ImputationBeagle.contigs": ["chr21","chr22"], - "ImputationBeagle.genetic_maps_path": "gs://broad-gotc-test-storage/imputation_beagle/scientific/plink-genetic-maps/", + "ImputationBeagle.genetic_maps_path": "gs://pd-test-storage-public/ImputationBeagle/input/plumbing/NA12878_x10_hg38_arrays/plink-genetic-maps/", "ImputationBeagle.output_basename": "plumbing_test" } diff --git a/pipelines/broad/arrays/imputation_beagle/test_inputs/Scientific/NA12878_x10_hg38_arrays.json b/pipelines/broad/arrays/imputation_beagle/test_inputs/Scientific/NA12878_x10_hg38_arrays.json index 4263609e29..821b09dcf1 100644 --- a/pipelines/broad/arrays/imputation_beagle/test_inputs/Scientific/NA12878_x10_hg38_arrays.json +++ b/pipelines/broad/arrays/imputation_beagle/test_inputs/Scientific/NA12878_x10_hg38_arrays.json @@ -1,8 +1,8 @@ { - "ImputationBeagle.multi_sample_vcf": "gs://broad-gotc-test-storage/imputation_beagle/scientific/vcfs/NA12878_10_duplicate.merged.cleaned.vcf.gz", + "ImputationBeagle.multi_sample_vcf": "gs://pd-test-storage-public/ImputationBeagle/input/scientific/NA12878_x10_hg38_arrays/NA12878_10_duplicate.merged.cleaned.vcf.gz", "ImputationBeagle.ref_dict": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dict", - "ImputationBeagle.reference_panel_path_prefix": "gs://broad-gotc-test-storage/imputation_beagle/scientific/1000G_HGDP_no_singletons_reference_panel/hgdp.tgp.gwaspy.AN_added.bcf.ac2", + "ImputationBeagle.reference_panel_path_prefix": "gs://pd-test-storage-public/ImputationBeagle/input/scientific/NA12878_x10_hg38_arrays/1000G_HGDP_no_singletons_reference_panel/hgdp.tgp.gwaspy.AN_added.bcf.ac2", "ImputationBeagle.contigs": ["chr1","chr2","chr3","chr4","chr5","chr6","chr7","chr8","chr9","chr10","chr11","chr12","chr13","chr14","chr15","chr16","chr17","chr18","chr19","chr20","chr21","chr22"], - "ImputationBeagle.genetic_maps_path": "gs://broad-gotc-test-storage/imputation_beagle/scientific/plink-genetic-maps/", + "ImputationBeagle.genetic_maps_path": "gs://pd-test-storage-public/ImputationBeagle/input/scientific/NA12878_x10_hg38_arrays/plink-genetic-maps/", "ImputationBeagle.output_basename": "scientific_test" } diff --git a/pipelines/skylab/atac/test_inputs/Plumbing/10k_pbmc_downsampled.json b/pipelines/skylab/atac/test_inputs/Plumbing/10k_pbmc_downsampled.json index 71be349541..f28014aef2 100644 --- a/pipelines/skylab/atac/test_inputs/Plumbing/10k_pbmc_downsampled.json +++ b/pipelines/skylab/atac/test_inputs/Plumbing/10k_pbmc_downsampled.json @@ -2,13 +2,13 @@ "ATAC.input_id":"10k_PBMC_downsampled", "ATAC.cloud_provider":"gcp", "ATAC.read1_fastq_gzipped":[ - "gs://pd-test-storage-public/Multiome/input/plumbing/fastq_R1_atac.fastq.gz" + "gs://pd-test-storage-public/atac/input/plumbing/10k_pbmc_downsampled/fastq_R1_atac.fastq.gz" ], "ATAC.read2_fastq_gzipped":[ - "gs://pd-test-storage-public/Multiome/input/plumbing/fastq_R2_atac.fastq.gz" + "gs://pd-test-storage-public/atac/input/plumbing/10k_pbmc_downsampled/fastq_R2_atac.fastq.gz" ], "ATAC.read3_fastq_gzipped":[ - "gs://pd-test-storage-public/Multiome/input/plumbing/fastq_R3_atac.fastq.gz" + "gs://pd-test-storage-public/atac/input/plumbing/10k_pbmc_downsampled/fastq_R3_atac.fastq.gz" ], "ATAC.tar_bwa_reference":"gs://gcp-public-data--broad-references/hg38/v0/bwa/v2_2_1/bwa-mem2-2.2.1-Human-GENCODE-build-GRCh38.tar", "ATAC.chrom_sizes":"gs://pd-test-storage-public/Multiome/input/hg38.chrom.sizes", diff --git a/pipelines/skylab/atac/test_inputs/Plumbing/8k_cortex.json b/pipelines/skylab/atac/test_inputs/Plumbing/8k_cortex.json index d4a1858e72..cdf9e24394 100644 --- a/pipelines/skylab/atac/test_inputs/Plumbing/8k_cortex.json +++ b/pipelines/skylab/atac/test_inputs/Plumbing/8k_cortex.json @@ -2,13 +2,13 @@ "ATAC.input_id":"10x_8k_cortex_downsampled", "ATAC.cloud_provider":"gcp", "ATAC.read1_fastq_gzipped":[ - "gs://pd-test-storage-public/atac/input/plumbing/fastq_R1_0.fastq.gz" + "gs://pd-test-storage-public/atac/input/plumbing/8k_cortex/fastq_R1_0.fastq.gz" ], "ATAC.read2_fastq_gzipped":[ - "gs://pd-test-storage-public/atac/input/plumbing/fastq_R2_0.fastq.gz" + "gs://pd-test-storage-public/atac/input/plumbing/8k_cortex/fastq_R2_0.fastq.gz" ], "ATAC.read3_fastq_gzipped":[ - "gs://pd-test-storage-public/atac/input/plumbing/fastq_R3_0.fastq.gz" + "gs://pd-test-storage-public/atac/input/plumbing/8k_cortex/fastq_R3_0.fastq.gz" ], "ATAC.tar_bwa_reference":"gs://gcp-public-data--broad-references/GRCm39/bwa/v2_2_1/bwa-mem2-2.2.1-Mouse-GENCODE-build-GRCm39.tar", "ATAC.chrom_sizes":"gs://gcp-public-data--broad-references/GRCm39/bwa/v2_2_1/chrom.sizes", @@ -17,7 +17,7 @@ "ATAC.mem_size_bwa":"64", "ATAC.atac_nhash_id":"example_1234", "ATAC.annotations_gtf":" gs://gcp-public-data--broad-references/GRCm39/star/v2_7_10a/modified_vM32.annotation.gtf", - "ATAC.whitelist":"gs://pd-test-storage-public/atac/input/plumbing/737K-cratac-v1.txt", + "ATAC.whitelist":"gs://pd-test-storage-public/atac/input/plumbing/8k_cortex/737K-cratac-v1.txt", "ATAC.vm_size":"Standard_M128s", "ATAC.peak_calling":true, "ATAC.PeakCalling.probability_threshold":"1.0", diff --git a/pipelines/skylab/optimus/test_inputs/Scientific/inputs_8k_pbmc.json b/pipelines/skylab/optimus/test_inputs/Scientific/inputs_8k_pbmc.json index 466716fdf8..628988194b 100644 --- a/pipelines/skylab/optimus/test_inputs/Scientific/inputs_8k_pbmc.json +++ b/pipelines/skylab/optimus/test_inputs/Scientific/inputs_8k_pbmc.json @@ -1,15 +1,15 @@ { "Optimus.r1_fastq": [ -"gs://broad-gotc-test-storage/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L007_R1_001.fastq.gz", -"gs://broad-gotc-test-storage/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L008_R1_001.fastq.gz" +"gs://pd-test-storage-private/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L007_R1_001.fastq.gz", +"gs://pd-test-storage-private/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L008_R1_001.fastq.gz" ], "Optimus.r2_fastq": [ -"gs://broad-gotc-test-storage/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L007_R2_001.fastq.gz", -"gs://broad-gotc-test-storage/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L008_R2_001.fastq.gz" +"gs://pd-test-storage-private/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L007_R2_001.fastq.gz", +"gs://pd-test-storage-private/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L008_R2_001.fastq.gz" ], "Optimus.i1_fastq": [ -"gs://broad-gotc-test-storage/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L007_I1_001.fastq.gz", -"gs://broad-gotc-test-storage/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L008_I1_001.fastq.gz" +"gs://pd-test-storage-private/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L007_I1_001.fastq.gz", +"gs://pd-test-storage-private/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L008_I1_001.fastq.gz" ], "Optimus.tar_star_reference": "gs://gcp-public-data--broad-references/hg38/v0/star/v2_7_10a/modified_star2.7.10a-Human-GENCODE-build-GRCh38-43.tar", "Optimus.input_id": "8k_pbmc", diff --git a/pipelines/skylab/optimus/test_inputs/Scientific/inputs_8k_pbmc_stranded.json b/pipelines/skylab/optimus/test_inputs/Scientific/inputs_8k_pbmc_stranded.json index 0a9ee3adc2..de2e17b569 100644 --- a/pipelines/skylab/optimus/test_inputs/Scientific/inputs_8k_pbmc_stranded.json +++ b/pipelines/skylab/optimus/test_inputs/Scientific/inputs_8k_pbmc_stranded.json @@ -1,15 +1,15 @@ { "Optimus.r1_fastq": [ -"gs://broad-gotc-test-storage/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L007_R1_001.fastq.gz", -"gs://broad-gotc-test-storage/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L008_R1_001.fastq.gz" +"gs://pd-test-storage-private/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L007_R1_001.fastq.gz", +"gs://pd-test-storage-private/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L008_R1_001.fastq.gz" ], "Optimus.r2_fastq": [ -"gs://broad-gotc-test-storage/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L007_R2_001.fastq.gz", -"gs://broad-gotc-test-storage/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L008_R2_001.fastq.gz" +"gs://pd-test-storage-private/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L007_R2_001.fastq.gz", +"gs://pd-test-storage-private/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L008_R2_001.fastq.gz" ], "Optimus.i1_fastq": [ -"gs://broad-gotc-test-storage/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L007_I1_001.fastq.gz", -"gs://broad-gotc-test-storage/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L008_I1_001.fastq.gz" +"gs://pd-test-storage-private/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L007_I1_001.fastq.gz", +"gs://pd-test-storage-private/Optimus/input/scientific/chemistry_X10_V2/8k_pbmc/fastqs/pbmc8k_S1_L008_I1_001.fastq.gz" ], "Optimus.tar_star_reference": "gs://gcp-public-data--broad-references/hg38/v0/star/star_2.7.9a_primary_gencode_human_v27.tar", "Optimus.input_id": "8k_pbmc", diff --git a/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json b/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json index cc083e1d9a..d58ae44f9e 100644 --- a/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json +++ b/pipelines/skylab/paired_tag/test_inputs/Plumbing/10k_pbmc_downsampled.json @@ -2,23 +2,23 @@ "PairedTag.annotations_gtf":"gs://gcp-public-data--broad-references/hg38/v0/star/v2_7_10a/modified_v43.annotation.gtf", "PairedTag.input_id":"10k_PBMC_downsampled", "PairedTag.gex_r1_fastq":[ - "gs://pd-test-storage-public/Multiome/input/plumbing/fastq_R1_gex.fastq.gz" + "gs://pd-test-storage-public/PairedTag/input/plumbing/10k_PBMC_downsampled/fastq_R1_gex.fastq.gz" ], "PairedTag.gex_r2_fastq":[ - "gs://pd-test-storage-public/Multiome/input/plumbing/fastq_R2_gex.fastq.gz" + "gs://pd-test-storage-public/PairedTag/input/plumbing/10k_PBMC_downsampled/fastq_R2_gex.fastq.gz" ], "PairedTag.atac_r1_fastq":[ - "gs://pd-test-storage-public/Multiome/input/plumbing/fastq_R1_atac.fastq.gz" + "gs://pd-test-storage-public/PairedTag/input/plumbing/10k_PBMC_downsampled/fastq_R1_atac.fastq.gz" ], "PairedTag.atac_r2_fastq":[ - "gs://pd-test-storage-public/Multiome/input/plumbing/fastq_R2_atac.fastq.gz" + "gs://pd-test-storage-public/PairedTag/input/plumbing/10k_PBMC_downsampled/fastq_R2_atac.fastq.gz" ], "PairedTag.atac_r3_fastq":[ - "gs://pd-test-storage-public/Multiome/input/plumbing/fastq_R3_atac.fastq.gz" + "gs://pd-test-storage-public/PairedTag/input/plumbing/10k_PBMC_downsampled/fastq_R3_atac.fastq.gz" ], "PairedTag.tar_bwa_reference":"gs://gcp-public-data--broad-references/hg38/v0/bwa/v2_2_1/bwa-mem2-2.2.1-Human-GENCODE-build-GRCh38.tar", "PairedTag.tar_star_reference":"gs://gcp-public-data--broad-references/hg38/v0/star/v2_7_10a/modified_star2.7.10a-Human-GENCODE-build-GRCh38-43.tar", - "PairedTag.chrom_sizes":"gs://pd-test-storage-public/Multiome/input/hg38.chrom.sizes", + "PairedTag.chrom_sizes":"gs://pd-test-storage-public/PairedTag/input/hg38.chrom.sizes", "PairedTag.preindex":"false", "PairedTag.Atac_preindex.cpu_platform_bwa":"Intel Cascade Lake", "PairedTag.Atac_preindex.num_threads_bwa":"16", diff --git a/pipelines/skylab/paired_tag/test_inputs/Scientific/BC011_10kPBMC.json b/pipelines/skylab/paired_tag/test_inputs/Scientific/BC011_10kPBMC.json index fd0183b5c5..90bfeae64e 100644 --- a/pipelines/skylab/paired_tag/test_inputs/Scientific/BC011_10kPBMC.json +++ b/pipelines/skylab/paired_tag/test_inputs/Scientific/BC011_10kPBMC.json @@ -14,13 +14,13 @@ "gs://pd-test-storage-public/Multiome/input/scientific/10k_PBMC_Multiome/10k_PBMC_Multiome_nextgem_Chromium_Controller_gex_S1_L002_R2_001.fastq.gz" ], "PairedTag.atac_r1_fastq":[ - "gs://pd-test-storage-public/PairedTag/input/scientific/BC011_S1_L001_R1_001.fastq.gz" + "gs://pd-test-storage-public/PairedTag/input/scientific/BC011_10kPBMC/BC011_S1_L001_R1_001.fastq.gz" ], "PairedTag.atac_r2_fastq":[ - "gs://pd-test-storage-public/PairedTag/input/scientific/BC011_S1_L001_R2_001.fastq.gz" + "gs://pd-test-storage-public/PairedTag/input/scientific/BC011_10kPBMC/BC011_S1_L001_R2_001.fastq.gz" ], "PairedTag.atac_r3_fastq":[ - "gs://pd-test-storage-public/PairedTag/input/scientific/BC011_S1_L001_R3_001.fastq.gz" + "gs://pd-test-storage-public/PairedTag/input/scientific/BC011_10kPBMC/BC011_S1_L001_R3_001.fastq.gz" ], "PairedTag.chrom_sizes":"gs://pd-test-storage-public/Multiome/input/hg38.chrom.sizes", "PairedTag.preindex":"true", diff --git a/pipelines/skylab/slideseq/test_inputs/Plumbing/Puck_210817_11.mm10.json b/pipelines/skylab/slideseq/test_inputs/Plumbing/Puck_210817_11.mm10.json index 035b22c58e..0263c76edf 100644 --- a/pipelines/skylab/slideseq/test_inputs/Plumbing/Puck_210817_11.mm10.json +++ b/pipelines/skylab/slideseq/test_inputs/Plumbing/Puck_210817_11.mm10.json @@ -1,18 +1,18 @@ { - "SlideSeq.r1_fastq": ["gs://broad-gotc-test-storage/slide_seq/plumbing/truth/master/210817_Puck_210817_11.L001_sampled_down.R1", - "gs://broad-gotc-test-storage/slide_seq/plumbing/truth/master/210817_Puck_210817_11.L002_sampled_down.R1", - "gs://broad-gotc-test-storage/slide_seq/plumbing/truth/master/210817_Puck_210817_11.L003_sampled_down.R1", - "gs://broad-gotc-test-storage/slide_seq/plumbing/truth/master/210817_Puck_210817_11.L004_sampled_down.R1"], - "SlideSeq.r2_fastq": ["gs://broad-gotc-test-storage/slide_seq/plumbing/truth/master/210817_Puck_210817_11.L001_sampled_down.R2", - "gs://broad-gotc-test-storage/slide_seq/plumbing/truth/master/210817_Puck_210817_11.L002_sampled_down.R2", - "gs://broad-gotc-test-storage/slide_seq/plumbing/truth/master/210817_Puck_210817_11.L003_sampled_down.R2", - "gs://broad-gotc-test-storage/slide_seq/plumbing/truth/master/210817_Puck_210817_11.L004_sampled_down.R2"], + "SlideSeq.r1_fastq": ["gs://pd-test-storage-public/SlideSeq/input/plumbing/Puck_210817_11/210817_Puck_210817_11.L001_sampled_down.R1", + "gs://pd-test-storage-public/SlideSeq/input/plumbing/Puck_210817_11/210817_Puck_210817_11.L002_sampled_down.R1", + "gs://pd-test-storage-public/SlideSeq/input/plumbing/Puck_210817_11/210817_Puck_210817_11.L003_sampled_down.R1", + "gs://pd-test-storage-public/SlideSeq/input/plumbing/Puck_210817_11/210817_Puck_210817_11.L004_sampled_down.R1"], + "SlideSeq.r2_fastq": ["gs://pd-test-storage-public/SlideSeq/input/plumbing/Puck_210817_11/210817_Puck_210817_11.L001_sampled_down.R2", + "gs://pd-test-storage-public/SlideSeq/input/plumbing/Puck_210817_11/210817_Puck_210817_11.L002_sampled_down.R2", + "gs://pd-test-storage-public/SlideSeq/input/plumbing/Puck_210817_11/210817_Puck_210817_11.L003_sampled_down.R2", + "gs://pd-test-storage-public/SlideSeq/input/plumbing/Puck_210817_11/210817_Puck_210817_11.L004_sampled_down.R2"], "SlideSeq.read_structure": "8C18X6C9M1X", "SlideSeq.input_id": "Puck_210817_11", "SlideSeq.output_bam_basename": "Puck_210817_11", "SlideSeq.tar_star_reference": "gs://gcp-public-data--broad-references/mm10/v0/single_nucleus/star/modified_star_2.7.9a_primary_gencode_mouse_vM23.tar", "SlideSeq.annotations_gtf": "gs://gcp-public-data--broad-references/mm10/v0/single_nucleus/modified_gencode.vM23.primary_assembly.annotation.gtf", "SlideSeq.count_exons": true, - "SlideSeq.bead_locations": " gs://broad-gotc-test-storage/SlideSeq/inputs/plumbing/Puck_210817_11/Puck_210817_11.tsv", + "SlideSeq.bead_locations": "gs://pd-test-storage-public/SlideSeq/input/plumbing/Puck_210817_11/Puck_210817_11.tsv", "SlideSeq.cloud_provider": "gcp" } \ No newline at end of file diff --git a/pipelines/skylab/slideseq/test_inputs/Scientific/Puck_210817_11.mm10.json b/pipelines/skylab/slideseq/test_inputs/Scientific/Puck_210817_11.mm10.json index 458c4412ed..bf0bde8f34 100644 --- a/pipelines/skylab/slideseq/test_inputs/Scientific/Puck_210817_11.mm10.json +++ b/pipelines/skylab/slideseq/test_inputs/Scientific/Puck_210817_11.mm10.json @@ -1,20 +1,20 @@ { "SlideSeq.r1_fastq": [ -"gs://broad-gotc-test-storage/SlideSeq/inputs/scientific/Puck_210817_11/Puck_210817_11_L01_R1.fastq.gz", -"gs://broad-gotc-test-storage/SlideSeq/inputs/scientific/Puck_210817_11/Puck_210817_11_L02_R1.fastq.gz", -"gs://broad-gotc-test-storage/SlideSeq/inputs/scientific/Puck_210817_11/Puck_210817_11_L03_R1.fastq.gz", -"gs://broad-gotc-test-storage/SlideSeq/inputs/scientific/Puck_210817_11/Puck_210817_11_L04_R1.fastq.gz"], +"gs://pd-test-storage-public/SlideSeq/input/scientific/Puck_210817_11/Puck_210817_11_L01_R1.fastq.gz", +"gs://pd-test-storage-public/SlideSeq/input/scientific/Puck_210817_11/Puck_210817_11_L02_R1.fastq.gz", +"gs://pd-test-storage-public/SlideSeq/input/scientific/Puck_210817_11/Puck_210817_11_L03_R1.fastq.gz", +"gs://pd-test-storage-public/SlideSeq/input/scientific/Puck_210817_11/Puck_210817_11_L04_R1.fastq.gz"], "SlideSeq.r2_fastq": [ -"gs://broad-gotc-test-storage/SlideSeq/inputs/scientific/Puck_210817_11/Puck_210817_11_L01_R2.fastq.gz", -"gs://broad-gotc-test-storage/SlideSeq/inputs/scientific/Puck_210817_11/Puck_210817_11_L02_R2.fastq.gz", -"gs://broad-gotc-test-storage/SlideSeq/inputs/scientific/Puck_210817_11/Puck_210817_11_L03_R2.fastq.gz", -"gs://broad-gotc-test-storage/SlideSeq/inputs/scientific/Puck_210817_11/Puck_210817_11_L04_R2.fastq.gz"], +"gs://pd-test-storage-public/SlideSeq/input/scientific/Puck_210817_11/Puck_210817_11_L01_R2.fastq.gz", +"gs://pd-test-storage-public/SlideSeq/input/scientific/Puck_210817_11/Puck_210817_11_L02_R2.fastq.gz", +"gs://pd-test-storage-public/SlideSeq/input/scientific/Puck_210817_11/Puck_210817_11_L03_R2.fastq.gz", +"gs://pd-test-storage-public/SlideSeq/input/scientific/Puck_210817_11/Puck_210817_11_L04_R2.fastq.gz"], "SlideSeq.read_structure": "8C18X6C9M1X", "SlideSeq.input_id": "210817_Puck_210817_11", "SlideSeq.output_bam_basename": "210817_Puck_210817_11", "SlideSeq.tar_star_reference": "gs://gcp-public-data--broad-references/mm10/v0/single_nucleus/star/modified_star_2.7.9a_primary_gencode_mouse_vM23.tar", "SlideSeq.annotations_gtf": "gs://gcp-public-data--broad-references/mm10/v0/single_nucleus/modified_gencode.vM23.primary_assembly.annotation.gtf", "SlideSeq.count_exons": true, - "SlideSeq.bead_locations": "gs://broad-gotc-test-storage/SlideSeq/inputs/scientific/Puck_210817_11/Puck_210817_11.tsv", + "SlideSeq.bead_locations": "gs://pd-test-storage-public/SlideSeq/input/scientific/Puck_210817_11/Puck_210817_11.tsv", "SlideSeq.cloud_provider": "gcp" } diff --git a/pipelines/skylab/smartseq2_single_nucleus_multisample/test_inputs/Plumbing/mouse_example.json b/pipelines/skylab/smartseq2_single_nucleus_multisample/test_inputs/Plumbing/mouse_example.json index 587a4aa5e4..bff802dd22 100644 --- a/pipelines/skylab/smartseq2_single_nucleus_multisample/test_inputs/Plumbing/mouse_example.json +++ b/pipelines/skylab/smartseq2_single_nucleus_multisample/test_inputs/Plumbing/mouse_example.json @@ -2,16 +2,16 @@ "MultiSampleSmartSeq2SingleNucleus.genome_ref_fasta": "gs://gcp-public-data--broad-references/mm10/v0/single_nucleus/modified_mm10.primary_assembly.genome.fa", "MultiSampleSmartSeq2SingleNucleus.tar_star_reference": "gs://gcp-public-data--broad-references/mm10/v0/single_nucleus/star/modified_star_2.7.9a_primary_gencode_mouse_vM23.tar", "MultiSampleSmartSeq2SingleNucleus.annotations_gtf": "gs://gcp-public-data--broad-references/mm10/v0/single_nucleus/modified_gencode.vM23.primary_assembly.annotation.gtf", - "MultiSampleSmartSeq2SingleNucleus.adapter_list": "gs://broad-gotc-test-storage/MultiSampleSmartSeq2SingleNucleus/adapters/Illumina_adapters_list.fa", + "MultiSampleSmartSeq2SingleNucleus.adapter_list": "gs://pd-test-storage-public/Smartseq2_Single_Nucleus_Multisample/input/plumbing/SM-GE644/adapters/Illumina_adapters_list.fa", "MultiSampleSmartSeq2SingleNucleus.fastq1_input_files": [ - "gs://broad-gotc-test-storage/Smartseq2_Single_Nucleus_Multisample/truth/plumbing/master/pr_test_truth/SM-GE644_S110_E1-50_GGAGCTAC-CTAAGCCT_R1.fastq.gz", - "gs://broad-gotc-test-storage/Smartseq2_Single_Nucleus_Multisample/truth/plumbing/master/pr_test_truth/SM-GE644_S117_E1-50_GCGTAGTA-AAGGAGTA_R1.fastq.gz", - "gs://broad-gotc-test-storage/Smartseq2_Single_Nucleus_Multisample/truth/plumbing/master/pr_test_truth/SM-GE644_S118_E1-50_GCGTAGTA-CTAAGCCT_R1.fastq.gz" + "gs://pd-test-storage-public/Smartseq2_Single_Nucleus_Multisample/input/plumbing/SM-GE644/SM-GE644_S110_E1-50_GGAGCTAC-CTAAGCCT_R1.fastq.gz", + "gs://pd-test-storage-public/Smartseq2_Single_Nucleus_Multisample/input/plumbing/SM-GE644/SM-GE644_S117_E1-50_GCGTAGTA-AAGGAGTA_R1.fastq.gz", + "gs://pd-test-storage-public/Smartseq2_Single_Nucleus_Multisample/input/plumbing/SM-GE644/SM-GE644_S118_E1-50_GCGTAGTA-CTAAGCCT_R1.fastq.gz" ], "MultiSampleSmartSeq2SingleNucleus.fastq2_input_files": [ - "gs://broad-gotc-test-storage/Smartseq2_Single_Nucleus_Multisample/truth/plumbing/master/pr_test_truth/SM-GE644_S110_E1-50_GGAGCTAC-CTAAGCCT_R2.fastq.gz", - "gs://broad-gotc-test-storage/Smartseq2_Single_Nucleus_Multisample/truth/plumbing/master/pr_test_truth/SM-GE644_S117_E1-50_GCGTAGTA-AAGGAGTA_R2.fastq.gz", - "gs://broad-gotc-test-storage/Smartseq2_Single_Nucleus_Multisample/truth/plumbing/master/pr_test_truth/SM-GE644_S118_E1-50_GCGTAGTA-CTAAGCCT_R2.fastq.gz" + "gs://pd-test-storage-public/Smartseq2_Single_Nucleus_Multisample/input/plumbing/SM-GE644/SM-GE644_S110_E1-50_GGAGCTAC-CTAAGCCT_R2.fastq.gz", + "gs://pd-test-storage-public/Smartseq2_Single_Nucleus_Multisample/input/plumbing/SM-GE644/SM-GE644_S117_E1-50_GCGTAGTA-AAGGAGTA_R2.fastq.gz", + "gs://pd-test-storage-public/Smartseq2_Single_Nucleus_Multisample/input/plumbing/SM-GE644/SM-GE644_S118_E1-50_GCGTAGTA-CTAAGCCT_R2.fastq.gz" ], "MultiSampleSmartSeq2SingleNucleus.input_ids": [ "SM-GE644_S110_E1-50_GGAGCTAC-CTAAGCCT", diff --git a/pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json b/pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json index 047314bea1..a35c153044 100644 --- a/pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json +++ b/pipelines/skylab/snm3C/test_inputs/Plumbing/miseq_M16_G13.json @@ -1,21 +1,21 @@ { "snm3C.fastq_input_read1": [ - "gs://broad-gotc-test-storage/snM3C/inputs/plumbing/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L001_R1_001_fastq_R1_0.fastq.gz", - "gs://broad-gotc-test-storage/snM3C/inputs/plumbing/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L002_R1_001_fastq_R1_0.fastq.gz", - "gs://broad-gotc-test-storage/snM3C/inputs/plumbing/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L003_R1_001_fastq_R1_0.fastq.gz", - "gs://broad-gotc-test-storage/snM3C/inputs/plumbing/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L004_R1_001_fastq_R1_0.fastq.gz" + "gs://pd-test-storage-public/snm3C-seq/input/plumbing/miseq_M16_G13/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L001_R1_001_fastq_R1_0.fastq.gz", + "gs://pd-test-storage-public/snm3C-seq/input/plumbing/miseq_M16_G13/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L002_R1_001_fastq_R1_0.fastq.gz", + "gs://pd-test-storage-public/snm3C-seq/input/plumbing/miseq_M16_G13/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L003_R1_001_fastq_R1_0.fastq.gz", + "gs://pd-test-storage-public/snm3C-seq/input/plumbing/miseq_M16_G13/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L004_R1_001_fastq_R1_0.fastq.gz" ], "snm3C.fastq_input_read2": [ - "gs://broad-gotc-test-storage/snM3C/inputs/plumbing/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L001_R1_001_fastq_R2_0.fastq.gz", - "gs://broad-gotc-test-storage/snM3C/inputs/plumbing/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L002_R1_001_fastq_R2_0.fastq.gz", - "gs://broad-gotc-test-storage/snM3C/inputs/plumbing/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L003_R1_001_fastq_R2_0.fastq.gz", - "gs://broad-gotc-test-storage/snM3C/inputs/plumbing/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L004_R1_001_fastq_R2_0.fastq.gz" + "gs://pd-test-storage-public/snm3C-seq/input/plumbing/miseq_M16_G13/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L001_R1_001_fastq_R2_0.fastq.gz", + "gs://pd-test-storage-public/snm3C-seq/input/plumbing/miseq_M16_G13/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L002_R1_001_fastq_R2_0.fastq.gz", + "gs://pd-test-storage-public/snm3C-seq/input/plumbing/miseq_M16_G13/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L003_R1_001_fastq_R2_0.fastq.gz", + "gs://pd-test-storage-public/snm3C-seq/input/plumbing/miseq_M16_G13/230117-AD3C-hs-snm3C_seq-NovaSeq-pe-150-PB-AD3C_BA17_2027_P1-1-B11_S1_L004_R1_001_fastq_R2_0.fastq.gz" ], - "snm3C.random_primer_indexes": "gs://broad-gotc-test-storage/snM3C/inputs/plumbing/random_index_M16_G13.fa", + "snm3C.random_primer_indexes": "gs://pd-test-storage-public/snm3C-seq/input/plumbing/miseq_M16_G13/random_index_M16_G13.fa", "snm3C.plate_id": "AD3C_BA17_2027_P1-1-B11", - "snm3C.tarred_index_files":"gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38_index_files.tar.gz", - "snm3C.chromosome_sizes": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.chrom.sizes", - "snm3C.genome_fa": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.fa", + "snm3C.tarred_index_files":"gs://pd-test-storage-public/snm3C-seq/input/plumbing/miseq_M16_G13/index_files/hg38_index_files.tar.gz", + "snm3C.chromosome_sizes": "gs://pd-test-storage-public/snm3C-seq/input/plumbing/miseq_M16_G13/index_files/hg38.chrom.sizes", + "snm3C.genome_fa": "gs://pd-test-storage-public/snm3C-seq/input/plumbing/miseq_M16_G13/index_files/hg38.fa", "snm3C.Hisat_paired_end.cpu_platform" : "Intel Cascade Lake", "snm3C.Hisat_single_end.cpu_platform" : "Intel Cascade Lake", "snm3C.Merge_sort_analyze.cpu_platform" : "Intel Cascade Lake", diff --git a/pipelines/skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json b/pipelines/skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json index 14441d35ae..ca590c16b6 100644 --- a/pipelines/skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json +++ b/pipelines/skylab/snm3C/test_inputs/Scientific/novaseq_M16_G13.json @@ -1,21 +1,21 @@ { "snm3C.fastq_input_read1": [ - "gs://broad-gotc-test-storage/snM3C/inputs/scientific/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L001_R1_001_fastq_R1_0.fastq.gz", - "gs://broad-gotc-test-storage/snM3C/inputs/scientific/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L002_R1_001_fastq_R1_0.fastq.gz", - "gs://broad-gotc-test-storage/snM3C/inputs/scientific/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L003_R1_001_fastq_R1_0.fastq.gz", - "gs://broad-gotc-test-storage/snM3C/inputs/scientific/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L004_R1_001_fastq_R1_0.fastq.gz" + "gs://pd-test-storage-private/snm3C-seq/input/scientific/novaseq_M16_G13/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L001_R1_001_fastq_R1_0.fastq.gz", + "gs://pd-test-storage-private/snm3C-seq/input/scientific/novaseq_M16_G13/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L002_R1_001_fastq_R1_0.fastq.gz", + "gs://pd-test-storage-private/snm3C-seq/input/scientific/novaseq_M16_G13/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L003_R1_001_fastq_R1_0.fastq.gz", + "gs://pd-test-storage-private/snm3C-seq/input/scientific/novaseq_M16_G13/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L004_R1_001_fastq_R1_0.fastq.gz" ], "snm3C.fastq_input_read2": [ - "gs://broad-gotc-test-storage/snM3C/inputs/scientific/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L001_R1_001_fastq_R2_0.fastq.gz", - "gs://broad-gotc-test-storage/snM3C/inputs/scientific/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L002_R1_001_fastq_R2_0.fastq.gz", - "gs://broad-gotc-test-storage/snM3C/inputs/scientific/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L003_R1_001_fastq_R2_0.fastq.gz", - "gs://broad-gotc-test-storage/snM3C/inputs/scientific/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L004_R1_001_fastq_R2_0.fastq.gz" + "gs://pd-test-storage-private/snm3C-seq/input/scientific/novaseq_M16_G13/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L001_R1_001_fastq_R2_0.fastq.gz", + "gs://pd-test-storage-private/snm3C-seq/input/scientific/novaseq_M16_G13/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L002_R1_001_fastq_R2_0.fastq.gz", + "gs://pd-test-storage-private/snm3C-seq/input/scientific/novaseq_M16_G13/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L003_R1_001_fastq_R2_0.fastq.gz", + "gs://pd-test-storage-private/snm3C-seq/input/scientific/novaseq_M16_G13/230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3_S1_L004_R1_001_fastq_R2_0.fastq.gz" ], - "snm3C.random_primer_indexes": "gs://broad-gotc-test-storage/snM3C/inputs/plumbing/random_index_M16_G13.fa", + "snm3C.random_primer_indexes": "gs://pd-test-storage-private/snm3C-seq/input/scientific/novaseq_M16_G13/random_index_M16_G13.fa", "snm3C.plate_id": "230419-iN-hs-snm3C_seq-NovaSeq-pe-150-BW-iN230412_Entorhinal_Cortex_Adult_2115_R1_1-1-I3", - "snm3C.tarred_index_files":"gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38_index_files.tar.gz", - "snm3C.chromosome_sizes": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.chrom.sizes", - "snm3C.genome_fa": "gs://broad-gotc-test-storage/methylome/input/plumbing/index_files/hg38.fa", + "snm3C.tarred_index_files":"gs://pd-test-storage-private/snm3C-seq/input/scientific/novaseq_M16_G13/index_files/hg38_index_files.tar.gz", + "snm3C.chromosome_sizes": "gs://pd-test-storage-private/snm3C-seq/input/scientific/novaseq_M16_G13/index_files/hg38.chrom.sizes", + "snm3C.genome_fa": "gs://pd-test-storage-private/snm3C-seq/input/scientific/novaseq_M16_G13/index_files/hg38.fa", "snm3C.Hisat_paired_end.cpu_platform" : "Intel Cascade Lake", "snm3C.Hisat_single_end.cpu_platform" : "Intel Cascade Lake", "snm3C.Merge_sort_analyze.cpu_platform" : "Intel Cascade Lake", diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py index 1eb45d85e9..056efadf0f 100644 --- a/scripts/firecloud_api/UpdateTestInputs.py +++ b/scripts/firecloud_api/UpdateTestInputs.py @@ -11,7 +11,29 @@ def format_float(value): return str(Decimal(str(value))) return value -def update_test_inputs(inputs_json, truth_path, results_path, update_truth, branch_name): +def determine_bucket_from_inputs(test_inputs): + """ + Determine whether inputs are from public or private bucket by examining the values. + Returns the appropriate bucket path. + """ + # Bucket paths to check + public_bucket = "gs://pd-test-storage-public" + private_bucket = "gs://pd-test-storage-private" + + # Convert inputs to a string for easier searching + inputs_str = json.dumps(test_inputs) + + # Check if any value contains reference to public bucket + if public_bucket in inputs_str: + return public_bucket + # Check if any value contains reference to private bucket + elif private_bucket in inputs_str: + return private_bucket + + # Default to private bucket if no match found + return private_bucket + +def update_test_inputs(inputs_json, results_path, update_truth, branch_name, dockstore_pipeline_name): with open(inputs_json, 'r') as file: test_inputs = json.load(file) @@ -19,10 +41,34 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, bran sample_name = os.path.splitext(os.path.basename(inputs_json))[0] # Get the pipeline name from the test inputs JSON - pipeline_name = next(iter(test_inputs)).split('.')[0] + pipeline_name_from_test_inputs = next(iter(test_inputs)).split('.')[0] # Append "Test" in front of the pipeline name - test_name = f"Test{pipeline_name}" + test_name = f"Test{pipeline_name_from_test_inputs}" + + # Determine the correct bucket based on the input data + bucket_path = determine_bucket_from_inputs(test_inputs) + + # Get the test type from the sample name or infer from the inputs + if "plumbing" in sample_name.lower(): + test_type = "plumbing" + elif "scientific" in sample_name.lower(): + test_type = "scientific" + else: + # Try to infer from the inputs + inputs_str = json.dumps(test_inputs).lower() + if "plumbing" in inputs_str: + test_type = "plumbing" + elif "scientific" in inputs_str: + test_type = "scientific" + else: + # Default to plumbing if we can't determine + test_type = "plumbing" + + # Create the truth and results paths based on the determined bucket + # Extract the pipeline name from the input JSON for creating the bucket path + dockstore_pipeline_name = dockstore_pipeline_name + truth_path = f"{bucket_path}/{dockstore_pipeline_name}/truth/{test_type}/{branch_name}" # Update all keys and ensure nested inputs are handled correctly updated_inputs = {} @@ -35,7 +81,7 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, bran # For nested keys (more than two parts), append the original pipeline name with a `.` if len(key_parts) > 2: - key_parts[1] = f"{pipeline_name}.{key_parts[1]}" + key_parts[1] = f"{pipeline_name_from_test_inputs}.{key_parts[1]}" # Reconstruct the updated key new_key = '.'.join(key_parts) @@ -77,22 +123,17 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth, bran def main(): description = """This script updates the test inputs JSON to work with the test wrapper WDL, - which runs the pipeline and verification""" + which runs the pipeline and verification. It now determines the appropriate bucket (public or private) + based on the input data locations.""" parser = argparse.ArgumentParser(description=description) - parser.add_argument( - "--truth_path", - dest="truth_path", - required=True, - help="The base path where the truth data is stored", - ) - parser.add_argument( "--results_path", dest="results_path", - required=True, - help="The base path where the test data will be stored", + required=False, + default="gs://pd-test-results", + help="The base path where the test data will be stored (now determined dynamically based on input data)", ) parser.add_argument( @@ -117,12 +158,18 @@ def main(): required=True, help="Branch name of the current pipeline run") + parser.add_argument( + "--dockstore_pipeline_name", + required=True, + help="The pipeline name from Dockstore", + ) + args = parser.parse_args() # convert the update_truth flag to a boolean update_truth_bool = args.update_truth.lower() == "true" # Update the test inputs to work with the test wrapper WDL - update_test_inputs(args.inputs_json, args.truth_path, args.results_path, update_truth_bool, args.branch_name) + update_test_inputs(args.inputs_json, args.results_path, update_truth_bool, args.branch_name, args.dockstore_pipeline_name) if __name__ == "__main__": main() \ No newline at end of file diff --git a/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl index 66b6eb69a4..1860151bff 100644 --- a/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl +++ b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl @@ -30,9 +30,9 @@ task TerraCopyFilesFromCloudToCloud { echo ~{default='no_contamination' contamination} > contamination if ! grep -q no_contamination contamination; then - gcloud storage cp -L cp.log contamination ~{destination_cloud_path}.contamination + gcloud storage cp --project=warp-pipeline-dev --billing-project=terra-f8e3de20 -L cp.log contamination ~{destination_cloud_path}.contamination fi - gcloud storage cp ~{sep=' ' files_to_copy} ~{destination_cloud_path} + gcloud storage cp --project=warp-pipeline-dev --billing-project=terra-f8e3de20 ~{sep=' ' files_to_copy} ~{destination_cloud_path} } output {