From e1a1dcda6cda20e48f567c2028f04f02066b3e20 Mon Sep 17 00:00:00 2001 From: Saman Vaisipour Date: Thu, 6 Jun 2019 16:40:57 -0400 Subject: [PATCH] Remove redudant codes from unit tests --- gcp_deepvariant_runner_test.py | 403 ++++++++------------------------- 1 file changed, 95 insertions(+), 308 deletions(-) diff --git a/gcp_deepvariant_runner_test.py b/gcp_deepvariant_runner_test.py index 4f50b41..507c24e 100644 --- a/gcp_deepvariant_runner_test.py +++ b/gcp_deepvariant_runner_test.py @@ -115,6 +115,11 @@ def setUp(self): 'gs://bucket/ref', ] + def _update_argv(self, bam_suffix, model_suffix, extend_auto_flags): + self._argv[self._argv.index('--bam') + 1] += bam_suffix + self._argv[self._argv.index('--model') + 1] += model_suffix + self._argv.extend(extend_auto_flags) + @mock.patch('gcp_deepvariant_runner._run_job') @mock.patch.object(multiprocessing, 'Pool') @mock.patch('gcp_deepvariant_runner._get_gcs_object_size') @@ -521,30 +526,14 @@ def testRunFailsCannotWriteOutputBucket(self, mock_blob_exists): gcp_deepvariant_runner.run(self._argv) def testRunFailsSetOptimizedFlagsMissingExpectedModel(self): - self._argv.extend(['--set_optimized_flags_based_on_bam_size']) + self._update_argv(gcp_deepvariant_runner._BAM_FILE_SUFFIX, '', + ['--set_optimized_flags_based_on_bam_size']) with self.assertRaises(ValueError): gcp_deepvariant_runner.run(self._argv) def testRunFailsSetOptimizedFlagsMissingBamFile(self): - self._argv = [ - '--project', - 'project', - '--docker_image', - 'gcr.io/dockerimage', - '--zones', - 'zone-a', - '--outfile', - 'gs://bucket/output.vcf', - '--staging', - 'gs://bucket/staging', - '--model', - 'gs://bucket/model' + gcp_deepvariant_runner._WES_STANDARD, - '--bam', - 'gs://bucket/bam', - '--ref', - 'gs://bucket/ref', - '--set_optimized_flags_based_on_bam_size', - ] + self._update_argv('', gcp_deepvariant_runner._WES_STANDARD, + ['--set_optimized_flags_based_on_bam_size']) with self.assertRaises(ValueError): gcp_deepvariant_runner.run(self._argv) @@ -561,29 +550,13 @@ def testRunSetOptimizedFlagsBasedOnBamSizeWesSmall_makeExamples( expected_cores = 1 expected_shards = expected_workers * expected_cores expected_ram = expected_cores * gcp_deepvariant_runner._RAM_PER_CORE * 1024 - self._argv = [ - '--project', - 'project', - '--docker_image', - 'gcr.io/dockerimage', - '--zones', - 'zone-a', - '--outfile', - 'gs://bucket/output.vcf', - '--staging', - 'gs://bucket/staging', - '--model', - 'gs://bucket/model' + gcp_deepvariant_runner._WES_STANDARD, - '--bam', - 'gs://bucket/bam' + gcp_deepvariant_runner._BAM_FILE_SUFFIX, - '--ref', - 'gs://bucket/ref', - '--jobs_to_run', - 'make_examples', - '--set_optimized_flags_based_on_bam_size', - '--docker_image_gpu', - 'gcr.io/dockerimage_gpu', - ] + self._update_argv(gcp_deepvariant_runner._BAM_FILE_SUFFIX, + gcp_deepvariant_runner._WES_STANDARD, + ['--jobs_to_run', + 'make_examples', + '--set_optimized_flags_based_on_bam_size', + '--docker_image_gpu', + 'gcr.io/dockerimage_gpu']) expected_mock_calls = [] for i in range(expected_workers): @@ -615,30 +588,13 @@ def testRunSetOptimizedFlagsBasedOnBamSizeWesLarge_makeExamples( expected_cores = 1 expected_shards = expected_workers * expected_cores expected_ram = expected_cores * gcp_deepvariant_runner._RAM_PER_CORE * 1024 - self._argv = [ - '--project', - 'project', - '--docker_image', - 'gcr.io/dockerimage', - '--zones', - 'zone-a', - '--outfile', - 'gs://bucket/output.vcf', - '--staging', - 'gs://bucket/staging', - '--model', - 'gs://bucket/model' + gcp_deepvariant_runner._WES_STANDARD, - '--bam', - 'gs://bucket/bam' + gcp_deepvariant_runner._BAM_FILE_SUFFIX, - '--ref', - 'gs://bucket/ref', - '--jobs_to_run', - 'make_examples', - '--set_optimized_flags_based_on_bam_size', - '--docker_image_gpu', - 'gcr.io/dockerimage_gpu', - ] - + self._update_argv(gcp_deepvariant_runner._BAM_FILE_SUFFIX, + gcp_deepvariant_runner._WES_STANDARD, + ['--jobs_to_run', + 'make_examples', + '--set_optimized_flags_based_on_bam_size', + '--docker_image_gpu', + 'gcr.io/dockerimage_gpu']) expected_mock_calls = [] for i in range(expected_workers): expected_mock_calls.append( @@ -670,30 +626,13 @@ def testRunSetOptimizedFlagsBasedOnBamSizeWgsSmall_makeExamples( expected_cores = 1 expected_shards = expected_workers * expected_cores expected_ram = expected_cores * gcp_deepvariant_runner._RAM_PER_CORE * 1024 - self._argv = [ - '--project', - 'project', - '--docker_image', - 'gcr.io/dockerimage', - '--zones', - 'zone-a', - '--outfile', - 'gs://bucket/output.vcf', - '--staging', - 'gs://bucket/staging', - '--model', - 'gs://bucket/model' + gcp_deepvariant_runner._WGS_STANDARD, - '--bam', - 'gs://bucket/bam' + gcp_deepvariant_runner._BAM_FILE_SUFFIX, - '--ref', - 'gs://bucket/ref', - '--jobs_to_run', - 'make_examples', - '--set_optimized_flags_based_on_bam_size', - '--docker_image_gpu', - 'gcr.io/dockerimage_gpu', - ] - + self._update_argv(gcp_deepvariant_runner._BAM_FILE_SUFFIX, + gcp_deepvariant_runner._WGS_STANDARD, + ['--jobs_to_run', + 'make_examples', + '--set_optimized_flags_based_on_bam_size', + '--docker_image_gpu', + 'gcr.io/dockerimage_gpu']) expected_mock_calls = [] for i in range(expected_workers): expected_mock_calls.append( @@ -725,30 +664,13 @@ def testRunSetOptimizedFlagsBasedOnBamSizeWgsMedium_make_examples( expected_cores = 1 expected_shards = expected_workers * expected_cores expected_ram = expected_cores * gcp_deepvariant_runner._RAM_PER_CORE * 1024 - self._argv = [ - '--project', - 'project', - '--docker_image', - 'gcr.io/dockerimage', - '--zones', - 'zone-a', - '--outfile', - 'gs://bucket/output.vcf', - '--staging', - 'gs://bucket/staging', - '--model', - 'gs://bucket/model' + gcp_deepvariant_runner._WGS_STANDARD, - '--bam', - 'gs://bucket/bam' + gcp_deepvariant_runner._BAM_FILE_SUFFIX, - '--ref', - 'gs://bucket/ref', - '--jobs_to_run', - 'make_examples', - '--set_optimized_flags_based_on_bam_size', - '--docker_image_gpu', - 'gcr.io/dockerimage_gpu', - ] - + self._update_argv(gcp_deepvariant_runner._BAM_FILE_SUFFIX, + gcp_deepvariant_runner._WGS_STANDARD, + ['--jobs_to_run', + 'make_examples', + '--set_optimized_flags_based_on_bam_size', + '--docker_image_gpu', + 'gcr.io/dockerimage_gpu']) expected_mock_calls = [] for i in range(expected_workers): expected_mock_calls.append( @@ -780,30 +702,13 @@ def testRunSetOptimizedFlagsBasedOnBamSizeWgsLarge_makeExamples( expected_cores = 1 expected_shards = expected_workers * expected_cores expected_ram = expected_cores * gcp_deepvariant_runner._RAM_PER_CORE * 1024 - self._argv = [ - '--project', - 'project', - '--docker_image', - 'gcr.io/dockerimage', - '--zones', - 'zone-a', - '--outfile', - 'gs://bucket/output.vcf', - '--staging', - 'gs://bucket/staging', - '--model', - 'gs://bucket/model' + gcp_deepvariant_runner._WGS_STANDARD, - '--bam', - 'gs://bucket/bam' + gcp_deepvariant_runner._BAM_FILE_SUFFIX, - '--ref', - 'gs://bucket/ref', - '--jobs_to_run', - 'make_examples', - '--set_optimized_flags_based_on_bam_size', - '--docker_image_gpu', - 'gcr.io/dockerimage_gpu', - ] - + self._update_argv(gcp_deepvariant_runner._BAM_FILE_SUFFIX, + gcp_deepvariant_runner._WGS_STANDARD, + ['--jobs_to_run', + 'make_examples', + '--set_optimized_flags_based_on_bam_size', + '--docker_image_gpu', + 'gcr.io/dockerimage_gpu']) expected_mock_calls = [] for i in range(expected_workers): expected_mock_calls.append( @@ -836,30 +741,13 @@ def testRunSetOptimizedFlagsBasedOnBamSizeWesSmall_callVariants( expected_cores = 4 expected_shards = 16 # equals to make_exmples: num_wokers * num_cores expected_ram = expected_cores * gcp_deepvariant_runner._RAM_PER_CORE * 1024 - self._argv = [ - '--project', - 'project', - '--docker_image', - 'gcr.io/dockerimage', - '--zones', - 'zone-a', - '--outfile', - 'gs://bucket/output.vcf', - '--staging', - 'gs://bucket/staging', - '--model', - 'gs://bucket/model' + gcp_deepvariant_runner._WES_STANDARD, - '--bam', - 'gs://bucket/bam' + gcp_deepvariant_runner._BAM_FILE_SUFFIX, - '--ref', - 'gs://bucket/ref', - '--jobs_to_run', - 'call_variants', - '--set_optimized_flags_based_on_bam_size', - '--docker_image_gpu', - 'gcr.io/dockerimage_gpu', - ] - + self._update_argv(gcp_deepvariant_runner._BAM_FILE_SUFFIX, + gcp_deepvariant_runner._WES_STANDARD, + ['--jobs_to_run', + 'call_variants', + '--set_optimized_flags_based_on_bam_size', + '--docker_image_gpu', + 'gcr.io/dockerimage_gpu']) expected_mock_calls = [] for i in range(expected_workers): expected_mock_calls.append( @@ -894,30 +782,13 @@ def testRunSetOptimizedFlagsBasedOnBamSizeWesLarge_callVariants( expected_cores = 4 expected_shards = 16 # equals to make_exmples: num_wokers * num_cores expected_ram = expected_cores * gcp_deepvariant_runner._RAM_PER_CORE * 1024 - self._argv = [ - '--project', - 'project', - '--docker_image', - 'gcr.io/dockerimage', - '--zones', - 'zone-a', - '--outfile', - 'gs://bucket/output.vcf', - '--staging', - 'gs://bucket/staging', - '--model', - 'gs://bucket/model' + gcp_deepvariant_runner._WES_STANDARD, - '--bam', - 'gs://bucket/bam' + gcp_deepvariant_runner._BAM_FILE_SUFFIX, - '--ref', - 'gs://bucket/ref', - '--jobs_to_run', - 'call_variants', - '--set_optimized_flags_based_on_bam_size', - '--docker_image_gpu', - 'gcr.io/dockerimage_gpu', - ] - + self._update_argv(gcp_deepvariant_runner._BAM_FILE_SUFFIX, + gcp_deepvariant_runner._WES_STANDARD, + ['--jobs_to_run', + 'call_variants', + '--set_optimized_flags_based_on_bam_size', + '--docker_image_gpu', + 'gcr.io/dockerimage_gpu']) expected_mock_calls = [] for i in range(expected_workers): expected_mock_calls.append( @@ -952,30 +823,13 @@ def testRunSetOptimizedFlagsBasedOnBamSizeWgsSmall_callVariants( expected_cores = 4 expected_shards = 32 # equals to make_exmples: num_wokers * num_cores expected_ram = expected_cores * gcp_deepvariant_runner._RAM_PER_CORE * 1024 - self._argv = [ - '--project', - 'project', - '--docker_image', - 'gcr.io/dockerimage', - '--zones', - 'zone-a', - '--outfile', - 'gs://bucket/output.vcf', - '--staging', - 'gs://bucket/staging', - '--model', - 'gs://bucket/model' + gcp_deepvariant_runner._WGS_STANDARD, - '--bam', - 'gs://bucket/bam' + gcp_deepvariant_runner._BAM_FILE_SUFFIX, - '--ref', - 'gs://bucket/ref', - '--jobs_to_run', - 'call_variants', - '--set_optimized_flags_based_on_bam_size', - '--docker_image_gpu', - 'gcr.io/dockerimage_gpu', - ] - + self._update_argv(gcp_deepvariant_runner._BAM_FILE_SUFFIX, + gcp_deepvariant_runner._WGS_STANDARD, + ['--jobs_to_run', + 'call_variants', + '--set_optimized_flags_based_on_bam_size', + '--docker_image_gpu', + 'gcr.io/dockerimage_gpu']) expected_mock_calls = [] for i in range(expected_workers): expected_mock_calls.append( @@ -996,7 +850,6 @@ def testRunSetOptimizedFlagsBasedOnBamSizeWgsSmall_callVariants( mock_apply_async.assert_has_calls(expected_mock_calls,) self.assertEqual(mock_apply_async.call_count, expected_workers) - @mock.patch.object(multiprocessing, 'Pool') @mock.patch('gcp_deepvariant_runner._can_write_to_bucket') @mock.patch('gcp_deepvariant_runner._get_gcs_object_size') @@ -1011,30 +864,13 @@ def testRunSetOptimizedFlagsBasedOnBamSizeWgsMedium_callVariants( expected_cores = 4 expected_shards = 64 # equals to make_exmples: num_wokers * num_cores expected_ram = expected_cores * gcp_deepvariant_runner._RAM_PER_CORE * 1024 - self._argv = [ - '--project', - 'project', - '--docker_image', - 'gcr.io/dockerimage', - '--zones', - 'zone-a', - '--outfile', - 'gs://bucket/output.vcf', - '--staging', - 'gs://bucket/staging', - '--model', - 'gs://bucket/model' + gcp_deepvariant_runner._WGS_STANDARD, - '--bam', - 'gs://bucket/bam' + gcp_deepvariant_runner._BAM_FILE_SUFFIX, - '--ref', - 'gs://bucket/ref', - '--jobs_to_run', - 'call_variants', - '--set_optimized_flags_based_on_bam_size', - '--docker_image_gpu', - 'gcr.io/dockerimage_gpu', - ] - + self._update_argv(gcp_deepvariant_runner._BAM_FILE_SUFFIX, + gcp_deepvariant_runner._WGS_STANDARD, + ['--jobs_to_run', + 'call_variants', + '--set_optimized_flags_based_on_bam_size', + '--docker_image_gpu', + 'gcr.io/dockerimage_gpu']) expected_mock_calls = [] for i in range(expected_workers): expected_mock_calls.append( @@ -1069,30 +905,13 @@ def testRunSetOptimizedFlagsBasedOnBamSizeWgsLarge_callVariants( expected_cores = 4 expected_shards = 128 # equals to make_exmples: num_wokers * num_cores expected_ram = expected_cores * gcp_deepvariant_runner._RAM_PER_CORE * 1024 - self._argv = [ - '--project', - 'project', - '--docker_image', - 'gcr.io/dockerimage', - '--zones', - 'zone-a', - '--outfile', - 'gs://bucket/output.vcf', - '--staging', - 'gs://bucket/staging', - '--model', - 'gs://bucket/model' + gcp_deepvariant_runner._WGS_STANDARD, - '--bam', - 'gs://bucket/bam' + gcp_deepvariant_runner._BAM_FILE_SUFFIX, - '--ref', - 'gs://bucket/ref', - '--jobs_to_run', - 'call_variants', - '--set_optimized_flags_based_on_bam_size', - '--docker_image_gpu', - 'gcr.io/dockerimage_gpu', - ] - + self._update_argv(gcp_deepvariant_runner._BAM_FILE_SUFFIX, + gcp_deepvariant_runner._WGS_STANDARD, + ['--jobs_to_run', + 'call_variants', + '--set_optimized_flags_based_on_bam_size', + '--docker_image_gpu', + 'gcr.io/dockerimage_gpu']) expected_mock_calls = [] for i in range(expected_workers): expected_mock_calls.append( @@ -1126,31 +945,15 @@ def testRunSetOptimizedFlagsBasedOnBamSizeWgsSmall_postProcessVariants( expected_cores = 8 expected_ram = 30 * 1024 expected_gvcf_disk = gcp_deepvariant_runner._POSTPROCESS_VARIANTS_DISK_GVCF - self._argv = [ - '--project', - 'project', - '--docker_image', - 'gcr.io/dockerimage', - '--zones', - 'zone-a', - '--outfile', - 'gs://bucket/output.vcf', - '--staging', - 'gs://bucket/staging', - '--model', - 'gs://bucket/model' + gcp_deepvariant_runner._WGS_STANDARD, - '--bam', - 'gs://bucket/bam' + gcp_deepvariant_runner._BAM_FILE_SUFFIX, - '--ref', - 'gs://bucket/ref', - '--jobs_to_run', - 'postprocess_variants', - '--set_optimized_flags_based_on_bam_size', - '--docker_image_gpu', - 'gcr.io/dockerimage_gpu', - '--gvcf_outfile', - 'gvcf-folder-path', - ] + self._update_argv(gcp_deepvariant_runner._BAM_FILE_SUFFIX, + gcp_deepvariant_runner._WGS_STANDARD, + ['--jobs_to_run', + 'postprocess_variants', + '--set_optimized_flags_based_on_bam_size', + '--docker_image_gpu', + 'gcr.io/dockerimage_gpu', + '--gvcf_outfile', + 'gvcf-folder-path']) gcp_deepvariant_runner.run(self._argv) mock_run_job.assert_called_once_with( _HasAllOf( @@ -1177,29 +980,13 @@ def testRunSetOptimizedFlagsBasedOnBamSizeWgsLarge_postProcessVariants( expected_cores = 8 expected_ram = 30 * 1024 expected_disk = 30 - self._argv = [ - '--project', - 'project', - '--docker_image', - 'gcr.io/dockerimage', - '--zones', - 'zone-a', - '--outfile', - 'gs://bucket/output.vcf', - '--staging', - 'gs://bucket/staging', - '--model', - 'gs://bucket/model' + gcp_deepvariant_runner._WGS_STANDARD, - '--bam', - 'gs://bucket/bam' + gcp_deepvariant_runner._BAM_FILE_SUFFIX, - '--ref', - 'gs://bucket/ref', - '--jobs_to_run', - 'postprocess_variants', - '--set_optimized_flags_based_on_bam_size', - '--docker_image_gpu', - 'gcr.io/dockerimage_gpu', - ] + self._update_argv(gcp_deepvariant_runner._BAM_FILE_SUFFIX, + gcp_deepvariant_runner._WGS_STANDARD, + ['--jobs_to_run', + 'postprocess_variants', + '--set_optimized_flags_based_on_bam_size', + '--docker_image_gpu', + 'gcr.io/dockerimage_gpu']) gcp_deepvariant_runner.run(self._argv) mock_run_job.assert_called_once_with( _HasAllOf('postprocess_variants', 'gcr.io/dockerimage',