Skip to content

Commit

Permalink
Unify _gcs_object_exist() and _gcs_object_size(); Fix a bug in _is_va…
Browse files Browse the repository at this point in the history
…lid_gcs_path()
  • Loading branch information
samanvp committed Mar 29, 2019
1 parent c28167f commit b80d730
Showing 1 changed file with 17 additions and 15 deletions.
32 changes: 17 additions & 15 deletions gcp_deepvariant_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,8 @@ def _is_valid_gcs_path(gcs_path):
Args:
gcs_path: (str) a path to directory or an obj on GCS.
"""
return urlparse.urlparse(gcs_path).scheme == 'gs'
return (urlparse.urlparse(gcs_path).scheme == 'gs' and
urlparse.urlparse(gcs_path).netloc != '')


def _gcs_object_exist(gcs_obj_path):
Expand All @@ -323,16 +324,7 @@ def _gcs_object_exist(gcs_obj_path):
Args:
gcs_obj_path: (str) a path to an obj on GCS.
"""
try:
storage_client = storage.Client()
bucket_name = _get_gcs_bucket(gcs_obj_path)
obj_name = _get_gcs_relative_path(gcs_obj_path)
bucket = storage_client.bucket(bucket_name)
obj = bucket.blob(obj_name)
return obj.exists()
except google_exceptions.Forbidden as e:
logging.error('Missing GCS object: %s', str(e))
return False
return _get_gcs_object_size(gcs_obj_path) != 0


def _get_gcs_object_size(gcs_obj_path):
Expand All @@ -341,13 +333,23 @@ def _get_gcs_object_size(gcs_obj_path):
Args:
gcs_obj_path: (str) a path to an obj on GCS.
"""
if not _gcs_object_exist(gcs_obj_path):
try:
bucket_name = _get_gcs_bucket(gcs_obj_path)
obj_name = _get_gcs_relative_path(gcs_obj_path)
except ValueError as e:
logging.error('Invalid GCS path: %s', str(e))
return 0

storage_client = storage.Client()
bucket_name = _get_gcs_bucket(gcs_obj_path)
obj_name = _get_gcs_relative_path(gcs_obj_path)
bucket = storage_client.get_bucket(bucket_name)
try:
bucket = storage_client.get_bucket(bucket_name)
except (google_exceptions.NotFound, google_exceptions.Forbidden) as e:
logging.error('Missing GCS bucket: %s', str(e))
return 0

blob = bucket.get_blob(obj_name)
if blob is None:
return 0
return blob.size


Expand Down

0 comments on commit b80d730

Please sign in to comment.