From a380296b4de6ea88781033e8c30d9409852ba090 Mon Sep 17 00:00:00 2001 From: yami Date: Fri, 26 May 2017 11:21:31 +0800 Subject: [PATCH] use InconsistentError for content-length mismatch issue and add request-id. --- oss2/api.py | 2 +- oss2/exceptions.py | 6 +++--- oss2/resumable.py | 2 +- oss2/utils.py | 6 ++++-- tests/test_download.py | 17 +++++++++++++++-- 5 files changed, 24 insertions(+), 9 deletions(-) diff --git a/oss2/api.py b/oss2/api.py index cd8ceda6..84528cb7 100644 --- a/oss2/api.py +++ b/oss2/api.py @@ -482,7 +482,7 @@ def get_object_to_file(self, key, filename, if result.content_length is None: shutil.copyfileobj(result, f) else: - utils.copyfileobj_and_verify(result, f, result.content_length) + utils.copyfileobj_and_verify(result, f, result.content_length, request_id=result.request_id) return result diff --git a/oss2/exceptions.py b/oss2/exceptions.py index 99c790a7..ebfb76b4 100644 --- a/oss2/exceptions.py +++ b/oss2/exceptions.py @@ -16,7 +16,7 @@ from .compat import to_string -_OSS_ERROR_TO_EXCEPTION = {} # populated at end of module +_OSS_ERROR_TO_EXCEPTION = {} # populated at end of module OSS_CLIENT_ERROR_STATUS = -1 @@ -72,8 +72,8 @@ def __str__(self): class InconsistentError(OssError): - def __init__(self, message): - OssError.__init__(self, OSS_INCONSISTENT_ERROR_STATUS, {}, 'InconsistentError: ' + message, {}) + def __init__(self, message, request_id=''): + OssError.__init__(self, OSS_INCONSISTENT_ERROR_STATUS, {'x-oss-request-id': request_id}, 'InconsistentError: ' + message, {}) def __str__(self): error = {'status': self.status, diff --git a/oss2/resumable.py b/oss2/resumable.py index 99818876..96718c0c 100644 --- a/oss2/resumable.py +++ b/oss2/resumable.py @@ -285,7 +285,7 @@ def __download_part(self, part): headers = {'If-Match': self.objectInfo.etag, 'If-Unmodified-Since': utils.http_date(self.objectInfo.mtime)} result = self.bucket.get_object(self.key, byte_range=(part.start, part.end - 1), headers=headers) - utils.copyfileobj_and_verify(result, f, part.end - part.start) + utils.copyfileobj_and_verify(result, f, part.end - part.start, request_id=result.request_id) self.__finish_part(part) diff --git a/oss2/utils.py b/oss2/utils.py index fc91be2b..645ea73c 100644 --- a/oss2/utils.py +++ b/oss2/utils.py @@ -467,7 +467,9 @@ def force_rename(src, dst): raise -def copyfileobj_and_verify(fsrc, fdst, expected_len, chunk_size=16*1024): +def copyfileobj_and_verify(fsrc, fdst, expected_len, + chunk_size=16*1024, + request_id=''): """copy data from file-like object fsrc to file-like object fdst, and verify length""" num_read = 0 @@ -481,4 +483,4 @@ def copyfileobj_and_verify(fsrc, fdst, expected_len, chunk_size=16*1024): fdst.write(buf) if num_read != expected_len: - raise RequestError(IOError("IncompleteRead from source")) + raise InconsistentError("IncompleteRead from source", request_id) diff --git a/tests/test_download.py b/tests/test_download.py index 7f3ae3d8..2b15de47 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -18,6 +18,7 @@ class SizedFileAdapterForMock(object): def __init__(self, file_object, size, content_length=None): self.f = oss2.utils.SizedFileAdapter(file_object, size) self.content_length = content_length + self.request_id = 'fake-request-id' def read(self, amt=None): return self.f.read(amt) @@ -551,7 +552,13 @@ def test_get_object_to_file_incomplete_download(self): with patch.object(oss2.Bucket, 'get_object', side_effect=partial(mock_get_object, content_length=file_size), autospec=True): - self.assertRaises(oss2.exceptions.RequestError, oss2.Bucket.get_object_to_file, self.bucket, key, filename) + try: + self.bucket.get_object_to_file(key, filename) + except oss2.exceptions.InconsistentError as e: + self.assertTrue(e.request_id) + self.assertEqual(e.body, 'InconsistentError: IncompleteRead from source') + except: + self.assertTrue(False) def test_get_object_to_file_incomplete_download_gzip(self): file_size = 1024 * 1024 @@ -577,8 +584,14 @@ def test_resumable_incomplete_download(self): with patch.object(oss2.Bucket, 'get_object', side_effect=partial(mock_get_object, content_length=file_size), autospec=True): + try: + oss2.resumable_download(self.bucket, key, filename) + except oss2.exceptions.InconsistentError as e: + self.assertTrue(e.request_id) + self.assertEqual(e.body, 'InconsistentError: IncompleteRead from source') + except: + self.assertTrue(False) - self.assertRaises(oss2.exceptions.RequestError, oss2.resumable_download, self.bucket, key, filename) if __name__ == '__main__': unittest.main() \ No newline at end of file