From da4d785028e86899f1efc23394883836b94cd0d3 Mon Sep 17 00:00:00 2001 From: Andrew Smith Date: Wed, 22 Nov 2023 00:05:58 +0000 Subject: [PATCH 1/2] feat: add update existing file function --- storage3/_async/file_api.py | 43 +++++++++++++++--- tests/_async/test_client.py | 88 +++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 6 deletions(-) diff --git a/storage3/_async/file_api.py b/storage3/_async/file_api.py index 5557d419..3a947eb6 100644 --- a/storage3/_async/file_api.py +++ b/storage3/_async/file_api.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field from io import BufferedReader, FileIO from pathlib import Path -from typing import Any, Optional, Union, cast +from typing import Any, Literal, Optional, Union, cast from httpx import HTTPError, Response @@ -344,8 +344,9 @@ async def download(self, path: str, options: DownloadOptions = {}) -> bytes: ) return response.content - async def upload( + async def _upload_or_update( self, + method: Literal["POST", "PUT"], path: str, file: Union[BufferedReader, bytes, FileIO, str, Path], file_options: Optional[FileOptions] = None, @@ -367,9 +368,6 @@ async def upload( file_options = {} cache_control = file_options.get("cache-control") _data = {} - if cache_control: - file_options["cache-control"] = f"max-age={cache_control}" - _data = {"cacheControl": cache_control} headers = { **self._client.headers, @@ -378,6 +376,10 @@ async def upload( } filename = path.rsplit("/", maxsplit=1)[-1] + if cache_control: + headers["cache-control"] = f"max-age={cache_control}" + _data = {"cacheControl": cache_control} + if ( isinstance(file, BufferedReader) or isinstance(file, bytes) @@ -398,9 +400,38 @@ async def upload( _path = self._get_final_path(path) return await self._request( - "POST", f"/object/{_path}", files=files, headers=headers, data=_data + method, f"/object/{_path}", files=files, headers=headers, data=_data ) + async def upload( + self, + path: str, + file: Union[BufferedReader, bytes, FileIO, str, Path], + file_options: Optional[FileOptions] = None, + ) -> Response: + """ + Uploads a file to an existing bucket. + + Parameters + ---------- + path + The relative file path including the bucket ID. Should be of the format `bucket/folder/subfolder/filename.png`. + The bucket must already exist before attempting to upload. + file + The File object to be stored in the bucket. or a async generator of chunks + file_options + HTTP headers. + """ + return await self._upload_or_update("POST", path, file, file_options) + + async def update( + self, + path: str, + file: Union[BufferedReader, bytes, FileIO, str, Path], + file_options: Optional[FileOptions] = None, + ) -> Response: + return await self._upload_or_update("PUT", path, file, file_options) + def _get_final_path(self, path: str) -> str: return f"{self.id}/{path}" diff --git a/tests/_async/test_client.py b/tests/_async/test_client.py index 95b001b0..7c9c283e 100644 --- a/tests/_async/test_client.py +++ b/tests/_async/test_client.py @@ -156,6 +156,67 @@ def file(tmp_path: Path, uuid_factory: Callable[[], str]) -> FileForTesting: ) +@pytest.fixture +def two_files(tmp_path: Path, uuid_factory: Callable[[], str]) -> list[FileForTesting]: + """Creates multiple test files (different content, same bucket/folder path, different file names)""" + file_name_1 = "test_image_1.svg" + file_name_2 = "test_image_2.svg" + file_content = ( + b' ' + b' ' + b' ' + b' ' + b' ' + ) + file_content_2 = ( + b' ' + b' ' + b' ' + b' ' + b' ' + ) + bucket_folder = uuid_factory() + bucket_path_1 = f"{bucket_folder}/{file_name_1}" + bucket_path_2 = f"{bucket_folder}/{file_name_2}" + file_path_1 = tmp_path / file_name_1 + file_path_2 = tmp_path / file_name_2 + with open(file_path_1, "wb") as f: + f.write(file_content) + with open(file_path_2, "wb") as f: + f.write(file_content_2) + + return [ + FileForTesting( + name=file_name_1, + local_path=str(file_path_1), + bucket_folder=bucket_folder, + bucket_path=bucket_path_1, + mime_type="image/svg+xml", + file_content=file_content, + ), + FileForTesting( + name=file_name_2, + local_path=str(file_path_2), + bucket_folder=bucket_folder, + bucket_path=bucket_path_2, + mime_type="image/svg+xml", + file_content=file_content_2, + ), + ] + + @pytest.fixture def multi_file(tmp_path: Path, uuid_factory: Callable[[], str]) -> list[FileForTesting]: """Creates multiple test files (same content, same bucket/folder path, different file names)""" @@ -223,6 +284,33 @@ async def test_client_upload( assert image_info.get("metadata", {}).get("mimetype") == file.mime_type +async def test_client_update( + storage_file_client: AsyncBucketProxy, + two_files: list[FileForTesting], +) -> None: + """Ensure we can upload files to a bucket""" + await storage_file_client.upload( + two_files[0].bucket_path, + two_files[0].local_path, + {"content-type": two_files[0].mime_type}, + ) + + await storage_file_client.update( + two_files[0].bucket_path, + two_files[1].local_path, + {"content-type": two_files[1].mime_type}, + ) + + image = await storage_file_client.download(two_files[0].bucket_path) + file_list = await storage_file_client.list(two_files[0].bucket_folder) + image_info = next( + (f for f in file_list if f.get("name") == two_files[0].name), None + ) + + assert image == two_files[1].file_content + assert image_info.get("metadata", {}).get("mimetype") == two_files[1].mime_type + + @pytest.mark.parametrize( "path", ["foobar.txt", "example/nested.jpg", "/leading/slash.png"] ) From 18c14a4e7888cd14d09b7cbbd2e8b02f9cef02f0 Mon Sep 17 00:00:00 2001 From: Andrew Smith Date: Wed, 22 Nov 2023 00:38:53 +0000 Subject: [PATCH 2/2] chore: add sync version of the update function --- storage3/_sync/file_api.py | 43 ++++++++++++++++--- tests/_sync/test_client.py | 88 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 6 deletions(-) diff --git a/storage3/_sync/file_api.py b/storage3/_sync/file_api.py index f1e30b4a..18aa7e0f 100644 --- a/storage3/_sync/file_api.py +++ b/storage3/_sync/file_api.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field from io import BufferedReader, FileIO from pathlib import Path -from typing import Any, Optional, Union, cast +from typing import Any, Literal, Optional, Union, cast from httpx import HTTPError, Response @@ -342,8 +342,9 @@ def download(self, path: str, options: DownloadOptions = {}) -> bytes: ) return response.content - def upload( + def _upload_or_update( self, + method: Literal["POST", "PUT"], path: str, file: Union[BufferedReader, bytes, FileIO, str, Path], file_options: Optional[FileOptions] = None, @@ -365,9 +366,6 @@ def upload( file_options = {} cache_control = file_options.get("cache-control") _data = {} - if cache_control: - file_options["cache-control"] = f"max-age={cache_control}" - _data = {"cacheControl": cache_control} headers = { **self._client.headers, @@ -376,6 +374,10 @@ def upload( } filename = path.rsplit("/", maxsplit=1)[-1] + if cache_control: + headers["cache-control"] = f"max-age={cache_control}" + _data = {"cacheControl": cache_control} + if ( isinstance(file, BufferedReader) or isinstance(file, bytes) @@ -396,9 +398,38 @@ def upload( _path = self._get_final_path(path) return self._request( - "POST", f"/object/{_path}", files=files, headers=headers, data=_data + method, f"/object/{_path}", files=files, headers=headers, data=_data ) + def upload( + self, + path: str, + file: Union[BufferedReader, bytes, FileIO, str, Path], + file_options: Optional[FileOptions] = None, + ) -> Response: + """ + Uploads a file to an existing bucket. + + Parameters + ---------- + path + The relative file path including the bucket ID. Should be of the format `bucket/folder/subfolder/filename.png`. + The bucket must already exist before attempting to upload. + file + The File object to be stored in the bucket. or a async generator of chunks + file_options + HTTP headers. + """ + return self._upload_or_update("POST", path, file, file_options) + + def update( + self, + path: str, + file: Union[BufferedReader, bytes, FileIO, str, Path], + file_options: Optional[FileOptions] = None, + ) -> Response: + return self._upload_or_update("PUT", path, file, file_options) + def _get_final_path(self, path: str) -> str: return f"{self.id}/{path}" diff --git a/tests/_sync/test_client.py b/tests/_sync/test_client.py index f897dcf5..9cf62421 100644 --- a/tests/_sync/test_client.py +++ b/tests/_sync/test_client.py @@ -154,6 +154,67 @@ def file(tmp_path: Path, uuid_factory: Callable[[], str]) -> FileForTesting: ) +@pytest.fixture +def two_files(tmp_path: Path, uuid_factory: Callable[[], str]) -> list[FileForTesting]: + """Creates multiple test files (different content, same bucket/folder path, different file names)""" + file_name_1 = "test_image_1.svg" + file_name_2 = "test_image_2.svg" + file_content = ( + b' ' + b' ' + b' ' + b' ' + b' ' + ) + file_content_2 = ( + b' ' + b' ' + b' ' + b' ' + b' ' + ) + bucket_folder = uuid_factory() + bucket_path_1 = f"{bucket_folder}/{file_name_1}" + bucket_path_2 = f"{bucket_folder}/{file_name_2}" + file_path_1 = tmp_path / file_name_1 + file_path_2 = tmp_path / file_name_2 + with open(file_path_1, "wb") as f: + f.write(file_content) + with open(file_path_2, "wb") as f: + f.write(file_content_2) + + return [ + FileForTesting( + name=file_name_1, + local_path=str(file_path_1), + bucket_folder=bucket_folder, + bucket_path=bucket_path_1, + mime_type="image/svg+xml", + file_content=file_content, + ), + FileForTesting( + name=file_name_2, + local_path=str(file_path_2), + bucket_folder=bucket_folder, + bucket_path=bucket_path_2, + mime_type="image/svg+xml", + file_content=file_content_2, + ), + ] + + @pytest.fixture def multi_file(tmp_path: Path, uuid_factory: Callable[[], str]) -> list[FileForTesting]: """Creates multiple test files (same content, same bucket/folder path, different file names)""" @@ -221,6 +282,33 @@ def test_client_upload( assert image_info.get("metadata", {}).get("mimetype") == file.mime_type +def test_client_update( + storage_file_client: SyncBucketProxy, + two_files: list[FileForTesting], +) -> None: + """Ensure we can upload files to a bucket""" + storage_file_client.upload( + two_files[0].bucket_path, + two_files[0].local_path, + {"content-type": two_files[0].mime_type}, + ) + + storage_file_client.update( + two_files[0].bucket_path, + two_files[1].local_path, + {"content-type": two_files[1].mime_type}, + ) + + image = storage_file_client.download(two_files[0].bucket_path) + file_list = storage_file_client.list(two_files[0].bucket_folder) + image_info = next( + (f for f in file_list if f.get("name") == two_files[0].name), None + ) + + assert image == two_files[1].file_content + assert image_info.get("metadata", {}).get("mimetype") == two_files[1].mime_type + + @pytest.mark.parametrize( "path", ["foobar.txt", "example/nested.jpg", "/leading/slash.png"] )