Skip to content

Commit

Permalink
Remove exception handling in functions
Browse files Browse the repository at this point in the history 
Error handling will be done in the upper layer.

Signed-off-by: Volker Theile <[email protected]>
  • Loading branch information
@votdev
votdev committed Nov 24, 2023
1 parent 961ad5b commit 83c4fe7
Show file tree
Hide file tree
Showing 2 changed files with 95 additions and 96 deletions.
182 changes: 88 additions & 94 deletions src/backend/api/objects.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,14 +138,14 @@ async def stream_response(self, send: Send) -> None:

@router.post(
"/{bucket}",
response_model=Optional[List[Object]],
response_model=List[Object],
responses=s3gw_client_responses(),
)
async def list_objects(
conn: S3GWClientDep,
bucket: str,
params: ListObjectsRequest = ListObjectsRequest(),
) -> Optional[List[Object]]:
) -> List[Object]:
"""
Note that this is a POST request instead of a usual GET request
because the parameters specified in `ListObjectsRequest` need to
Expand All @@ -160,55 +160,52 @@ async def list_objects(
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/list_objects_v2.html
"""
async with conn.conn() as s3:
try:
res: List[Object] = []
continuation_token: str = ""
while True:
s3_res: ListObjectsV2OutputTypeDef = await s3.list_objects_v2(
Bucket=bucket,
Prefix=params.Prefix,
Delimiter=params.Delimiter,
ContinuationToken=continuation_token,
)
content: ObjectTypeDef
for content in s3_res.get("Contents", []):
res.append(
parse_obj_as(
Object,
{
"Name": split_key(content["Key"]).pop(),
"Type": "OBJECT",
**content,
},
)
res: List[Object] = []
continuation_token: str = ""
while True:
s3_res: ListObjectsV2OutputTypeDef = await s3.list_objects_v2(
Bucket=bucket,
Prefix=params.Prefix,
Delimiter=params.Delimiter,
ContinuationToken=continuation_token,
)
content: ObjectTypeDef
for content in s3_res.get("Contents", []):
res.append(
parse_obj_as(
Object,
{
"Name": split_key(content["Key"]).pop(),
"Type": "OBJECT",
**content,
},
)
cp: CommonPrefixTypeDef
for cp in s3_res.get("CommonPrefixes", []):
res.append(
Object(
Key=build_key(cp["Prefix"]),
Name=split_key(cp["Prefix"]).pop(),
Type="FOLDER",
)
)
cp: CommonPrefixTypeDef
for cp in s3_res.get("CommonPrefixes", []):
res.append(
Object(
Key=build_key(cp["Prefix"]),
Name=split_key(cp["Prefix"]).pop(),
Type="FOLDER",
)
if not s3_res.get("IsTruncated", False):
break
continuation_token = s3_res["NextContinuationToken"]
except s3.exceptions.ClientError:
return None
)
if not s3_res.get("IsTruncated", False):
break
continuation_token = s3_res["NextContinuationToken"]
return res


@router.post(
"/{bucket}/versions",
response_model=Optional[List[ObjectVersion]],
response_model=List[ObjectVersion],
responses=s3gw_client_responses(),
)
async def list_object_versions(
conn: S3GWClientDep,
bucket: str,
params: ListObjectVersionsRequest = ListObjectVersionsRequest(),
) -> Optional[List[ObjectVersion]]:
) -> List[ObjectVersion]:
"""
Note that this is a POST request instead of a usual GET request
because the parameters specified in `ListObjectVersionsRequest`
Expand All @@ -219,60 +216,57 @@ async def list_object_versions(
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/list_object_versions.html
"""
async with conn.conn() as s3:
try:
res: List[ObjectVersion] = []
key_marker: str = ""
while True:
s3_res: ListObjectVersionsOutputTypeDef = (
await s3.list_object_versions(
Bucket=bucket,
Prefix=params.Prefix,
Delimiter=params.Delimiter,
KeyMarker=key_marker,
)
res: List[ObjectVersion] = []
key_marker: str = ""
while True:
s3_res: ListObjectVersionsOutputTypeDef = (
await s3.list_object_versions(
Bucket=bucket,
Prefix=params.Prefix,
Delimiter=params.Delimiter,
KeyMarker=key_marker,
)
version: ObjectVersionTypeDef
for version in s3_res.get("Versions", []):
res.append(
parse_obj_as(
ObjectVersion,
{
"Name": split_key(version["Key"]).pop(),
"Type": "OBJECT",
"IsDeleted": False,
**version,
},
)
)
version: ObjectVersionTypeDef
for version in s3_res.get("Versions", []):
res.append(
parse_obj_as(
ObjectVersion,
{
"Name": split_key(version["Key"]).pop(),
"Type": "OBJECT",
"IsDeleted": False,
**version,
},
)
cp: CommonPrefixTypeDef
for cp in s3_res.get("CommonPrefixes", []):
res.append(
ObjectVersion(
Key=build_key(cp["Prefix"]),
Name=split_key(cp["Prefix"]).pop(),
Type="FOLDER",
IsDeleted=False,
IsLatest=True,
)
)
cp: CommonPrefixTypeDef
for cp in s3_res.get("CommonPrefixes", []):
res.append(
ObjectVersion(
Key=build_key(cp["Prefix"]),
Name=split_key(cp["Prefix"]).pop(),
Type="FOLDER",
IsDeleted=False,
IsLatest=True,
)
dm: DeleteMarkerEntryTypeDef
for dm in s3_res.get("DeleteMarkers", []):
res.append(
ObjectVersion.parse_obj(
{
"Name": split_key(dm["Key"]).pop(),
"Type": "OBJECT",
"Size": 0,
"IsDeleted": True,
**dm,
}
)
)
dm: DeleteMarkerEntryTypeDef
for dm in s3_res.get("DeleteMarkers", []):
res.append(
ObjectVersion.parse_obj(
{
"Name": split_key(dm["Key"]).pop(),
"Type": "OBJECT",
"Size": 0,
"IsDeleted": True,
**dm,
}
)
if not s3_res.get("IsTruncated", False):
break
key_marker = s3_res["NextKeyMarker"]
except s3.exceptions.ClientError:
return None
)
if not s3_res.get("IsTruncated", False):
break
key_marker = s3_res["NextKeyMarker"]

if params.Strict:
# Return only that object versions that exactly match the given
Expand Down Expand Up @@ -558,14 +552,14 @@ async def restore_object(
https://www.middlewareinventory.com/blog/recover-s3/
"""
# Remove existing deletion markers.
api_res: Optional[List[ObjectVersion]] = await list_object_versions(
api_res: List[ObjectVersion] = await list_object_versions(
conn,
bucket,
ListObjectVersionsRequest(Prefix=params.Key, Strict=True),
)
del_objects: List[ObjectIdentifierTypeDef] = [
parse_obj_as(ObjectIdentifierTypeDef, obj)
for obj in api_res or []
for obj in api_res
if obj.IsDeleted
]
if del_objects:
Expand Down Expand Up @@ -603,14 +597,14 @@ async def delete_object(
"""

async def collect_objects() -> List[ObjectIdentifierTypeDef]:
api_res: Optional[List[ObjectVersion]] = await list_object_versions(
api_res: List[ObjectVersion] = await list_object_versions(
conn,
bucket,
ListObjectVersionsRequest(Prefix=params.Key, Strict=True),
)
return [
parse_obj_as(ObjectIdentifierTypeDef, obj)
for obj in api_res or []
for obj in api_res
if obj.Type == "OBJECT"
]

Expand Down Expand Up @@ -641,7 +635,7 @@ async def delete_object_by_prefix(
"""

async def collect_objects(prefix: str) -> List[ObjectIdentifierTypeDef]:
api_res: Optional[List[ObjectVersion]] = await list_object_versions(
api_res: List[ObjectVersion] = await list_object_versions(
conn,
bucket,
ListObjectVersionsRequest(
Expand All @@ -650,7 +644,7 @@ async def collect_objects(prefix: str) -> List[ObjectIdentifierTypeDef]:
)
obj: ObjectVersion
res_objects: List[ObjectIdentifierTypeDef] = []
for obj in api_res or []:
for obj in api_res:
if not (params.AllVersions or (obj.IsLatest and not obj.IsDeleted)):
continue
if obj.Type == "OBJECT":
Expand Down
9 changes: 7 additions & 2 deletions src/backend/tests/unit/api/test_api_objects.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,8 +260,13 @@ async def test_get_object_list_truncated(
async def test_get_object_list_failure(
s3_client: S3GWClient,
) -> None:
res = await objects.list_objects(s3_client, bucket="not-exists")
assert res is None
with pytest.raises(HTTPException) as e:
await objects.list_objects(s3_client, bucket="not-exists")
assert e.value.status_code == 404
assert e.value.detail in [
"No such bucket",
"The specified bucket does not exist",
]


@pytest.mark.anyio
Expand Down

0 comments on commit 83c4fe7

Please sign in to comment.