Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

UI: The prefix handling is buggy at several places in the REST API #300

Merged
merged 2 commits into from
Nov 26, 2023
Merged

UI: The prefix handling is buggy at several places in the REST API #300

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6e836b2
UI: The prefix handling is buggy at several places in the REST API
votdev Nov 24, 2023
fb161d8
Remove exception handling in functions
votdev Nov 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
232 changes: 115 additions & 117 deletions src/backend/api/objects.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,14 +138,14 @@ async def stream_response(self, send: Send) -> None:

@router.post(
"/{bucket}",
response_model=Optional[List[Object]],
response_model=List[Object],
responses=s3gw_client_responses(),
)
async def list_objects(
conn: S3GWClientDep,
bucket: str,
params: ListObjectsRequest = ListObjectsRequest(),
) -> Optional[List[Object]]:
) -> List[Object]:
"""
Note that this is a POST request instead of a usual GET request
because the parameters specified in `ListObjectsRequest` need to
Expand All @@ -160,55 +160,52 @@ async def list_objects(
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/list_objects_v2.html
"""
async with conn.conn() as s3:
try:
res: List[Object] = []
continuation_token: str = ""
while True:
s3_res: ListObjectsV2OutputTypeDef = await s3.list_objects_v2(
Bucket=bucket,
Prefix=params.Prefix,
Delimiter=params.Delimiter,
ContinuationToken=continuation_token,
)
content: ObjectTypeDef
for content in s3_res.get("Contents", []):
res.append(
parse_obj_as(
Object,
{
"Name": split_key(content["Key"]).pop(),
"Type": "OBJECT",
**content,
},
)
res: List[Object] = []
continuation_token: str = ""
while True:
s3_res: ListObjectsV2OutputTypeDef = await s3.list_objects_v2(
Bucket=bucket,
Prefix=params.Prefix,
Delimiter=params.Delimiter,
ContinuationToken=continuation_token,
)
content: ObjectTypeDef
for content in s3_res.get("Contents", []):
res.append(
parse_obj_as(
Object,
{
"Name": split_key(content["Key"]).pop(),
"Type": "OBJECT",
**content,
},
)
cp: CommonPrefixTypeDef
for cp in s3_res.get("CommonPrefixes", []):
res.append(
Object(
Key=build_key(cp["Prefix"]),
Name=split_key(cp["Prefix"]).pop(),
Type="FOLDER",
)
)
cp: CommonPrefixTypeDef
for cp in s3_res.get("CommonPrefixes", []):
res.append(
Object(
Key=build_key(cp["Prefix"]),
Name=split_key(cp["Prefix"]).pop(),
Type="FOLDER",
)
if not s3_res.get("IsTruncated", False):
break
continuation_token = s3_res["NextContinuationToken"]
except s3.exceptions.ClientError:
return None
)
if not s3_res.get("IsTruncated", False):
break
continuation_token = s3_res["NextContinuationToken"]
return res


@router.post(
"/{bucket}/versions",
response_model=Optional[List[ObjectVersion]],
response_model=List[ObjectVersion],
responses=s3gw_client_responses(),
)
async def list_object_versions(
conn: S3GWClientDep,
bucket: str,
params: ListObjectVersionsRequest = ListObjectVersionsRequest(),
) -> Optional[List[ObjectVersion]]:
) -> List[ObjectVersion]:
"""
Note that this is a POST request instead of a usual GET request
because the parameters specified in `ListObjectVersionsRequest`
Expand All @@ -219,60 +216,63 @@ async def list_object_versions(
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/list_object_versions.html
"""
async with conn.conn() as s3:
try:
res: List[ObjectVersion] = []
key_marker: str = ""
while True:
s3_res: ListObjectVersionsOutputTypeDef = (
await s3.list_object_versions(
Bucket=bucket,
Prefix=params.Prefix,
Delimiter=params.Delimiter,
KeyMarker=key_marker,
)
res: List[ObjectVersion] = []
key_marker: str = ""
while True:
s3_res: ListObjectVersionsOutputTypeDef = (
await s3.list_object_versions(
Bucket=bucket,
Prefix=params.Prefix,
Delimiter=params.Delimiter,
KeyMarker=key_marker,
)
version: ObjectVersionTypeDef
for version in s3_res.get("Versions", []):
res.append(
parse_obj_as(
ObjectVersion,
{
"Name": split_key(version["Key"]).pop(),
"Type": "OBJECT",
"IsDeleted": False,
**version,
},
)
)
version: ObjectVersionTypeDef
for version in s3_res.get("Versions", []):
res.append(
parse_obj_as(
ObjectVersion,
{
"Name": split_key(version["Key"]).pop(),
"Type": "OBJECT",
"IsDeleted": False,
**version,
},
)
cp: CommonPrefixTypeDef
for cp in s3_res.get("CommonPrefixes", []):
res.append(
ObjectVersion(
Key=build_key(cp["Prefix"]),
Name=split_key(cp["Prefix"]).pop(),
Type="FOLDER",
IsDeleted=False,
IsLatest=True,
)
)
cp: CommonPrefixTypeDef
for cp in s3_res.get("CommonPrefixes", []):
res.append(
ObjectVersion(
Key=build_key(cp["Prefix"]),
Name=split_key(cp["Prefix"]).pop(),
Type="FOLDER",
IsDeleted=False,
IsLatest=True,
)
dm: DeleteMarkerEntryTypeDef
for dm in s3_res.get("DeleteMarkers", []):
res.append(
ObjectVersion.parse_obj(
{
"Name": split_key(dm["Key"]).pop(),
"Type": "OBJECT",
"Size": 0,
"IsDeleted": True,
**dm,
}
)
)
dm: DeleteMarkerEntryTypeDef
for dm in s3_res.get("DeleteMarkers", []):
res.append(
ObjectVersion.parse_obj(
{
"Name": split_key(dm["Key"]).pop(),
"Type": "OBJECT",
"Size": 0,
"IsDeleted": True,
**dm,
}
)
if not s3_res.get("IsTruncated", False):
break
key_marker = s3_res["NextKeyMarker"]
except s3.exceptions.ClientError:
return None
)
if not s3_res.get("IsTruncated", False):
break
key_marker = s3_res["NextKeyMarker"]

if params.Strict:
# Return only that object versions that exactly match the given
# prefix.
res = [obj for obj in res if obj.Key == params.Prefix]

return res


Expand Down Expand Up @@ -551,22 +551,22 @@ async def restore_object(
https://repost.aws/knowledge-center/s3-undelete-configuration
https://www.middlewareinventory.com/blog/recover-s3/
"""
# Remove existing deletion markers.
api_res: List[ObjectVersion] = await list_object_versions(
conn,
bucket,
ListObjectVersionsRequest(Prefix=params.Key, Strict=True),
)
del_objects: List[ObjectIdentifierTypeDef] = [
parse_obj_as(ObjectIdentifierTypeDef, obj)
for obj in api_res
if obj.IsDeleted
]
if del_objects:
await delete_objects(conn, bucket, del_objects)

# Make a copy of the object to restore.
async with conn.conn() as s3:
# Remove existing deletion markers.
s3_res = await s3.list_object_versions(Bucket=bucket, Prefix=params.Key)
del_objects: List[ObjectIdentifierTypeDef] = []
dm = DeleteMarkerEntryTypeDef
for dm in s3_res.get("DeleteMarkers", []):
if dm["IsLatest"]:
del_objects.append(
{"Key": dm["Key"], "VersionId": dm["VersionId"]}
)
if del_objects:
await s3.delete_objects(
Bucket=bucket,
Delete={"Objects": del_objects, "Quiet": True},
)
# Make a copy of the object to restore.
copy_source: CopySourceTypeDef = {
"Bucket": bucket,
"Key": params.Key,
Expand Down Expand Up @@ -597,21 +597,16 @@ async def delete_object(
"""

async def collect_objects() -> List[ObjectIdentifierTypeDef]:
api_res: Optional[List[ObjectVersion]] = await list_object_versions(
api_res: List[ObjectVersion] = await list_object_versions(
conn,
bucket,
ListObjectVersionsRequest(Prefix=params.Key, Delimiter=""),
ListObjectVersionsRequest(Prefix=params.Key, Strict=True),
)
obj: ObjectVersion
res_objects: List[ObjectIdentifierTypeDef] = []
for obj in api_res or []:
# Skip "virtual folders" and objects that do not match
# the given key.
if obj.Type != "OBJECT" or obj.Key != params.Key:
continue
version_id: str = obj.VersionId if obj.VersionId else ""
res_objects.append({"Key": obj.Key, "VersionId": version_id})
return res_objects
return [
parse_obj_as(ObjectIdentifierTypeDef, obj)
for obj in api_res
if obj.Type == "OBJECT"
]

objects: List[ObjectIdentifierTypeDef]
if params.AllVersions:
Expand Down Expand Up @@ -640,7 +635,7 @@ async def delete_object_by_prefix(
"""

async def collect_objects(prefix: str) -> List[ObjectIdentifierTypeDef]:
api_res: Optional[List[ObjectVersion]] = await list_object_versions(
api_res: List[ObjectVersion] = await list_object_versions(
conn,
bucket,
ListObjectVersionsRequest(
Expand All @@ -649,7 +644,7 @@ async def collect_objects(prefix: str) -> List[ObjectIdentifierTypeDef]:
)
obj: ObjectVersion
res_objects: List[ObjectIdentifierTypeDef] = []
for obj in api_res or []:
for obj in api_res:
if not (params.AllVersions or (obj.IsLatest and not obj.IsDeleted)):
continue
if obj.Type == "OBJECT":
Expand Down Expand Up @@ -678,6 +673,9 @@ async def collect_objects(prefix: str) -> List[ObjectIdentifierTypeDef]:
async def delete_objects(
conn: S3GWClientDep, bucket: str, objects: List[ObjectIdentifierTypeDef]
) -> List[DeletedObject]:
"""
Helper function to delete the specified objects.
"""
async with conn.conn() as s3:
s3_res: DeleteObjectsOutputTypeDef = await s3.delete_objects(
Bucket=bucket, Delete={"Objects": objects}
Expand Down
10 changes: 7 additions & 3 deletions src/backend/api/types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,11 @@ class ListObjectsRequest(BaseModel):


class ListObjectVersionsRequest(ListObjectsRequest):
pass
Strict: bool = Field(
default=False,
description="If `True`, then only the objects whose key "
"exactly match the specified prefix are returned.",
)


class ObjectVersion(Object):
Expand Down Expand Up @@ -165,7 +169,7 @@ class RestoreObjectRequest(ObjectIdentifier):

class DeleteObjectRequest(ObjectIdentifier):
AllVersions: bool = Field(
False,
default=False,
description="If `True`, all versions will be deleted, otherwise "
"only the specified one.",
)
Expand All @@ -180,7 +184,7 @@ class DeleteObjectByPrefixRequest(BaseModel):
)
Delimiter: str = "/"
AllVersions: bool = Field(
False,
default=False,
description="If `True`, all versions will be deleted, otherwise "
"the latest one.",
)
Loading