Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add delimiter support for S3 List API #2996

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add delimiter support for S3 List API #2996

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 17 additions & 5 deletions ambry-api/src/main/java/com/github/ambry/frontend/NamedBlobListEntry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,40 +29,47 @@ public class NamedBlobListEntry {
private static final String EXPIRATION_TIME_MS_KEY = "expirationTimeMs";
private static final String BLOB_SIZE_KEY = "blobSize";
private static final String MODIFIED_TIME_MS_KEY = "modifiedTimeMs";
private static final String IS_DIRECTORY_KEY = "isDirectory";

private final String blobName;
private final long expirationTimeMs;
private final long blobSize;
private final long modifiedTimeMs;
private final boolean isDirectory;

/**
* Read a {@link NamedBlobRecord} from JSON.
* @param jsonObject the {@link JSONObject} to deserialize.
*/
public NamedBlobListEntry(JSONObject jsonObject) {
this(jsonObject.getString(BLOB_NAME_KEY), jsonObject.optLong(EXPIRATION_TIME_MS_KEY, Utils.Infinite_Time),
jsonObject.optLong(BLOB_SIZE_KEY, 0), jsonObject.optLong(MODIFIED_TIME_MS_KEY, Utils.Infinite_Time));
jsonObject.optLong(BLOB_SIZE_KEY, 0), jsonObject.optLong(MODIFIED_TIME_MS_KEY, Utils.Infinite_Time),
jsonObject.optBoolean(IS_DIRECTORY_KEY, false));
}

/**
* Convert a {@link NamedBlobRecord} into a {@link NamedBlobListEntry}.
* @param record the {@link NamedBlobRecord}.
*/
NamedBlobListEntry(NamedBlobRecord record) {
this(record.getBlobName(), record.getExpirationTimeMs(), record.getBlobSize(), record.getModifiedTimeMs());
this(record.getBlobName(), record.getExpirationTimeMs(), record.getBlobSize(), record.getModifiedTimeMs(),
record.isDirectory());
}

/**
* @param blobName the blob name within a container.
* @param blobName the blob name within a container.
* @param expirationTimeMs the expiration time in milliseconds since epoch, or -1 if the blob should be permanent.
* @param blobSize the size of the blob
* @param modifiedTimeMs the modified time of the blob in milliseconds since epoch
* @param isDirectory whether the blob is a directory (virtual folder name separated by '/')
*/
private NamedBlobListEntry(String blobName, long expirationTimeMs, long blobSize, long modifiedTimeMs) {
private NamedBlobListEntry(String blobName, long expirationTimeMs, long blobSize, long modifiedTimeMs,
boolean isDirectory) {
this.blobName = blobName;
this.expirationTimeMs = expirationTimeMs;
this.blobSize = blobSize;
this.modifiedTimeMs = modifiedTimeMs;
this.isDirectory = isDirectory;
}

/**
Expand Down Expand Up @@ -103,6 +110,7 @@ public JSONObject toJson() {
}
jsonObject.put(BLOB_SIZE_KEY, blobSize);
jsonObject.put(MODIFIED_TIME_MS_KEY, modifiedTimeMs);
jsonObject.put(IS_DIRECTORY_KEY, isDirectory);
return jsonObject;
}

Expand All @@ -116,6 +124,10 @@ public boolean equals(Object o) {
}
NamedBlobListEntry that = (NamedBlobListEntry) o;
return expirationTimeMs == that.expirationTimeMs && Objects.equals(blobName, that.blobName)
&& modifiedTimeMs == that.modifiedTimeMs && blobSize == that.blobSize;
&& modifiedTimeMs == that.modifiedTimeMs && blobSize == that.blobSize && isDirectory == that.isDirectory;
}

public boolean isDirectory() {
return isDirectory;
}
}
61 changes: 54 additions & 7 deletions ambry-api/src/main/java/com/github/ambry/frontend/s3/S3MessagePayload.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
*/
package com.github.ambry.frontend.s3;

import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.List;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
Expand Down Expand Up @@ -154,6 +155,7 @@ public String toString() {
}
}

@JsonInclude(JsonInclude.Include.NON_EMPTY)
public static abstract class AbstractListBucketResult {
@JacksonXmlProperty(localName = "Name")
private String name;
Expand All @@ -172,13 +174,17 @@ public static abstract class AbstractListBucketResult {
private String encodingType;
@JacksonXmlProperty(localName = "IsTruncated")
private Boolean isTruncated;
// New optional field for CommonPrefixes with wrapping and nested Prefix elements
@JacksonXmlProperty(localName = "CommonPrefixes")
@JacksonXmlElementWrapper(useWrapping = false)
private List<Prefix> commonPrefixes;

private AbstractListBucketResult() {

}

public AbstractListBucketResult(String name, String prefix, int maxKeys, int keyCount, String delimiter,
List<Contents> contents, String encodingType, boolean isTruncated) {
List<Contents> contents, String encodingType, boolean isTruncated, List<Prefix> commonPrefixes) {
this.name = name;
this.prefix = prefix;
this.maxKeys = maxKeys;
Expand All @@ -187,6 +193,7 @@ public AbstractListBucketResult(String name, String prefix, int maxKeys, int key
this.contents = contents;
this.encodingType = encodingType;
this.isTruncated = isTruncated;
this.commonPrefixes = commonPrefixes;
}

public String getPrefix() {
Expand Down Expand Up @@ -220,13 +227,23 @@ public boolean getIsTruncated() {
@Override
public String toString() {
return "Name=" + name + ", Prefix=" + prefix + ", MaxKeys=" + maxKeys + ", KeyCount=" + keyCount + ", Delimiter="
+ delimiter + ", Contents=" + contents + ", Encoding type=" + encodingType + ", IsTruncated=" + isTruncated;
+ delimiter + ", Contents=" + contents + ", Encoding type=" + encodingType + ", IsTruncated=" + isTruncated
+ ", CommonPrefixes=" + commonPrefixes;
}

public List<Prefix> getCommonPrefixes() {
return commonPrefixes;
}

public void setCommonPrefixes(List<Prefix> commonPrefixes) {
this.commonPrefixes = commonPrefixes;
}
}

/**
* ListBucketResult for listObjects API.
*/
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public static class ListBucketResult extends AbstractListBucketResult {
@JacksonXmlProperty(localName = "Marker")
private String marker;
Expand All @@ -238,8 +255,9 @@ private ListBucketResult() {
}

public ListBucketResult(String name, String prefix, int maxKeys, int keyCount, String delimiter,
List<Contents> contents, String encodingType, String marker, String nextMarker, boolean isTruncated) {
super(name, prefix, maxKeys, keyCount, delimiter, contents, encodingType, isTruncated);
List<Contents> contents, String encodingType, String marker, String nextMarker, boolean isTruncated,
List<Prefix> commonPrefixes) {
super(name, prefix, maxKeys, keyCount, delimiter, contents, encodingType, isTruncated, commonPrefixes);
this.marker = marker;
this.nextMarker = nextMarker;
}
Expand All @@ -261,6 +279,7 @@ public String toString() {
/**
* ListBucketResult for listObjectsV2 API.
*/
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public static class ListBucketResultV2 extends AbstractListBucketResult {
@JacksonXmlProperty(localName = "ContinuationToken")
private String continuationToken;
Expand All @@ -273,8 +292,8 @@ private ListBucketResultV2() {

public ListBucketResultV2(String name, String prefix, int maxKeys, int keyCount, String delimiter,
List<Contents> contents, String encodingType, String continuationToken, String nextContinuationToken,
boolean isTruncated) {
super(name, prefix, maxKeys, keyCount, delimiter, contents, encodingType, isTruncated);
boolean isTruncated, List<Prefix> commonPrefixes) {
super(name, prefix, maxKeys, keyCount, delimiter, contents, encodingType, isTruncated, commonPrefixes);
this.continuationToken = continuationToken;
this.nextContinuationToken = nextContinuationToken;
}
Expand Down Expand Up @@ -315,7 +334,9 @@ public String getKey() {
return key;
}

public long getSize() { return size; }
public long getSize() {
return size;
}

public String getLastModified() {
return lastModified;
Expand Down Expand Up @@ -351,4 +372,30 @@ public String toString() {
return "Bucket=" + bucket + ", Key=" + key + ", UploadId=" + uploadId;
}
}

// Inner class for wrapping each Prefix inside CommmomPrefixes
public static class Prefix {
@JacksonXmlProperty(localName = "Prefix")
private String prefix;

public Prefix() {
}

public Prefix(String prefix) {
this.prefix = prefix;
}

public String getPrefix() {
return prefix;
}

public void setPrefix(String prefix) {
this.prefix = prefix;
}

@Override
public String toString() {
return "Prefix=" + prefix;
}
}
}
20 changes: 11 additions & 9 deletions ambry-api/src/main/java/com/github/ambry/named/NamedBlobDb.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,19 +54,21 @@ default CompletableFuture<NamedBlobRecord> get(String accountName, String contai
* List blobs that start with a provided prefix in a container. This returns paginated results. If there are
* additional pages to read, {@link Page#getNextPageToken()} will be non null.
*
* @param accountName the name of the account.
* @param containerName the name of the container.
* @param blobNamePrefix the name prefix to search for.
* @param pageToken if {@code null}, return the first page of {@link NamedBlobRecord}s that start with
* {@code blobNamePrefix}. If set, use this as a token to resume reading additional pages of
* records that start with the prefix.
* @param maxKey the maximum number of keys returned in the response. By default, the action returns up to listMaxResults
* which can be tuned by config.
* @param accountName the name of the account.
* @param containerName the name of the container.
* @param blobNamePrefix the name prefix to search for.
* @param pageToken if {@code null}, return the first page of {@link NamedBlobRecord}s that start with
* {@code blobNamePrefix}. If set, use this as a token to resume reading additional pages of
* records that start with the prefix.
* @param maxKey the maximum number of keys returned in the response. By default, the action returns up to
* listMaxResults which can be tuned by config.
* @param groupDirectories if true, group the blobs by directory. (Blobs with the same directory name will be grouped
* and only the directory names will be returned)
* @return a {@link CompletableFuture} that will eventually contain a {@link Page} of {@link NamedBlobRecord}s
* starting with the specified prefix or an exception if an error occurred.
*/
CompletableFuture<Page<NamedBlobRecord>> list(String accountName, String containerName, String blobNamePrefix,
String pageToken, Integer maxKey);
String pageToken, Integer maxKey, boolean groupDirectories);

/**
* Persist a {@link NamedBlobRecord} in the database.
Expand Down
14 changes: 12 additions & 2 deletions ambry-api/src/main/java/com/github/ambry/named/NamedBlobRecord.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ public class NamedBlobRecord {
private final String blobId;
private final long blobSize;
private long modifiedTimeMs;
private final boolean isDirectory;

/**
* @param accountName the account name.
Expand Down Expand Up @@ -67,7 +68,7 @@ public NamedBlobRecord(String accountName, String containerName, String blobName
*/
public NamedBlobRecord(String accountName, String containerName, String blobName, String blobId,
long expirationTimeMs, long version, long blobSize) {
this(accountName, containerName, blobName, blobId, expirationTimeMs, version, blobSize, 0);
this(accountName, containerName, blobName, blobId, expirationTimeMs, version, blobSize, 0, false);
}

/**
Expand All @@ -79,9 +80,10 @@ public NamedBlobRecord(String accountName, String containerName, String blobName
* @param version the version of this named blob.
* @param blobSize the size of the blob.
* @param modifiedTimeMs the modified time of the blob in milliseconds since epoch
* @param isDirectory whether the blob is a directory (virtual folder name separated by '/')
*/
public NamedBlobRecord(String accountName, String containerName, String blobName, String blobId,
long expirationTimeMs, long version, long blobSize, long modifiedTimeMs) {
long expirationTimeMs, long version, long blobSize, long modifiedTimeMs, boolean isDirectory) {
this.accountName = accountName;
this.containerName = containerName;
this.blobName = blobName;
Expand All @@ -90,6 +92,7 @@ public NamedBlobRecord(String accountName, String containerName, String blobName
this.version = version;
this.blobSize = blobSize;
this.modifiedTimeMs = modifiedTimeMs;
this.isDirectory = isDirectory;
}

/**
Expand Down Expand Up @@ -180,4 +183,11 @@ public long getModifiedTimeMs() {
public void setModifiedTimeMs(long modifiedTimeMs) {
this.modifiedTimeMs = modifiedTimeMs;
}

/**
* @return whether the blob is a directory (virtual folder name separated by '/')
*/
public boolean isDirectory() {
return isDirectory;
}
}
4 changes: 3 additions & 1 deletion ambry-frontend/src/main/java/com/github/ambry/frontend/NamedBlobListHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,12 @@ private Callback<Void> securityPostProcessRequestCallback() {
return buildCallback(frontendMetrics.listSecurityPostProcessRequestMetrics, securityCheckResult -> {
NamedBlobPath namedBlobPath = NamedBlobPath.parse(RestUtils.getRequestPath(restRequest), restRequest.getArgs());
String maxKeys = getHeader(restRequest.getArgs(), MAXKEYS_PARAM_NAME, false);
String delimiter = getHeader(restRequest.getArgs(), DELIMITER_PARAM_NAME, false);
CallbackUtils.callCallbackAfter(
namedBlobDb.list(namedBlobPath.getAccountName(), namedBlobPath.getContainerName(),
namedBlobPath.getBlobNamePrefix(), namedBlobPath.getPageToken(),
maxKeys == null ? null : Integer.parseInt(maxKeys)), listBlobsCallback());
maxKeys == null ? null : Integer.parseInt(maxKeys), delimiter != null && delimiter.equals("/")),
listBlobsCallback());
}, uri, LOGGER, finalCallback);
}

Expand Down
36 changes: 25 additions & 11 deletions ambry-frontend/src/main/java/com/github/ambry/frontend/s3/S3ListHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.xml.XmlMapper;
import com.github.ambry.account.Container;
import com.github.ambry.commons.ByteBufferReadableStreamChannel;
import com.github.ambry.commons.Callback;
import com.github.ambry.frontend.FrontendMetrics;
Expand All @@ -30,18 +31,15 @@
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.json.JSONObject;
import org.json.JSONTokener;
import org.slf4j.Logger;
Expand Down Expand Up @@ -104,6 +102,8 @@ protected void doHandle(RestRequest restRequest, RestResponseChannel restRespons
private ReadableStreamChannel serializeAsXml(RestRequest restRequest, Page<NamedBlobListEntry> namedBlobRecordPage)
throws IOException, RestServiceException {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
Container container = (Container) restRequest.getArgs().get(InternalKeys.TARGET_CONTAINER_KEY);
String containerName = container.getName();
String prefix = getHeader(restRequest.getArgs(), PREFIX_PARAM_NAME, false);
String delimiter = getHeader(restRequest.getArgs(), DELIMITER_PARAM_NAME, false);
String encodingType = getHeader(restRequest.getArgs(), ENCODING_TYPE_PARAM_NAME, false);
Expand All @@ -115,7 +115,15 @@ private ReadableStreamChannel serializeAsXml(RestRequest restRequest, Page<Named
// Iterate through list of blob names.
List<Contents> contentsList = new ArrayList<>();
int keyCount = 0;
Set<String> dirSet = new HashSet<>();
for (NamedBlobListEntry namedBlobRecord : namedBlobRecordPage.getEntries()) {
if (namedBlobRecord.isDirectory()) {
dirSet.add(namedBlobRecord.getBlobName());
if (++keyCount == maxKeysValue) {
break;
}
continue;
}
String blobName = namedBlobRecord.getBlobName();
long blobSize = namedBlobRecord.getBlobSize();
long modifiedTimeMs = namedBlobRecord.getModifiedTimeMs();
Expand All @@ -129,19 +137,25 @@ private ReadableStreamChannel serializeAsXml(RestRequest restRequest, Page<Named
break;
}
}

List<Prefix> commonPrefixes = new ArrayList<>();
for (String dir : dirSet) {
commonPrefixes.add(new Prefix(dir));
}

if (LIST_TYPE_VERSION_2.equals(getHeader(restRequest.getArgs(), LIST_TYPE, false))) {
ListBucketResultV2 resultV2 =
new ListBucketResultV2(restRequest.getPath(), prefix, maxKeysValue, keyCount, delimiter, contentsList,
encodingType, continuationToken, namedBlobRecordPage.getNextPageToken(),
namedBlobRecordPage.getNextPageToken() != null);
new ListBucketResultV2(containerName, prefix, maxKeysValue, keyCount, delimiter, contentsList, encodingType,
continuationToken, namedBlobRecordPage.getNextPageToken(), namedBlobRecordPage.getNextPageToken() != null,
delimiter == null ? null : commonPrefixes);
LOGGER.debug("Sending response for S3 ListObjects {}", resultV2);
// Serialize xml
xmlMapper.writeValue(outputStream, resultV2);
} else {
ListBucketResult result =
new ListBucketResult(restRequest.getPath(), prefix, maxKeysValue, keyCount, delimiter, contentsList,
encodingType, marker, namedBlobRecordPage.getNextPageToken(),
namedBlobRecordPage.getNextPageToken() != null);
new ListBucketResult(containerName, prefix, maxKeysValue, keyCount, delimiter, contentsList, encodingType,
marker, namedBlobRecordPage.getNextPageToken(), namedBlobRecordPage.getNextPageToken() != null,
delimiter == null ? null : commonPrefixes);
LOGGER.debug("Sending response for S3 ListObjects {}", result);
// Serialize xml
xmlMapper.writeValue(outputStream, result);
Expand Down
Loading
Loading