diff --git a/ambry-api/src/main/java/com/github/ambry/frontend/NamedBlobListEntry.java b/ambry-api/src/main/java/com/github/ambry/frontend/NamedBlobListEntry.java index 523ed786ab..9b9c468518 100644 --- a/ambry-api/src/main/java/com/github/ambry/frontend/NamedBlobListEntry.java +++ b/ambry-api/src/main/java/com/github/ambry/frontend/NamedBlobListEntry.java @@ -29,11 +29,13 @@ public class NamedBlobListEntry { private static final String EXPIRATION_TIME_MS_KEY = "expirationTimeMs"; private static final String BLOB_SIZE_KEY = "blobSize"; private static final String MODIFIED_TIME_MS_KEY = "modifiedTimeMs"; + private static final String IS_DIRECTORY_KEY = "isDirectory"; private final String blobName; private final long expirationTimeMs; private final long blobSize; private final long modifiedTimeMs; + private final boolean isDirectory; /** * Read a {@link NamedBlobRecord} from JSON. @@ -41,7 +43,8 @@ public class NamedBlobListEntry { */ public NamedBlobListEntry(JSONObject jsonObject) { this(jsonObject.getString(BLOB_NAME_KEY), jsonObject.optLong(EXPIRATION_TIME_MS_KEY, Utils.Infinite_Time), - jsonObject.optLong(BLOB_SIZE_KEY, 0), jsonObject.optLong(MODIFIED_TIME_MS_KEY, Utils.Infinite_Time)); + jsonObject.optLong(BLOB_SIZE_KEY, 0), jsonObject.optLong(MODIFIED_TIME_MS_KEY, Utils.Infinite_Time), + jsonObject.optBoolean(IS_DIRECTORY_KEY, false)); } /** @@ -49,20 +52,24 @@ public NamedBlobListEntry(JSONObject jsonObject) { * @param record the {@link NamedBlobRecord}. */ NamedBlobListEntry(NamedBlobRecord record) { - this(record.getBlobName(), record.getExpirationTimeMs(), record.getBlobSize(), record.getModifiedTimeMs()); + this(record.getBlobName(), record.getExpirationTimeMs(), record.getBlobSize(), record.getModifiedTimeMs(), + record.isDirectory()); } /** - * @param blobName the blob name within a container. + * @param blobName the blob name within a container. * @param expirationTimeMs the expiration time in milliseconds since epoch, or -1 if the blob should be permanent. * @param blobSize the size of the blob * @param modifiedTimeMs the modified time of the blob in milliseconds since epoch + * @param isDirectory whether the blob is a directory (virtual folder name separated by '/') */ - private NamedBlobListEntry(String blobName, long expirationTimeMs, long blobSize, long modifiedTimeMs) { + private NamedBlobListEntry(String blobName, long expirationTimeMs, long blobSize, long modifiedTimeMs, + boolean isDirectory) { this.blobName = blobName; this.expirationTimeMs = expirationTimeMs; this.blobSize = blobSize; this.modifiedTimeMs = modifiedTimeMs; + this.isDirectory = isDirectory; } /** @@ -103,6 +110,7 @@ public JSONObject toJson() { } jsonObject.put(BLOB_SIZE_KEY, blobSize); jsonObject.put(MODIFIED_TIME_MS_KEY, modifiedTimeMs); + jsonObject.put(IS_DIRECTORY_KEY, isDirectory); return jsonObject; } @@ -116,6 +124,10 @@ public boolean equals(Object o) { } NamedBlobListEntry that = (NamedBlobListEntry) o; return expirationTimeMs == that.expirationTimeMs && Objects.equals(blobName, that.blobName) - && modifiedTimeMs == that.modifiedTimeMs && blobSize == that.blobSize; + && modifiedTimeMs == that.modifiedTimeMs && blobSize == that.blobSize && isDirectory == that.isDirectory; + } + + public boolean isDirectory() { + return isDirectory; } } diff --git a/ambry-api/src/main/java/com/github/ambry/frontend/s3/S3MessagePayload.java b/ambry-api/src/main/java/com/github/ambry/frontend/s3/S3MessagePayload.java index 8f57876d26..8465605139 100644 --- a/ambry-api/src/main/java/com/github/ambry/frontend/s3/S3MessagePayload.java +++ b/ambry-api/src/main/java/com/github/ambry/frontend/s3/S3MessagePayload.java @@ -14,6 +14,7 @@ */ package com.github.ambry.frontend.s3; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.List; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; @@ -154,6 +155,7 @@ public String toString() { } } + @JsonInclude(JsonInclude.Include.NON_EMPTY) public static abstract class AbstractListBucketResult { @JacksonXmlProperty(localName = "Name") private String name; @@ -172,13 +174,17 @@ public static abstract class AbstractListBucketResult { private String encodingType; @JacksonXmlProperty(localName = "IsTruncated") private Boolean isTruncated; + // New optional field for CommonPrefixes with wrapping and nested Prefix elements + @JacksonXmlProperty(localName = "CommonPrefixes") + @JacksonXmlElementWrapper(useWrapping = false) + private List commonPrefixes; private AbstractListBucketResult() { } public AbstractListBucketResult(String name, String prefix, int maxKeys, int keyCount, String delimiter, - List contents, String encodingType, boolean isTruncated) { + List contents, String encodingType, boolean isTruncated, List commonPrefixes) { this.name = name; this.prefix = prefix; this.maxKeys = maxKeys; @@ -187,6 +193,7 @@ public AbstractListBucketResult(String name, String prefix, int maxKeys, int key this.contents = contents; this.encodingType = encodingType; this.isTruncated = isTruncated; + this.commonPrefixes = commonPrefixes; } public String getPrefix() { @@ -220,13 +227,23 @@ public boolean getIsTruncated() { @Override public String toString() { return "Name=" + name + ", Prefix=" + prefix + ", MaxKeys=" + maxKeys + ", KeyCount=" + keyCount + ", Delimiter=" - + delimiter + ", Contents=" + contents + ", Encoding type=" + encodingType + ", IsTruncated=" + isTruncated; + + delimiter + ", Contents=" + contents + ", Encoding type=" + encodingType + ", IsTruncated=" + isTruncated + + ", CommonPrefixes=" + commonPrefixes; + } + + public List getCommonPrefixes() { + return commonPrefixes; + } + + public void setCommonPrefixes(List commonPrefixes) { + this.commonPrefixes = commonPrefixes; } } /** * ListBucketResult for listObjects API. */ + @JsonInclude(JsonInclude.Include.NON_EMPTY) public static class ListBucketResult extends AbstractListBucketResult { @JacksonXmlProperty(localName = "Marker") private String marker; @@ -238,8 +255,9 @@ private ListBucketResult() { } public ListBucketResult(String name, String prefix, int maxKeys, int keyCount, String delimiter, - List contents, String encodingType, String marker, String nextMarker, boolean isTruncated) { - super(name, prefix, maxKeys, keyCount, delimiter, contents, encodingType, isTruncated); + List contents, String encodingType, String marker, String nextMarker, boolean isTruncated, + List commonPrefixes) { + super(name, prefix, maxKeys, keyCount, delimiter, contents, encodingType, isTruncated, commonPrefixes); this.marker = marker; this.nextMarker = nextMarker; } @@ -261,6 +279,7 @@ public String toString() { /** * ListBucketResult for listObjectsV2 API. */ + @JsonInclude(JsonInclude.Include.NON_EMPTY) public static class ListBucketResultV2 extends AbstractListBucketResult { @JacksonXmlProperty(localName = "ContinuationToken") private String continuationToken; @@ -273,8 +292,8 @@ private ListBucketResultV2() { public ListBucketResultV2(String name, String prefix, int maxKeys, int keyCount, String delimiter, List contents, String encodingType, String continuationToken, String nextContinuationToken, - boolean isTruncated) { - super(name, prefix, maxKeys, keyCount, delimiter, contents, encodingType, isTruncated); + boolean isTruncated, List commonPrefixes) { + super(name, prefix, maxKeys, keyCount, delimiter, contents, encodingType, isTruncated, commonPrefixes); this.continuationToken = continuationToken; this.nextContinuationToken = nextContinuationToken; } @@ -315,7 +334,9 @@ public String getKey() { return key; } - public long getSize() { return size; } + public long getSize() { + return size; + } public String getLastModified() { return lastModified; @@ -351,4 +372,30 @@ public String toString() { return "Bucket=" + bucket + ", Key=" + key + ", UploadId=" + uploadId; } } + + // Inner class for wrapping each Prefix inside CommmomPrefixes + public static class Prefix { + @JacksonXmlProperty(localName = "Prefix") + private String prefix; + + public Prefix() { + } + + public Prefix(String prefix) { + this.prefix = prefix; + } + + public String getPrefix() { + return prefix; + } + + public void setPrefix(String prefix) { + this.prefix = prefix; + } + + @Override + public String toString() { + return "Prefix=" + prefix; + } + } } diff --git a/ambry-api/src/main/java/com/github/ambry/named/NamedBlobDb.java b/ambry-api/src/main/java/com/github/ambry/named/NamedBlobDb.java index 5c94ef076a..ea1b6d9010 100644 --- a/ambry-api/src/main/java/com/github/ambry/named/NamedBlobDb.java +++ b/ambry-api/src/main/java/com/github/ambry/named/NamedBlobDb.java @@ -54,19 +54,21 @@ default CompletableFuture get(String accountName, String contai * List blobs that start with a provided prefix in a container. This returns paginated results. If there are * additional pages to read, {@link Page#getNextPageToken()} will be non null. * - * @param accountName the name of the account. - * @param containerName the name of the container. - * @param blobNamePrefix the name prefix to search for. - * @param pageToken if {@code null}, return the first page of {@link NamedBlobRecord}s that start with - * {@code blobNamePrefix}. If set, use this as a token to resume reading additional pages of - * records that start with the prefix. - * @param maxKey the maximum number of keys returned in the response. By default, the action returns up to listMaxResults - * which can be tuned by config. + * @param accountName the name of the account. + * @param containerName the name of the container. + * @param blobNamePrefix the name prefix to search for. + * @param pageToken if {@code null}, return the first page of {@link NamedBlobRecord}s that start with + * {@code blobNamePrefix}. If set, use this as a token to resume reading additional pages of + * records that start with the prefix. + * @param maxKey the maximum number of keys returned in the response. By default, the action returns up to + * listMaxResults which can be tuned by config. + * @param groupDirectories if true, group the blobs by directory. (Blobs with the same directory name will be grouped + * and only the directory names will be returned) * @return a {@link CompletableFuture} that will eventually contain a {@link Page} of {@link NamedBlobRecord}s * starting with the specified prefix or an exception if an error occurred. */ CompletableFuture> list(String accountName, String containerName, String blobNamePrefix, - String pageToken, Integer maxKey); + String pageToken, Integer maxKey, boolean groupDirectories); /** * Persist a {@link NamedBlobRecord} in the database. diff --git a/ambry-api/src/main/java/com/github/ambry/named/NamedBlobRecord.java b/ambry-api/src/main/java/com/github/ambry/named/NamedBlobRecord.java index 48b11fc369..99ca9b85bd 100644 --- a/ambry-api/src/main/java/com/github/ambry/named/NamedBlobRecord.java +++ b/ambry-api/src/main/java/com/github/ambry/named/NamedBlobRecord.java @@ -30,6 +30,7 @@ public class NamedBlobRecord { private final String blobId; private final long blobSize; private long modifiedTimeMs; + private final boolean isDirectory; /** * @param accountName the account name. @@ -67,7 +68,7 @@ public NamedBlobRecord(String accountName, String containerName, String blobName */ public NamedBlobRecord(String accountName, String containerName, String blobName, String blobId, long expirationTimeMs, long version, long blobSize) { - this(accountName, containerName, blobName, blobId, expirationTimeMs, version, blobSize, 0); + this(accountName, containerName, blobName, blobId, expirationTimeMs, version, blobSize, 0, false); } /** @@ -79,9 +80,10 @@ public NamedBlobRecord(String accountName, String containerName, String blobName * @param version the version of this named blob. * @param blobSize the size of the blob. * @param modifiedTimeMs the modified time of the blob in milliseconds since epoch + * @param isDirectory whether the blob is a directory (virtual folder name separated by '/') */ public NamedBlobRecord(String accountName, String containerName, String blobName, String blobId, - long expirationTimeMs, long version, long blobSize, long modifiedTimeMs) { + long expirationTimeMs, long version, long blobSize, long modifiedTimeMs, boolean isDirectory) { this.accountName = accountName; this.containerName = containerName; this.blobName = blobName; @@ -90,6 +92,7 @@ public NamedBlobRecord(String accountName, String containerName, String blobName this.version = version; this.blobSize = blobSize; this.modifiedTimeMs = modifiedTimeMs; + this.isDirectory = isDirectory; } /** @@ -180,4 +183,11 @@ public long getModifiedTimeMs() { public void setModifiedTimeMs(long modifiedTimeMs) { this.modifiedTimeMs = modifiedTimeMs; } + + /** + * @return whether the blob is a directory (virtual folder name separated by '/') + */ + public boolean isDirectory() { + return isDirectory; + } } diff --git a/ambry-frontend/src/main/java/com/github/ambry/frontend/NamedBlobListHandler.java b/ambry-frontend/src/main/java/com/github/ambry/frontend/NamedBlobListHandler.java index c10e3bb20c..99be85c540 100644 --- a/ambry-frontend/src/main/java/com/github/ambry/frontend/NamedBlobListHandler.java +++ b/ambry-frontend/src/main/java/com/github/ambry/frontend/NamedBlobListHandler.java @@ -132,10 +132,12 @@ private Callback securityPostProcessRequestCallback() { return buildCallback(frontendMetrics.listSecurityPostProcessRequestMetrics, securityCheckResult -> { NamedBlobPath namedBlobPath = NamedBlobPath.parse(RestUtils.getRequestPath(restRequest), restRequest.getArgs()); String maxKeys = getHeader(restRequest.getArgs(), MAXKEYS_PARAM_NAME, false); + String delimiter = getHeader(restRequest.getArgs(), DELIMITER_PARAM_NAME, false); CallbackUtils.callCallbackAfter( namedBlobDb.list(namedBlobPath.getAccountName(), namedBlobPath.getContainerName(), namedBlobPath.getBlobNamePrefix(), namedBlobPath.getPageToken(), - maxKeys == null ? null : Integer.parseInt(maxKeys)), listBlobsCallback()); + maxKeys == null ? null : Integer.parseInt(maxKeys), delimiter != null && delimiter.equals("/")), + listBlobsCallback()); }, uri, LOGGER, finalCallback); } diff --git a/ambry-frontend/src/main/java/com/github/ambry/frontend/s3/S3ListHandler.java b/ambry-frontend/src/main/java/com/github/ambry/frontend/s3/S3ListHandler.java index eac9a73b88..ae1098396a 100644 --- a/ambry-frontend/src/main/java/com/github/ambry/frontend/s3/S3ListHandler.java +++ b/ambry-frontend/src/main/java/com/github/ambry/frontend/s3/S3ListHandler.java @@ -16,6 +16,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import com.github.ambry.account.Container; import com.github.ambry.commons.ByteBufferReadableStreamChannel; import com.github.ambry.commons.Callback; import com.github.ambry.frontend.FrontendMetrics; @@ -30,18 +31,15 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.text.SimpleDateFormat; import java.time.Instant; -import java.time.LocalDateTime; import java.time.ZoneId; -import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; import java.util.GregorianCalendar; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.json.JSONObject; import org.json.JSONTokener; import org.slf4j.Logger; @@ -104,6 +102,8 @@ protected void doHandle(RestRequest restRequest, RestResponseChannel restRespons private ReadableStreamChannel serializeAsXml(RestRequest restRequest, Page namedBlobRecordPage) throws IOException, RestServiceException { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + Container container = (Container) restRequest.getArgs().get(InternalKeys.TARGET_CONTAINER_KEY); + String containerName = container.getName(); String prefix = getHeader(restRequest.getArgs(), PREFIX_PARAM_NAME, false); String delimiter = getHeader(restRequest.getArgs(), DELIMITER_PARAM_NAME, false); String encodingType = getHeader(restRequest.getArgs(), ENCODING_TYPE_PARAM_NAME, false); @@ -115,7 +115,15 @@ private ReadableStreamChannel serializeAsXml(RestRequest restRequest, Page contentsList = new ArrayList<>(); int keyCount = 0; + Set dirSet = new HashSet<>(); for (NamedBlobListEntry namedBlobRecord : namedBlobRecordPage.getEntries()) { + if (namedBlobRecord.isDirectory()) { + dirSet.add(namedBlobRecord.getBlobName()); + if (++keyCount == maxKeysValue) { + break; + } + continue; + } String blobName = namedBlobRecord.getBlobName(); long blobSize = namedBlobRecord.getBlobSize(); long modifiedTimeMs = namedBlobRecord.getModifiedTimeMs(); @@ -129,19 +137,25 @@ private ReadableStreamChannel serializeAsXml(RestRequest restRequest, Page commonPrefixes = new ArrayList<>(); + for (String dir : dirSet) { + commonPrefixes.add(new Prefix(dir)); + } + if (LIST_TYPE_VERSION_2.equals(getHeader(restRequest.getArgs(), LIST_TYPE, false))) { ListBucketResultV2 resultV2 = - new ListBucketResultV2(restRequest.getPath(), prefix, maxKeysValue, keyCount, delimiter, contentsList, - encodingType, continuationToken, namedBlobRecordPage.getNextPageToken(), - namedBlobRecordPage.getNextPageToken() != null); + new ListBucketResultV2(containerName, prefix, maxKeysValue, keyCount, delimiter, contentsList, encodingType, + continuationToken, namedBlobRecordPage.getNextPageToken(), namedBlobRecordPage.getNextPageToken() != null, + delimiter == null ? null : commonPrefixes); LOGGER.debug("Sending response for S3 ListObjects {}", resultV2); // Serialize xml xmlMapper.writeValue(outputStream, resultV2); } else { ListBucketResult result = - new ListBucketResult(restRequest.getPath(), prefix, maxKeysValue, keyCount, delimiter, contentsList, - encodingType, marker, namedBlobRecordPage.getNextPageToken(), - namedBlobRecordPage.getNextPageToken() != null); + new ListBucketResult(containerName, prefix, maxKeysValue, keyCount, delimiter, contentsList, encodingType, + marker, namedBlobRecordPage.getNextPageToken(), namedBlobRecordPage.getNextPageToken() != null, + delimiter == null ? null : commonPrefixes); LOGGER.debug("Sending response for S3 ListObjects {}", result); // Serialize xml xmlMapper.writeValue(outputStream, result); diff --git a/ambry-frontend/src/test/java/com/github/ambry/frontend/FrontendRestRequestServiceTest.java b/ambry-frontend/src/test/java/com/github/ambry/frontend/FrontendRestRequestServiceTest.java index 02c10a869b..90e11d0594 100644 --- a/ambry-frontend/src/test/java/com/github/ambry/frontend/FrontendRestRequestServiceTest.java +++ b/ambry-frontend/src/test/java/com/github/ambry/frontend/FrontendRestRequestServiceTest.java @@ -2735,17 +2735,17 @@ private void doListNamedBlobsTest(String prefix, String pageToken, Page> future = new CompletableFuture<>(); future.completeExceptionally(new RestServiceException("NamedBlobDb error", expectedErrorCode)); - when(namedBlobDb.list(any(), any(), any(), any(), any())).thenReturn(future); + when(namedBlobDb.list(any(), any(), any(), any(), any(), any())).thenReturn(future); } if (expectedErrorCode == null) { assertNotNull("pageToReturn should be set", pageToReturn); doOperation(restRequest, restResponseChannel); - verify(namedBlobDb).list(refAccount.getName(), refContainer.getName(), prefix, pageToken, null); + verify(namedBlobDb).list(refAccount.getName(), refContainer.getName(), prefix, pageToken, null, any()); Page response = Page.fromJson(new JSONObject(new String(restResponseChannel.getResponseBody())), NamedBlobListEntry::new); assertEquals("Unexpected blobs returned", diff --git a/ambry-frontend/src/test/java/com/github/ambry/frontend/TestNamedBlobDb.java b/ambry-frontend/src/test/java/com/github/ambry/frontend/TestNamedBlobDb.java index fbcc5dd7ae..f60976addf 100644 --- a/ambry-frontend/src/test/java/com/github/ambry/frontend/TestNamedBlobDb.java +++ b/ambry-frontend/src/test/java/com/github/ambry/frontend/TestNamedBlobDb.java @@ -83,7 +83,7 @@ public CompletableFuture get(String accountName, String contain @Override public CompletableFuture> list(String accountName, String containerName, String blobNamePrefix, - String pageToken, Integer maxKey) { + String pageToken, Integer maxKey, boolean groupDirectories) { if (exception != null) { return FutureUtils.completedExceptionally(exception); } diff --git a/ambry-named-mysql/src/integration-test/java/com/github/ambry/named/MySqlNamedBlobDbIntegrationTest.java b/ambry-named-mysql/src/integration-test/java/com/github/ambry/named/MySqlNamedBlobDbIntegrationTest.java index ed3df2ff92..762867dcbf 100644 --- a/ambry-named-mysql/src/integration-test/java/com/github/ambry/named/MySqlNamedBlobDbIntegrationTest.java +++ b/ambry-named-mysql/src/integration-test/java/com/github/ambry/named/MySqlNamedBlobDbIntegrationTest.java @@ -24,6 +24,7 @@ import com.github.ambry.config.ClusterMapConfig; import com.github.ambry.config.MySqlNamedBlobDbConfig; import com.github.ambry.config.VerifiableProperties; +import com.github.ambry.frontend.NamedBlobListEntry; import com.github.ambry.frontend.Page; import com.github.ambry.protocol.GetOption; import com.github.ambry.protocol.NamedBlobState; @@ -124,7 +125,8 @@ public void testPutGetListDeleteSequence() throws Exception { time.setCurrentMilliseconds(System.currentTimeMillis()); for (Account account : accountService.getAllAccounts()) { for (Container container : account.getAllContainers()) { - Page page = namedBlobDb.list(account.getName(), container.getName(), "name", null, null).get(); + Page page = + namedBlobDb.list(account.getName(), container.getName(), "name", null, null, false).get(); assertNull("No continuation token expected", page.getNextPageToken()); assertEquals("Unexpected number of blobs in container", blobsPerContainer, page.getEntries().size()); } @@ -135,17 +137,18 @@ public void testPutGetListDeleteSequence() throws Exception { for (Account account : accountService.getAllAccounts()) { for (Container container : account.getAllContainers()) { //page with no token - Page page = namedBlobDb.list(account.getName(), container.getName(), null, null, null).get(); + Page page = + namedBlobDb.list(account.getName(), container.getName(), null, null, null, false).get(); assertNull("No continuation token expected", page.getNextPageToken()); assertEquals("Unexpected number of blobs in container", blobsPerContainer, page.getEntries().size()); //page with token Page pageWithToken = - namedBlobDb.list(account.getName(), container.getName(), null, "name/4", null).get(); + namedBlobDb.list(account.getName(), container.getName(), null, "name/4", null, false).get(); assertEquals("Unexpected number of blobs in container", blobsPerContainer / 5, pageWithToken.getEntries().size()); //page with maxKeys Page pageWithMaxKey = - namedBlobDb.list(account.getName(), container.getName(), null, null, 1).get(); + namedBlobDb.list(account.getName(), container.getName(), null, null, 1, false).get(); assertEquals("Unexpected number of blobs in container", blobsPerContainer / 5, pageWithMaxKey.getEntries().size()); // Verify that blob size and modified ts is returned for all blobs @@ -300,7 +303,7 @@ public void testListNamedBlobsWithStaleRecords() throws Exception { namedBlobDb.put(v1_other, NamedBlobState.READY, true).get(); NamedBlobRecord v1_other_get = namedBlobDb.get(a1.getName(), a1c1.getName(), blobName + "-other").get(); assertEquals(v1_other, v1_other_get); - page = namedBlobDb.list(a1.getName(), a1c1.getName(), blobName, null, null).get(); + page = namedBlobDb.list(a1.getName(), a1c1.getName(), blobName, null, null, false).get(); assertEquals(2, page.getEntries().size()); assertEquals(v1_get, page.getEntries().get(0)); assertEquals(v1_other_get, page.getEntries().get(1)); @@ -310,7 +313,7 @@ public void testListNamedBlobsWithStaleRecords() throws Exception { v2 = new NamedBlobRecord(a1.getName(), a1c1.getName(), blobName, getBlobId(a1, a1c1), Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTimeInMillis() + TimeUnit.HOURS.toMillis(1)); namedBlobDb.put(v2, NamedBlobState.IN_PROGRESS, true).get(); - page = namedBlobDb.list(a1.getName(), a1c1.getName(), blobName, null, null).get(); + page = namedBlobDb.list(a1.getName(), a1c1.getName(), blobName, null, null, false).get(); assertEquals(2, page.getEntries().size()); assertEquals(v1_get, page.getEntries().get(0)); assertEquals(v1_other_get, page.getEntries().get(1)); @@ -323,7 +326,7 @@ public void testListNamedBlobsWithStaleRecords() throws Exception { Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTimeInMillis() + TimeUnit.HOURS.toMillis(1)); namedBlobDb.put(v2, NamedBlobState.READY, true).get(); namedBlobDb.put(v2_other, NamedBlobState.READY, true).get(); - page = namedBlobDb.list(a1.getName(), a1c1.getName(), blobName, null, null).get(); + page = namedBlobDb.list(a1.getName(), a1c1.getName(), blobName, null, null, false).get(); assertEquals(2, page.getEntries().size()); assertEquals(v2, page.getEntries().get(0)); assertEquals(v2_other, page.getEntries().get(1)); @@ -331,12 +334,12 @@ public void testListNamedBlobsWithStaleRecords() throws Exception { // delete blob and list should return empty namedBlobDb.delete(a1.getName(), a1c1.getName(), blobName).get(); - page = namedBlobDb.list(a1.getName(), a1c1.getName(), blobName, null, null).get(); + page = namedBlobDb.list(a1.getName(), a1c1.getName(), blobName, null, null, false).get(); assertEquals(1, page.getEntries().size()); assertEquals(v2_other, page.getEntries().get(0)); time.sleep(100); namedBlobDb.delete(a1.getName(), a1c1.getName(), blobName + "-other").get(); - page = namedBlobDb.list(a1.getName(), a1c1.getName(), blobName, null, null).get(); + page = namedBlobDb.list(a1.getName(), a1c1.getName(), blobName, null, null, false).get(); assertEquals(0, page.getEntries().size()); } @@ -384,7 +387,7 @@ public void testListNamedBlobs() throws Exception { ); // List named blob should only put out valid ones without empty entries. - Page page = namedBlobDb.list(account.getName(), container.getName(), blobNamePrefix, null, null).get(); + Page page = namedBlobDb.list(account.getName(), container.getName(), blobNamePrefix, null, null, false).get(); assertEquals("List named blob entries should match the valid records", validRecords, new HashSet<>(page.getEntries())); assertNull("Next page token should be null", page.getNextPageToken()); } @@ -722,6 +725,60 @@ public void testCleanupBlobGoodCase6() throws Exception { assertTrue("Good blob case 6 pull stale blob result should be empty!", staleNamedBlobs.isEmpty()); } + @Test + public void testListDirectories() throws ExecutionException, InterruptedException { + int numSubDirectories = 2; + int blobsPerSubDirectory = 5; + + Account account = accountService.getAllAccounts().stream().iterator().next(); + Container container = account.getAllContainers().stream().iterator().next(); + List> blobIdsPerSubDirectory = new ArrayList<>(); + + String dir = "dir"; + String subdir = "subdir"; + for (int i = 0; i < numSubDirectories; i++) { + List blobs = new ArrayList<>(); + for (int j = 0; j < blobsPerSubDirectory; j++) { + String blobId = getBlobId(account, container); + String blobName = dir + "/" + subdir + "-" + i + "/" + "file-" + j + ".txt"; + long expirationTime = Utils.Infinite_Time; + long blobSize = 20; + NamedBlobRecord record = + new NamedBlobRecord(account.getName(), container.getName(), blobName, blobId, expirationTime, 0, blobSize); + namedBlobDb.put(record).get(); + blobs.add(blobId); + } + blobIdsPerSubDirectory.add(blobs); + } + + // list directories container + Page page = namedBlobDb.list(account.getName(), container.getName(), null, null, null, true).get(); + assertEquals("Unexpected number of directories", 1, page.getEntries().size()); + assertEquals("Mismatch in directory name", "dir/", page.getEntries().get(0).getBlobName()); + + // list sub-directories under the directory + page = namedBlobDb.list(account.getName(), container.getName(), dir, null, null, true).get(); + assertEquals("Unexpected number of sub-directories", numSubDirectories, page.getEntries().size()); + Set subDirs = page.getEntries().stream().map(NamedBlobRecord::getBlobName).collect(Collectors.toSet()); + System.out.println("Sub directories = " + subDirs); + for (int i = 0; i < numSubDirectories; i++) { + assertTrue("Should contain directory" + subdir + "-" + i + "/" + ", Received = " + subDirs, + subDirs.contains(subdir + "-" + i + "/")); + } + + // list blobs under the sub-directory + for (int i = 0; i < numSubDirectories; i++) { + Page subPage = + namedBlobDb.list(account.getName(), container.getName(), dir + "/" + subdir + "-" + i, null, null, false) + .get(); + assertEquals("Unexpected number of blobs", blobsPerSubDirectory, subPage.getEntries().size()); + Set blobIds = subPage.getEntries().stream().map(NamedBlobRecord::getBlobId).collect(Collectors.toSet()); + for (int j = 0; j < blobsPerSubDirectory; j++) { + assertTrue("Mismatch in blob ID", blobIds.contains(blobIdsPerSubDirectory.get(i).get(j))); + } + } + } + /** * Get a sample blob ID. * @param account the account of the blob. diff --git a/ambry-named-mysql/src/main/java/com/github/ambry/named/MySqlNamedBlobDb.java b/ambry-named-mysql/src/main/java/com/github/ambry/named/MySqlNamedBlobDb.java index 450c605d02..e12bd714b5 100644 --- a/ambry-named-mysql/src/main/java/com/github/ambry/named/MySqlNamedBlobDb.java +++ b/ambry-named-mysql/src/main/java/com/github/ambry/named/MySqlNamedBlobDb.java @@ -392,12 +392,12 @@ public CompletableFuture get(String accountName, String contain @Override public CompletableFuture> list(String accountName, String containerName, String blobNamePrefix, - String pageToken, Integer maxKeys) { + String pageToken, Integer maxKeys, boolean groupDirectories) { return executeTransactionAsync(accountName, containerName, true, (accountId, containerId, connection) -> { long startTime = this.time.milliseconds(); Page recordPage = run_list_v2(accountName, containerName, blobNamePrefix, pageToken, accountId, containerId, connection, - maxKeys); + maxKeys, groupDirectories); metricsRecoder.namedBlobListTimeInMs.update(this.time.milliseconds() - startTime); return recordPage; }, null); @@ -674,10 +674,13 @@ private NamedBlobRecord run_get_v2(String accountName, String containerName, Str } private Page run_list_v2(String accountName, String containerName, String blobNamePrefix, - String pageToken, short accountId, short containerId, Connection connection, Integer maxKeys) throws Exception { + String pageToken, short accountId, short containerId, Connection connection, Integer maxKeys, + boolean groupDirectories) throws Exception { String query = ""; String queryStatement = blobNamePrefix == null ? LIST_ALL_QUERY_V2 : LIST_NAMED_BLOBS_SQL; int maxKeysValue = maxKeys == null ? config.listMaxResults : maxKeys; + // Set to store unique subfolder names + Set directories = new HashSet<>(); try (PreparedStatement statement = connection.prepareStatement(queryStatement)) { if (blobNamePrefix == null) { // list-all no prefix @@ -706,10 +709,27 @@ private Page run_list_v2(String accountName, String containerNa int resultIndex = 0; while (resultSet.next()) { String blobName = resultSet.getString(1); - if (resultIndex++ == maxKeysValue) { + if (resultIndex == maxKeysValue) { nextContinuationToken = blobName; break; } + + if (groupDirectories) { + // Extract the portion after the prefix and before the next '/' + String remainingPath = blobName.substring(blobNamePrefix == null ? 0 : blobNamePrefix.length()); + remainingPath = remainingPath.startsWith("/") ? remainingPath.substring(1) : remainingPath; + int delimiterIndex = remainingPath.indexOf("/"); + if (delimiterIndex != -1) { + boolean validEntry = directories.add(remainingPath.substring(0, delimiterIndex) + "/"); + if (validEntry) { + resultIndex++; + } + // Since this file is part of a logical directory, continue to the next result + continue; + } + } + + resultIndex++; String blobId = Base64.encodeBase64URLSafeString(resultSet.getBytes(2)); long version = resultSet.getLong(3); Timestamp deletionTime = resultSet.getTimestamp(4); @@ -718,8 +738,17 @@ private Page run_list_v2(String accountName, String containerNa entries.add( new NamedBlobRecord(accountName, containerName, blobName, blobId, timestampToMs(deletionTime), version, - blobSize, timestampToMs(modifiedTime))); + blobSize, timestampToMs(modifiedTime), false)); } + + if (groupDirectories) { + // Add the directories to the result + entries.addAll(directories.stream() + .map(directory -> new NamedBlobRecord(accountName, containerName, directory, null, Utils.Infinite_Time, 0, + 0, 0, true)) + .collect(Collectors.toList())); + } + return new Page<>(entries, nextContinuationToken); } } catch (SQLException e) { diff --git a/ambry-tools/src/main/java/com/github/ambry/tools/perf/NamedBlobMysqlDatabasePerf.java b/ambry-tools/src/main/java/com/github/ambry/tools/perf/NamedBlobMysqlDatabasePerf.java index 274584d5e6..91b7825c0b 100644 --- a/ambry-tools/src/main/java/com/github/ambry/tools/perf/NamedBlobMysqlDatabasePerf.java +++ b/ambry-tools/src/main/java/com/github/ambry/tools/perf/NamedBlobMysqlDatabasePerf.java @@ -529,7 +529,7 @@ public void run() { int numberOfList = 0; for (NamedBlobRecord record : allRecords) { if (!record.getAccountName().equals(String.format(ACCOUNT_NAME_FORMAT, HUGE_LIST_ACCOUNT_ID))) { - namedBlobDb.list(record.getAccountName(), record.getContainerName(), "A", null, null).get(); + namedBlobDb.list(record.getAccountName(), record.getContainerName(), "A", null, null, false).get(); numberOfList++; if (numberOfList == 100) { break; @@ -542,8 +542,9 @@ public void run() { String containerName = String.format(CONTAINER_NAME_FORMAT, HUGE_LIST_CONTAINER_ID); String token = null; for (int i = 0; i < 100; i++) { - token = - namedBlobDb.list(accountName, containerName, HUGE_LIST_COMMON_PREFIX, token, null).get().getNextPageToken(); + token = namedBlobDb.list(accountName, containerName, HUGE_LIST_COMMON_PREFIX, token, null, false) + .get() + .getNextPageToken(); } System.out.println("PerformanceTestWorker " + id + " finishes listing for huge records"); }