Skip to content

Commit

Permalink
update trie
Browse files Browse the repository at this point in the history
  • Loading branch information
thu-david committed Sep 3, 2024
1 parent 2d0646d commit 3d1ae47
Show file tree
Hide file tree
Showing 9 changed files with 448 additions and 21 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
Expand Down Expand Up @@ -400,7 +401,7 @@ Optional<DataFileChannel> getDataFileChannel(
* @param ufsUrl the prefix of UfsUrl
* @return a list of PageInfos with the same prefix
*/
default List<PageInfo> getPageInfoByPrefix(UfsUrl ufsUrl) {
default Set<PageInfo> getPageInfoByPrefix(UfsUrl ufsUrl) {
throw new UnsupportedOperationException("Unsupported method: getPageInfoByPrefix");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@
import alluxio.client.file.cache.store.PageStoreDir;
import alluxio.client.quota.CacheScope;
import alluxio.collections.IndexDefinition;
import alluxio.collections.IndexedSet;
import alluxio.collections.IndexedSetTrie;
import alluxio.exception.FileDoesNotExistException;
import alluxio.exception.PageNotFoundException;
import alluxio.metrics.MetricKey;
import alluxio.metrics.MetricsSystem;
import alluxio.uri.UfsUrl;

import com.codahale.metrics.Counter;
import com.google.common.base.Preconditions;
Expand All @@ -49,7 +50,8 @@
public class DefaultPageMetaStore implements PageMetaStore {
private static final Logger LOG = LoggerFactory.getLogger(DefaultPageMetaStore.class);
/** A map from PageId to page info. */
private final IndexedSet<PageInfo> mPages = new IndexedSet<>(INDEX_PAGE_ID, INDEX_FILE_ID);
private final IndexedSetTrie<PageInfo> mPages =
new IndexedSetTrie<>(INDEX_PAGE_ID, INDEX_FILE_ID);
private final ImmutableList<PageStoreDir> mDirs;
/** The number of logical bytes used. */
private final AtomicLong mBytes = new AtomicLong(0);
Expand Down Expand Up @@ -78,7 +80,6 @@ public DefaultPageMetaStore(List<PageStoreDir> dirs) {
public DefaultPageMetaStore(List<PageStoreDir> dirs, Allocator allocator) {
mDirs = ImmutableList.copyOf(requireNonNull(dirs));
mAllcator = requireNonNull(allocator);
//metrics for the num of pages stored in the cache
MetricsSystem.registerGaugeIfAbsent(MetricKey.CLIENT_CACHE_PAGES.getName(),
mPages::size);
}
Expand Down Expand Up @@ -234,6 +235,12 @@ public Set<PageInfo> getAllPagesByFileId(String fileId) {
return pages;
}

@Override
public Set<PageInfo> getPageInfoByPrefix(UfsUrl ufsUrl) {
String fileIdPrefix = ufsUrl.toString();
return mPages.getByPrefix(INDEX_FILE_ID, fileIdPrefix);
}

@Override
public Optional<CacheUsage> getUsage() {
return Optional.of(new Usage());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import alluxio.metrics.MultiDimensionalMetricsSystem;
import alluxio.network.protocol.databuffer.DataFileChannel;
import alluxio.resource.LockResource;
import alluxio.uri.UfsUrl;
import alluxio.util.ThreadFactoryUtils;

import com.codahale.metrics.Counter;
Expand All @@ -48,6 +49,7 @@
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
Expand Down Expand Up @@ -1078,6 +1080,11 @@ public Optional<CacheUsage> partitionedBy(PartitionDescriptor<?> partitionDescri
}
}

@Override
public Set<PageInfo> getPageInfoByPrefix(UfsUrl ufsUrl) {
return mPageMetaStore.getPageInfoByPrefix(ufsUrl);
}

private static final class Metrics {
// Note that only counter/guage can be added here.
// Both meter and timer need to be used inline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,17 @@
import alluxio.metrics.MetricKey;
import alluxio.metrics.MetricsSystem;
import alluxio.network.protocol.databuffer.DataFileChannel;
import alluxio.uri.UfsUrl;

import com.codahale.metrics.Counter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;

/**
Expand Down Expand Up @@ -224,6 +227,17 @@ public void deleteFile(String fileId) {
}
}

@Override
public Set<PageInfo> getPageInfoByPrefix(UfsUrl ufsUrl) {
Set<PageInfo> set = Collections.emptySet();
try {
set = mCacheManager.getPageInfoByPrefix(ufsUrl);
} catch (Exception e) {
LOG.error("Failed to getPageInfoByPrefix for {}", ufsUrl, e);
}
return set;
}

@Override
public void deleteTempFile(String fileId) {
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import alluxio.client.quota.CacheScope;
import alluxio.exception.FileDoesNotExistException;
import alluxio.exception.PageNotFoundException;
import alluxio.uri.UfsUrl;

import java.io.IOException;
import java.util.List;
Expand Down Expand Up @@ -157,4 +158,13 @@ default PageInfo evict(PageStoreDir pageStoreDir) {
* @return a page to evict
*/
PageInfo evict(CacheScope cacheScope, PageStoreDir pageStoreDir);

/**
* Get pageInfo of a specific prefix.
* @param ufsUrl the prefix of UfsUrl
* @return a list of PageInfos with the same prefix
*/
default Set<PageInfo> getPageInfoByPrefix(UfsUrl ufsUrl) {
throw new UnsupportedOperationException("Unsupported method: getPageInfoByPrefix");
}
}
Loading

0 comments on commit 3d1ae47

Please sign in to comment.