Make kShards configurable in CacheAllocator

Summary: During T2 testing, to avoid lock contention in CacheAllocator, we had to increase this from 8K to 64-128K. Make it configurable and default to 8K

Reviewed By: therealgymmy

Differential Revision: D69193533

fbshipit-source-id: bfc0a292a9d00cf50bfbf7f5f8269cf08cd47f51

Author: Stuart Clark

cachelib/allocator/CacheAllocator.h

@@ -2185,8 +2185,8 @@ class CacheAllocator : public CacheBase {
                          std::unique_ptr<MoveCtx>,
                          folly::HeterogeneousAccessHash<folly::StringPiece>>;
 
-  static size_t getShardForKey(folly::StringPiece key) {
-    return folly::Hash()(key) % kShards;
+  size_t getShardForKey(folly::StringPiece key) {
+    return folly::Hash()(key) % shards_;
   }
 
   MoveMap& getMoveMapForShard(size_t shard) {
@@ -2300,7 +2300,7 @@ class CacheAllocator : public CacheBase {
   // poolResizer_, poolOptimizer_, memMonitor_, reaper_
   mutable std::mutex workersMutex_;
 
-  static constexpr size_t kShards = 8192; // TODO: need to define right value
+  const size_t shards_;
 
   struct MovesMapShard {
     alignas(folly::hardware_destructive_interference_size) MoveMap movesMap_;
@@ -2455,8 +2455,9 @@ CacheAllocator<CacheTrait>::CacheAllocator(
                               config.chainedItemAccessConfig)),
       chainedItemLocks_(config_.chainedItemsLockPower,
                         std::make_shared<MurmurHash2>()),
-      movesMap_(kShards),
-      moveLock_(kShards),
+      shards_{config_.numShards},
+      movesMap_(shards_),
+      moveLock_(shards_),
       cacheCreationTime_{
           type != InitMemType::kMemAttach
               ? util::getCurrentTimeSec()

cachelib/allocator/CacheAllocatorConfig.h

@@ -345,6 +345,9 @@ class CacheAllocatorConfig {
   // CacheAllocator::startCacheWorkers()
   CacheAllocatorConfig& setDelayCacheWorkersStart();
 
+  // Set numShards to use for CacheAllocator locks
+  CacheAllocatorConfig& setNumShards(size_t shards);
+
   // skip promote children items in chained when parent fail to promote
   bool isSkipPromoteChildrenWhenParentFailed() const noexcept {
     return skipPromoteChildrenWhenParentFailed;
@@ -655,6 +658,8 @@ class CacheAllocatorConfig {
   // CacheAllocator::startCacheWorkers()
   bool delayCacheWorkersStart{false};
 
+  size_t numShards{8192};
+
   friend CacheT;
 
  private:
@@ -1123,6 +1128,12 @@ CacheAllocatorConfig<T>& CacheAllocatorConfig<T>::setDelayCacheWorkersStart() {
   return *this;
 }
 
+template <typename T>
+CacheAllocatorConfig<T>& CacheAllocatorConfig<T>::setNumShards(size_t shards) {
+  numShards = shards;
+  return *this;
+}
+
 template <typename T>
 const CacheAllocatorConfig<T>& CacheAllocatorConfig<T>::validate() const {
   // we can track tail hits only if MMType is MM2Q
@@ -1274,6 +1285,7 @@ std::map<std::string, std::string> CacheAllocatorConfig<T>::serialize() const {
   configMap["nvmAdmissionMinTTL"] = std::to_string(nvmAdmissionMinTTL);
   configMap["delayCacheWorkersStart"] =
       delayCacheWorkersStart ? "true" : "false";
+  configMap["numShards"] = std::to_string(numShards);
   mergeWithPrefix(configMap, throttleConfig.serialize(), "throttleConfig");
   mergeWithPrefix(configMap,
                   chainedItemAccessConfig.serialize(),