Allow object cache with NVM cache

Summary:
Open issue:

If a NvmCache::createItem without inserting into DRAM cache, we need to find a way to clear up the memory.
This happens in peek, and if the regular find failed to insert (due to replaced item refcount overflow), and getSampleItem (createAsIoBuf but not for item destructor)

Reviewed By: lacora2017

Differential Revision: D58921494

fbshipit-source-id: 61c9f4db55bc03c73c838fae0dd1a16fbd9f2546

Author: Hao Wu

cachelib/navy/block_cache/BlockCache.cpp

@@ -42,7 +42,7 @@ BlockCache::Config& BlockCache::Config::validate() {
   if (!device || !evictionPolicy) {
     throw std::invalid_argument("missing required param");
   }
-  if (regionSize > 256u << 20) {
+  if (regionSize > 1024u << 20) {
     // We allocate region in memory to reclaim. Too large region will cause
     // problems: at least, long allocation times.
     throw std::invalid_argument("region is too large");

cachelib/navy/common/Buffer.h

@@ -261,6 +261,10 @@ inline BufferView makeView(folly::StringPiece str) {
   return {str.size(), reinterpret_cast<const uint8_t*>(str.data())};
 }
 
+inline Buffer makeBuffer(folly::StringPiece str) {
+  return Buffer{makeView(str)};
+}
+
 inline folly::StringPiece toStringPiece(BufferView view) {
   return {reinterpret_cast<const char*>(view.data()), view.size()};
 }

cachelib/object_cache/ObjectCache.h

@@ -159,6 +159,10 @@ class ObjectCache : public ObjectCacheBase<AllocatorT> {
   using Restorer = Restorer<ObjectCache<AllocatorT>>;
   using EvictionIterator = typename AllocatorT::EvictionIterator;
   using AccessIterator = typename AllocatorT::AccessIterator;
+  using NvmCache = typename AllocatorT::NvmCacheT;
+  using NvmCacheConfig = typename AllocatorT::NvmCacheT::Config;
+  using WriteHandle = typename AllocatorT::WriteHandle;
+  using CacheItem = typename AllocatorT::Item;
 
   enum class AllocStatus { kSuccess, kAllocError, kKeyAlreadyExists };
 
@@ -430,7 +434,11 @@ class ObjectCache : public ObjectCacheBase<AllocatorT> {
         util::stopPeriodicWorker(kSizeControllerName, sizeController_, timeout);
     success &= util::stopPeriodicWorker(kSizeDistTrackerName, sizeDistTracker_,
                                         timeout);
-    success &= this->l1Cache_->stopWorkers(timeout);
+    // Nullproof. This function can be called in the destructor before init()
+    // completes successfully.
+    if (this->l1Cache_) {
+      success &= this->l1Cache_->stopWorkers(timeout);
+    }
     return success;
   }
 
@@ -615,6 +623,10 @@ void ObjectCache<AllocatorT>::init() {
     l1Config.setDelayCacheWorkersStart();
   }
 
+  if (config_.nvmConfig.has_value()) {
+    l1Config.nvmConfig.assign(std::move(config_.nvmConfig.value()));
+  }
+
   this->l1Cache_ = std::make_unique<AllocatorT>(l1Config);
   // add a pool per shard
   for (size_t i = 0; i < config_.l1NumShards; i++) {
@@ -880,9 +892,11 @@ template <typename AllocatorT>
 ObjectCache<AllocatorT>::~ObjectCache() {
   stopAllWorkers();
 
-  for (auto itr = this->l1Cache_->begin(); itr != this->l1Cache_->end();
-       ++itr) {
-    this->l1Cache_->remove(itr.asHandle());
+  if (this->l1Cache_) {
+    for (auto itr = this->l1Cache_->begin(); itr != this->l1Cache_->end();
+         ++itr) {
+      this->l1Cache_->remove(itr.asHandle());
+    }
   }
 }
 

cachelib/object_cache/ObjectCacheConfig.h

@@ -20,6 +20,7 @@
 #include <string>
 
 #include "cachelib/allocator/KAllocation.h"
+#include "cachelib/allocator/nvmcache/NvmItem.h"
 #include "cachelib/common/EventInterface.h"
 #include "cachelib/common/Throttler.h"
 
@@ -36,6 +37,9 @@ struct ObjectCacheConfig {
   using SerializeCb = typename ObjectCache::SerializeCb;
   using DeserializeCb = typename ObjectCache::DeserializeCb;
   using EvictionPolicyConfig = typename ObjectCache::EvictionPolicyConfig;
+  using NvmCacheConfig = typename ObjectCache::NvmCacheConfig;
+  using ToBlobCb = std::function<std::unique_ptr<folly::IOBuf>(uintptr_t)>;
+  using ToPtrCb = std::function<uintptr_t(folly::StringPiece)>;
 
   // Set cache name as a string
   ObjectCacheConfig& setCacheName(const std::string& _cacheName);
@@ -147,6 +151,12 @@ struct ObjectCacheConfig {
   // ObjectCache::startCacheWorkers()
   ObjectCacheConfig& setDelayCacheWorkersStart();
 
+  /**
+   * Enable NVM cache.
+   */
+  ObjectCacheConfig& enableNvm(NvmCacheConfig config);
+  ObjectCacheConfig& overrideNvmCbs(ToBlobCb blobCb, ToPtrCb ptrCb);
+
   // With size controller disabled, above this many entries, L1 will start
   // evicting.
   // With size controller enabled, this is only a hint used for initialization.
@@ -237,6 +247,8 @@ struct ObjectCacheConfig {
   // ObjectCache::startCacheWorkers()
   bool delayCacheWorkersStart{false};
 
+  std::optional<typename ObjectCache::NvmCacheConfig> nvmConfig{};
+
   const ObjectCacheConfig& validate() const;
 };
 
@@ -353,6 +365,7 @@ ObjectCacheConfig<T>& ObjectCacheConfig<T>::enablePersistence(
         "Serialize and deserialize callback must be set to enable cache "
         "persistence");
   }
+  persistenceEnabled = true;
   persistThreadCount = threadCount;
   persistBaseFilePath = basefilePath;
   serializeCb = std::move(serializeCallback);
@@ -379,6 +392,7 @@ ObjectCacheConfig<T>& ObjectCacheConfig<T>::enablePersistenceWithEvictionOrder(
         "Serialize and deserialize callback must be set to enable cache "
         "persistence");
   }
+  persistenceEnabled = true;
   persistThreadCount = 1;
   persistBaseFilePath = basefilePath;
   serializeCb = std::move(serializeCallback);
@@ -413,6 +427,56 @@ ObjectCacheConfig<T>& ObjectCacheConfig<T>::setDelayCacheWorkersStart() {
   return *this;
 }
 
+template <typename T>
+ObjectCacheConfig<T>& ObjectCacheConfig<T>::enableNvm(NvmCacheConfig config) {
+  nvmConfig = std::move(config);
+  return *this;
+}
+
+template <typename T>
+ObjectCacheConfig<T>& ObjectCacheConfig<T>::overrideNvmCbs(ToBlobCb blobCb,
+                                                           ToPtrCb ptrCb) {
+  if (!nvmConfig || nvmConfig->makeBlobCb || nvmConfig->makeObjCb) {
+    throw std::invalid_argument(
+        "Do not set makeBlobCb or makeObjCb in nvmConfig before calling "
+        "overriceNvmCbs.");
+  }
+
+  nvmConfig->makeBlobCb =
+      [blobCb = std::move(blobCb)](
+          const typename T::CacheItem& item,
+          folly::Range<typename T::NvmCache::ChainedItemIter>) {
+        uintptr_t ptr =
+            item.template getMemoryAs<typename T::Item>()->objectPtr;
+        auto blob = blobCb(ptr);
+        std::vector<BufferedBlob> blobs;
+        if (blob == nullptr) {
+          return blobs;
+        }
+        blobs.emplace_back(BufferedBlob{static_cast<uint32_t>(item.getSize()),
+                                        std::move(blob)});
+        return blobs;
+      };
+
+  nvmConfig->makeObjCb =
+      [ptrCb = std::move(ptrCb)](
+          const NvmItem& nvmItem, typename T::CacheItem& it,
+          folly::Range<typename T::NvmCache::WritableChainedItemIter>) {
+        // Create ptr. Do not consider chained item.
+        auto pBlob = nvmItem.getBlob(0);
+        uintptr_t ptr = ptrCb(pBlob.data);
+        // TODO: Is there a better way to do this?
+        if (ptr == reinterpret_cast<uintptr_t>(nullptr)) {
+          return false;
+        }
+        *it.template getMemoryAs<typename T::Item>() =
+            typename T::Item{ptr, pBlob.data.size()};
+
+        return true;
+      };
+  return *this;
+}
+
 template <typename T>
 const ObjectCacheConfig<T>& ObjectCacheConfig<T>::validate() const {
   // checking missing params
@@ -450,6 +514,7 @@ const ObjectCacheConfig<T>& ObjectCacheConfig<T>::validate() const {
         "Object size tracking has to be enabled to track object size "
         "distribution");
   }
+
   return *this;
 }