allow removeCb for item destruction

Summary:
Allow to set either removeCb or item destructor in object cache.

For object cache in hybrid cache, we'll utilize removeCb.
Actually, all object cache should just use removeCb, because the tracking and freeing up memory should happen when the item leaves DRAM instead of the flash.

Reviewed By: therealgymmy, lacora2017

Differential Revision: D58987552

fbshipit-source-id: 00a60e2bc817f1a462bde8c565626843d98ecaea

Author: Hao Wu

cachelib/object_cache/ObjectCache.h

@@ -146,6 +146,7 @@ class ObjectCache : public ObjectCacheBase<AllocatorT> {
 
  public:
   using ItemDestructor = std::function<void(ObjectCacheDestructorData)>;
+  using RemoveCb = std::function<void(ObjectCacheDestructorData)>;
   using Key = KAllocation::Key;
   using Config = ObjectCacheConfig<ObjectCache<AllocatorT>>;
   using EvictionPolicyConfig = typename AllocatorT::MMType::Config;
@@ -550,34 +551,66 @@ void ObjectCache<AllocatorT>::init() {
       .setDefaultAllocSizes({l1AllocSize})
       .enableItemReaperInBackground(config_.reaperInterval)
       .setEventTracker(std::move(config_.eventTracker))
-      .setEvictionSearchLimit(config_.evictionSearchLimit)
-      .setItemDestructor([this](typename AllocatorT::DestructorData data) {
-        ObjectCacheDestructorContext ctx;
-        if (data.context == DestructorContext::kEvictedFromRAM) {
-          evictions_.inc();
-          ctx = ObjectCacheDestructorContext::kEvicted;
-        } else if (data.context == DestructorContext::kRemovedFromRAM) {
-          ctx = ObjectCacheDestructorContext::kRemoved;
-        } else { // should not enter here
-          ctx = ObjectCacheDestructorContext::kUnknown;
-        }
+      .setEvictionSearchLimit(config_.evictionSearchLimit);
+  if (config_.itemDestructor) {
+    l1Config.setItemDestructor(
+        [this](typename AllocatorT::DestructorData data) {
+          ObjectCacheDestructorContext ctx;
+          if (data.context == DestructorContext::kEvictedFromRAM) {
+            evictions_.inc();
+            ctx = ObjectCacheDestructorContext::kEvicted;
+          } else if (data.context == DestructorContext::kRemovedFromRAM) {
+            ctx = ObjectCacheDestructorContext::kRemoved;
+          } else { // should not enter here
+            ctx = ObjectCacheDestructorContext::kUnknown;
+          }
 
-        auto& item = data.item;
+          auto& item = data.item;
+
+          auto itemPtr = reinterpret_cast<ObjectCacheItem*>(item.getMemory());
+
+          SCOPE_EXIT {
+            if (config_.objectSizeTrackingEnabled) {
+              // update total object size
+              totalObjectSizeBytes_.fetch_sub(itemPtr->objectSize,
+                                              std::memory_order_relaxed);
+            }
+            // execute user defined item destructor
+            config_.itemDestructor(ObjectCacheDestructorData(
+                ctx, itemPtr->objectPtr, item.getKey(), item.getExpiryTime(),
+                item.getCreationTime(), item.getLastAccessTime()));
+          };
+        });
+  } else {
+    l1Config.setRemoveCallback([this](typename AllocatorT::RemoveCbData data) {
+      ObjectCacheDestructorContext ctx;
+      if (data.context == RemoveContext::kEviction) {
+        evictions_.inc();
+        ctx = ObjectCacheDestructorContext::kEvicted;
+      } else if (data.context == RemoveContext::kNormal) {
+        ctx = ObjectCacheDestructorContext::kRemoved;
+      } else { // should not enter here
+        ctx = ObjectCacheDestructorContext::kUnknown;
+      }
 
-        auto itemPtr = reinterpret_cast<ObjectCacheItem*>(item.getMemory());
+      auto& item = data.item;
+
+      auto itemPtr = reinterpret_cast<ObjectCacheItem*>(item.getMemory());
+
+      SCOPE_EXIT {
+        if (config_.objectSizeTrackingEnabled) {
+          // update total object size
+          totalObjectSizeBytes_.fetch_sub(itemPtr->objectSize,
+                                          std::memory_order_relaxed);
+        }
+        // execute user defined item destructor
+        config_.removeCb(ObjectCacheDestructorData(
+            ctx, itemPtr->objectPtr, item.getKey(), item.getExpiryTime(),
+            item.getCreationTime(), item.getLastAccessTime()));
+      };
+    });
+  }
 
-        SCOPE_EXIT {
-          if (config_.objectSizeTrackingEnabled) {
-            // update total object size
-            totalObjectSizeBytes_.fetch_sub(itemPtr->objectSize,
-                                            std::memory_order_relaxed);
-          }
-          // execute user defined item destructor
-          config_.itemDestructor(ObjectCacheDestructorData(
-              ctx, itemPtr->objectPtr, item.getKey(), item.getExpiryTime(),
-              item.getCreationTime(), item.getLastAccessTime()));
-        };
-      });
   if (config_.delayCacheWorkersStart) {
     l1Config.setDelayCacheWorkersStart();
   }

cachelib/object_cache/ObjectCacheConfig.h

@@ -32,6 +32,7 @@ struct ObjectCacheConfig {
   using Key = KAllocation::Key;
   using EventTrackerSharedPtr = std::shared_ptr<EventInterface<Key>>;
   using ItemDestructor = typename ObjectCache::ItemDestructor;
+  using RemoveCb = typename ObjectCache::RemoveCb;
   using SerializeCb = typename ObjectCache::SerializeCb;
   using DeserializeCb = typename ObjectCache::DeserializeCb;
   using EvictionPolicyConfig = typename ObjectCache::EvictionPolicyConfig;
@@ -106,6 +107,8 @@ struct ObjectCacheConfig {
   // });
   ObjectCacheConfig& setItemDestructor(ItemDestructor destructor);
 
+  ObjectCacheConfig& setRemoveCb(RemoveCb cb);
+
   // Run in a multi-thread mode, eviction order is not guaranteed to persist.
   // @param threadCount          number of threads to work on persistence
   //                             concurrently
@@ -197,6 +200,8 @@ struct ObjectCacheConfig {
   // or explicitly from cache
   ItemDestructor itemDestructor{};
 
+  RemoveCb removeCb{};
+
   // time to sleep between each reaping period.
   std::chrono::milliseconds reaperInterval{5000};
 
@@ -317,6 +322,12 @@ ObjectCacheConfig<T>& ObjectCacheConfig<T>::setItemDestructor(
   return *this;
 }
 
+template <typename T>
+ObjectCacheConfig<T>& ObjectCacheConfig<T>::setRemoveCb(RemoveCb cb) {
+  removeCb = std::move(cb);
+  return *this;
+}
+
 template <typename T>
 ObjectCacheConfig<T>& ObjectCacheConfig<T>::enablePersistence(
     uint32_t threadCount,
@@ -413,9 +424,10 @@ const ObjectCacheConfig<T>& ObjectCacheConfig<T>::validate() const {
     throw std::invalid_argument("l1EntriesLimit is not provided");
   }
 
-  if (!itemDestructor) {
+  if ((!itemDestructor && !removeCb) || (itemDestructor && removeCb)) {
     throw std::invalid_argument(
-        "ItemDestructor is mandatory, but not provided");
+        "Only one of ItemDestructor or RemoveCb can be set. Not both, nor "
+        "neither.");
   }
 
   if (objectSizeTrackingEnabled) {