[Enhancement] add memory statistic for cloud native persistent index'…

…s sstable (backport #50595) (#50678)

Co-authored-by: Yixin Luo <[email protected]>

be/src/storage/lake/lake_persistent_index.cpp

@@ -739,6 +739,11 @@ size_t LakePersistentIndex::memory_usage() const {
     if (_immutable_memtable != nullptr) {
         mem_usage += _immutable_memtable->memory_usage();
     }
+    for (const auto& sst_ptr : _sstables) {
+        if (sst_ptr != nullptr) {
+            mem_usage += sst_ptr->memory_usage();
+        }
+    }
     return mem_usage;
 }
 

be/src/storage/lake/persistent_index_sstable.cpp

@@ -105,4 +105,8 @@ Status PersistentIndexSstable::multi_get(const Slice* keys, const KeyIndexSet& k
     return Status::OK();
 }
 
+size_t PersistentIndexSstable::memory_usage() const {
+    return (_sst != nullptr) ? _sst->memory_usage() : 0;
+}
+
 } // namespace starrocks::lake

be/src/storage/lake/persistent_index_sstable.h

@@ -58,6 +58,8 @@ class PersistentIndexSstable {
 
     const PersistentIndexSstablePB& sstable_pb() const { return _sstable_pb; }
 
+    size_t memory_usage() const;
+
 private:
     std::unique_ptr<sstable::Table> _sst{nullptr};
     std::unique_ptr<sstable::FilterPolicy> _filter_policy{nullptr};

be/src/storage/sstable/table.cpp

@@ -27,18 +27,20 @@ struct Table::Rep {
     ~Rep() {
         delete filter;
         delete[] filter_data;
+        filter_data_size = 0;
         delete index_block;
     }
 
     Options options;
     Status status;
-    RandomAccessFile* file;
-    uint64_t cache_id;
-    FilterBlockReader* filter;
-    const char* filter_data;
+    RandomAccessFile* file = nullptr;
+    uint64_t cache_id = 0;
+    FilterBlockReader* filter = nullptr;
+    const char* filter_data = nullptr;
+    size_t filter_data_size = 0;
 
     BlockHandle metaindex_handle; // Handle to metaindex_block: saved from footer
-    Block* index_block;
+    Block* index_block = nullptr;
 };
 
 Status Table::Open(const Options& options, RandomAccessFile* file, uint64_t size, Table** table) {
@@ -135,7 +137,8 @@ void Table::ReadFilter(const Slice& filter_handle_value) {
         return;
     }
     if (block.heap_allocated) {
-        rep_->filter_data = block.data.get_data(); // Will need to delete later
+        rep_->filter_data = block.data.get_data();      // Will need to delete later
+        rep_->filter_data_size = block.data.get_size(); // mem tracker will track this piece of memory.
     }
     rep_->filter = new FilterBlockReader(rep_->options.filter_policy, block.data);
 }
@@ -302,6 +305,12 @@ Status Table::MultiGet(const ReadOptions& options, const Slice* keys, ForwardIt
     return s;
 }
 
+size_t Table::memory_usage() const {
+    const size_t index_block_sz = (rep_->index_block != nullptr) ? rep_->index_block->size() : 0;
+    const size_t filter_data_sz = rep_->filter_data_size;
+    return index_block_sz + filter_data_sz;
+}
+
 // If new container wants to be supported in MultiGet, the initialization can be added here.
 template Status Table::MultiGet<std::set<size_t>::iterator>(const ReadOptions& options, const Slice* keys,
                                                             std::set<size_t>::iterator begin,

be/src/storage/sstable/table.h

@@ -51,6 +51,8 @@ class Table {
     Status MultiGet(const ReadOptions&, const Slice* keys, ForwardIt begin, ForwardIt end,
                     std::vector<std::string>* values);
 
+    size_t memory_usage() const;
+
 private:
     struct Rep;
 

be/test/storage/lake/lake_persistent_index_test.cpp

@@ -91,8 +91,7 @@ TEST_F(LakePersistentIndexTest, test_basic_api) {
     auto index = std::make_unique<LakePersistentIndex>(_tablet_mgr.get(), tablet_id);
     ASSERT_OK(index->init(_tablet_metadata->sstable_meta()));
     ASSERT_OK(index->insert(N, key_slices.data(), values.data(), 0));
-    // insert duplicate should return error
-    // ASSERT_FALSE(index->insert(N, key_slices.data(), values.data(), 0).ok());
+    ASSERT_TRUE(index->memory_usage() > 0);
 
     // test get
     vector<IndexValue> get_values(keys.size());
@@ -233,6 +232,7 @@ TEST_F(LakePersistentIndexTest, test_major_compaction) {
         // generate sst files.
         index->minor_compact();
     }
+    ASSERT_TRUE(index->memory_usage() > 0);
 
     Tablet tablet(_tablet_mgr.get(), tablet_id);
     auto tablet_metadata_ptr = std::make_shared<TabletMetadata>();

be/test/storage/lake/persistent_index_sstable_test.cpp

@@ -229,6 +229,8 @@ TEST_F(PersistentIndexSstableTest, test_persistent_index_sstable) {
     sstable_pb.set_filename(filename);
     sstable_pb.set_filesize(filesize);
     ASSERT_OK(sst->init(std::move(read_file), sstable_pb, cache_ptr.get()));
+    // check memory usage
+    ASSERT_TRUE(sst->memory_usage() > 0);
 
     {
         // 3. multi get with version (all keys included)