From 1b6e59bf1b7728b74a20d9cfc9eb769af0035cdb Mon Sep 17 00:00:00 2001 From: qkrorlqr Date: Fri, 18 Oct 2024 15:40:26 +0000 Subject: [PATCH 1/3] issue-2137: calculating GarbageBlocksCount stats in CompactionMap --- .../libs/diagnostics/events/profile_events.ev | 1 + .../storage/tablet/model/mixed_blocks.cpp | 24 +++++++++++++ .../libs/storage/tablet/model/mixed_blocks.h | 2 ++ .../storage/tablet/profile_log_events.cpp | 2 ++ .../libs/storage/tablet/profile_log_events.h | 1 + .../storage/tablet/profile_log_events_ut.cpp | 5 ++- .../storage/tablet/tablet_actor_addblob.cpp | 23 ++++++++++++- .../storage/tablet/tablet_actor_counters.cpp | 34 +++++++++++-------- .../libs/storage/tablet/tablet_state.h | 2 ++ .../libs/storage/tablet/tablet_state_data.cpp | 12 +++++-- .../libs/storage/tablet/tablet_ut_data.cpp | 16 ++++----- .../storage/tablet/tablet_ut_data_stress.cpp | 2 +- .../filestore/libs/storage/testlib/test_env.h | 5 +-- .../filestore/private/api/protos/tablet.proto | 2 ++ .../libs/event-log/request_printer.cpp | 9 ++++- 15 files changed, 110 insertions(+), 30 deletions(-) diff --git a/cloud/filestore/libs/diagnostics/events/profile_events.ev b/cloud/filestore/libs/diagnostics/events/profile_events.ev index 25f14cfd7f..d0f2162899 100644 --- a/cloud/filestore/libs/diagnostics/events/profile_events.ev +++ b/cloud/filestore/libs/diagnostics/events/profile_events.ev @@ -69,6 +69,7 @@ message TProfileLogCompactionRangeInfo { optional uint64 RangeId = 2; optional uint64 BlobsCount = 3; optional uint64 DeletionsCount = 4; + optional uint64 GarbageBlocksCount = 5; }; message TProfileLogRequestInfo { diff --git a/cloud/filestore/libs/storage/tablet/model/mixed_blocks.cpp b/cloud/filestore/libs/storage/tablet/model/mixed_blocks.cpp index 50557dac42..8f4e97b775 100644 --- a/cloud/filestore/libs/storage/tablet/model/mixed_blocks.cpp +++ b/cloud/filestore/libs/storage/tablet/model/mixed_blocks.cpp @@ -345,4 +345,28 @@ TMixedBlobMeta TMixedBlocks::FindBlob(ui32 rangeId, TPartialBlobId blobId) const return {it->BlobId, std::move(blocks)}; } +ui32 TMixedBlocks::CalculateGarbageBlockCount(ui32 rangeId) const +{ + const auto* range = Impl->Ranges.FindPtr(rangeId); + Y_DEBUG_ABORT_UNLESS(range); + if (!range) { + return 0; + } + + ui32 result = 0; + for (const auto& blob: range->Blobs) { + auto blocks = blob.BlockList.DecodeBlocks(); + + range->DeletionMarkers.Apply(MakeArrayRef(blocks)); + for (const auto& block: blocks) { + // TODO(#1923): take checkpoints into account + if (block.MaxCommitId != InvalidCommitId) { + ++result; + } + } + } + + return result; +} + } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/model/mixed_blocks.h b/cloud/filestore/libs/storage/tablet/model/mixed_blocks.h index dd7fd492ff..d70c86c9ca 100644 --- a/cloud/filestore/libs/storage/tablet/model/mixed_blocks.h +++ b/cloud/filestore/libs/storage/tablet/model/mixed_blocks.h @@ -71,6 +71,8 @@ class TMixedBlocks TVector GetBlobsForCompaction(ui32 rangeId) const; TMixedBlobMeta FindBlob(ui32 rangeId, TPartialBlobId blobId) const; + + ui32 CalculateGarbageBlockCount(ui32 rangeId) const; }; } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/profile_log_events.cpp b/cloud/filestore/libs/storage/tablet/profile_log_events.cpp index de3e41ff0f..bedd8d4fda 100644 --- a/cloud/filestore/libs/storage/tablet/profile_log_events.cpp +++ b/cloud/filestore/libs/storage/tablet/profile_log_events.cpp @@ -177,6 +177,7 @@ void AddCompactionRange( ui32 rangeId, ui32 blobsCount, ui32 deletionsCount, + ui32 garbageBlocksCount, NProto::TProfileLogRequestInfo& profileLogRequest) { auto* range = profileLogRequest.AddCompactionRanges(); @@ -184,6 +185,7 @@ void AddCompactionRange( range->SetRangeId(rangeId); range->SetBlobsCount(blobsCount); range->SetDeletionsCount(deletionsCount); + range->SetGarbageBlocksCount(garbageBlocksCount); } template diff --git a/cloud/filestore/libs/storage/tablet/profile_log_events.h b/cloud/filestore/libs/storage/tablet/profile_log_events.h index d9d425286f..7053d8e656 100644 --- a/cloud/filestore/libs/storage/tablet/profile_log_events.h +++ b/cloud/filestore/libs/storage/tablet/profile_log_events.h @@ -87,6 +87,7 @@ void AddCompactionRange( ui32 rangeId, ui32 blobsCount, ui32 deletionsCount, + ui32 garbageBlocksCount, NProto::TProfileLogRequestInfo& profileLogRequest); template diff --git a/cloud/filestore/libs/storage/tablet/profile_log_events_ut.cpp b/cloud/filestore/libs/storage/tablet/profile_log_events_ut.cpp index 0699af7406..308f81cef5 100644 --- a/cloud/filestore/libs/storage/tablet/profile_log_events_ut.cpp +++ b/cloud/filestore/libs/storage/tablet/profile_log_events_ut.cpp @@ -409,7 +409,7 @@ Y_UNIT_TEST_SUITE(TProfileLogEvent) Y_UNIT_TEST(ShouldAddCompactionRange) { NProto::TProfileLogRequestInfo profileLogRequest; - AddCompactionRange(100500, 500100, 100, 500, profileLogRequest); + AddCompactionRange(100500, 500100, 100, 500, 1000, profileLogRequest); UNIT_ASSERT(!profileLogRequest.HasTimestampMcs()); UNIT_ASSERT(!profileLogRequest.HasDurationMcs()); @@ -429,6 +429,9 @@ Y_UNIT_TEST_SUITE(TProfileLogEvent) UNIT_ASSERT_VALUES_EQUAL( 500, profileLogRequest.GetCompactionRanges().at(0).GetDeletionsCount()); + UNIT_ASSERT_VALUES_EQUAL( + 1000, + profileLogRequest.GetCompactionRanges().at(0).GetGarbageBlocksCount()); UNIT_ASSERT(!profileLogRequest.HasNodeInfo()); UNIT_ASSERT(!profileLogRequest.HasLockInfo()); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_addblob.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_addblob.cpp index 5a59574eed..8e1f18fbe3 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_addblob.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_addblob.cpp @@ -147,6 +147,9 @@ class TAddBlobsExecutor block.BlockIndex, blob.BlocksCount); AccessCompactionStats(rangeId).BlobsCount += 1; + // conservative estimate + AccessCompactionStats(rangeId).GarbageBlocksCount += + blob.BlocksCount; } for (const auto& part: args.UnalignedDataParts) { @@ -221,6 +224,9 @@ class TAddBlobsExecutor if (written) { ui32 rangeId = Tablet.GetMixedRangeIndex(blob.Blocks); AccessCompactionStats(rangeId).BlobsCount += 1; + // conservative estimate + AccessCompactionStats(rangeId).GarbageBlocksCount += + blob.Blocks.size(); } } @@ -257,6 +263,8 @@ class TAddBlobsExecutor auto& stats = AccessCompactionStats(rangeId); if (Tablet.WriteMixedBlocks(db, blob.BlobId, blob.Blocks)) { stats.BlobsCount += 1; + // conservative estimate + stats.GarbageBlocksCount += blob.Blocks.size(); } } } @@ -273,6 +281,7 @@ class TAddBlobsExecutor auto& stats = AccessCompactionStats(rangeId); if (!Tablet.UpdateBlockLists(db, blob)) { stats.BlobsCount = Max(stats.BlobsCount, 1U) - 1; + // no proper way to reliably decrement stats.GarbageBlocksCount } } @@ -293,6 +302,8 @@ class TAddBlobsExecutor auto& stats = AccessCompactionStats(rangeId); if (Tablet.WriteMixedBlocks(db, blob.BlobId, blob.Blocks)) { stats.BlobsCount += 1; + // conservative estimate + stats.GarbageBlocksCount += blob.Blocks.size(); } } } @@ -304,6 +315,8 @@ class TAddBlobsExecutor TABLET_VERIFY(!args.MergedBlobs); TABLET_VERIFY(!args.UnalignedDataParts); + THashSet rangeIds; + for (const auto& blob: args.SrcBlobs) { const auto rangeId = Tablet.GetMixedRangeIndex(blob.Blocks); auto& stats = AccessCompactionStats(rangeId); @@ -337,8 +350,15 @@ class TAddBlobsExecutor } stats.BlobsCount += increment; } - } + // recalculating GarbageBlocksCount for each of the affected ranges + for (const auto& blob: args.SrcBlobs) { + const auto rangeId = Tablet.GetMixedRangeIndex(blob.Blocks); + + AccessCompactionStats(rangeId).GarbageBlocksCount = + Tablet.CalculateMixedIndexRangeGarbageBlockCount(rangeId); + } + } void UpdateCompactionMap( TIndexTabletDatabase& db, @@ -361,6 +381,7 @@ class TAddBlobsExecutor x.first, x.second.BlobsCount, x.second.DeletionsCount, + x.second.GarbageBlocksCount, args.ProfileLogRequest); } } diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp index 066d71577c..ee03ac5531 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp @@ -631,26 +631,32 @@ void TIndexTabletActor::HandleGetStorageStats( auto& req = ev->Get()->Record; + TVector topRanges; + if (req.GetCompactionRangeCountByCompactionScore()) { - const auto topRanges = GetTopRangesByCompactionScore( + const auto r = GetTopRangesByCompactionScore( req.GetCompactionRangeCountByCompactionScore()); - for (const auto& r: topRanges) { - auto* out = stats->AddCompactionRangeStats(); - out->SetRangeId(r.RangeId); - out->SetBlobCount(r.Stats.BlobsCount); - out->SetDeletionCount(r.Stats.DeletionsCount); - } + topRanges.insert(topRanges.end(), r.begin(), r.end()); } if (req.GetCompactionRangeCountByCleanupScore()) { - const auto topRanges = GetTopRangesByCleanupScore( + const auto r = GetTopRangesByCleanupScore( req.GetCompactionRangeCountByCleanupScore()); - for (const auto& r: topRanges) { - auto* out = stats->AddCompactionRangeStats(); - out->SetRangeId(r.RangeId); - out->SetBlobCount(r.Stats.BlobsCount); - out->SetDeletionCount(r.Stats.DeletionsCount); - } + topRanges.insert(topRanges.end(), r.begin(), r.end()); + } + + if (req.GetCompactionRangeCountByGarbageScore()) { + const auto r = GetTopRangesByGarbageScore( + req.GetCompactionRangeCountByGarbageScore()); + topRanges.insert(topRanges.end(), r.begin(), r.end()); + } + + for (const auto& r: topRanges) { + auto* out = stats->AddCompactionRangeStats(); + out->SetRangeId(r.RangeId); + out->SetBlobCount(r.Stats.BlobsCount); + out->SetDeletionCount(r.Stats.DeletionsCount); + out->SetGarbageBlockCount(r.Stats.GarbageBlocksCount); } stats->SetFlushState(static_cast(FlushState.GetOperationState())); diff --git a/cloud/filestore/libs/storage/tablet/tablet_state.h b/cloud/filestore/libs/storage/tablet/tablet_state.h index cc9a383ad7..3187a9a61d 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state.h +++ b/cloud/filestore/libs/storage/tablet/tablet_state.h @@ -908,6 +908,8 @@ FILESTORE_DUPCACHE_REQUESTS(FILESTORE_DECLARE_DUPCACHE) ui32 GetMixedRangeIndex(const TVector& blocks) const; const IBlockLocation2RangeIndex& GetRangeIdHasher() const; + ui32 CalculateMixedIndexRangeGarbageBlockCount(ui32 rangeId) const; + private: bool WriteMixedBlocks( TIndexTabletDatabase& db, diff --git a/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp b/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp index f790a24c27..f0f5c24792 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp @@ -954,8 +954,9 @@ ui32 TIndexTabletState::CleanupBlockDeletions( stats.BlobsCount = (stats.BlobsCount > removedBlobs) ? (stats.BlobsCount - removedBlobs) : 0; stats.DeletionsCount = 0; - // TODO: calculate GarbageBlocksCount - stats.GarbageBlocksCount = 0; + if (stats.BlobsCount == 0) { + stats.GarbageBlocksCount = 0; + } db.WriteCompactionMap( rangeId, @@ -973,6 +974,7 @@ ui32 TIndexTabletState::CleanupBlockDeletions( rangeId, stats.BlobsCount, stats.DeletionsCount, + stats.GarbageBlocksCount, profileLogRequest); return deletionMarkerCount; @@ -1078,6 +1080,12 @@ const IBlockLocation2RangeIndex& TIndexTabletState::GetRangeIdHasher() const return *Impl->RangeIdHasher; } +ui32 TIndexTabletState::CalculateMixedIndexRangeGarbageBlockCount( + ui32 rangeId) const +{ + return Impl->MixedBlocks.CalculateGarbageBlockCount(rangeId); +} + //////////////////////////////////////////////////////////////////////////////// // LargeBlocks diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp index 5554ca354c..c706e586ec 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp @@ -3566,28 +3566,28 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) stats.GetAllocatedCompactionRanges()); UNIT_ASSERT_VALUES_EQUAL(8, stats.CompactionRangeStatsSize()); UNIT_ASSERT_VALUES_EQUAL( - "r=1656356864 b=16 d=1024", + "r=1656356864 b=16 d=1024 g=1024", CompactionRangeToString(stats.GetCompactionRangeStats(0))); UNIT_ASSERT_VALUES_EQUAL( - "r=1656356865 b=16 d=1024", + "r=1656356865 b=16 d=1024 g=1024", CompactionRangeToString(stats.GetCompactionRangeStats(1))); UNIT_ASSERT_VALUES_EQUAL( - "r=4283236352 b=16 d=1024", + "r=4283236352 b=16 d=1024 g=1024", CompactionRangeToString(stats.GetCompactionRangeStats(2))); UNIT_ASSERT_VALUES_EQUAL( - "r=4283236353 b=16 d=1024", + "r=4283236353 b=16 d=1024 g=1024", CompactionRangeToString(stats.GetCompactionRangeStats(3))); UNIT_ASSERT_VALUES_EQUAL( - "r=1177944064 b=14 d=833", + "r=1177944064 b=14 d=833 g=832", CompactionRangeToString(stats.GetCompactionRangeStats(4))); UNIT_ASSERT_VALUES_EQUAL( - "r=1177944065 b=13 d=832", + "r=1177944065 b=13 d=832 g=832", CompactionRangeToString(stats.GetCompactionRangeStats(5))); UNIT_ASSERT_VALUES_EQUAL( - "r=737148928 b=3 d=192", + "r=737148928 b=3 d=192 g=192", CompactionRangeToString(stats.GetCompactionRangeStats(6))); UNIT_ASSERT_VALUES_EQUAL( - "r=737148929 b=3 d=192", + "r=737148929 b=3 d=192 g=192", CompactionRangeToString(stats.GetCompactionRangeStats(7))); }; diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_data_stress.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_data_stress.cpp index 40f9dde13a..1415956016 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut_data_stress.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut_data_stress.cpp @@ -465,7 +465,7 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data_Stress) UNIT_ASSERT_VALUES_EQUAL(1, stats.CompactionRangeStatsSize()); UNIT_ASSERT_VALUES_EQUAL( Sprintf( - "r=1177944064 b=%u d=%u", + "r=1177944064 b=%u d=%u g=0", (compactionThreshold - 1), deletionMarkers), CompactionRangeToString(stats.GetCompactionRangeStats(0))); diff --git a/cloud/filestore/libs/storage/testlib/test_env.h b/cloud/filestore/libs/storage/testlib/test_env.h index d5c658d962..5c1000225e 100644 --- a/cloud/filestore/libs/storage/testlib/test_env.h +++ b/cloud/filestore/libs/storage/testlib/test_env.h @@ -49,10 +49,11 @@ inline auto CompactionRangeToString( const NProtoPrivate::TCompactionRangeStats& rs) { return Sprintf( - "r=%u b=%u d=%u", + "r=%u b=%u d=%u g=%u", rs.GetRangeId(), rs.GetBlobCount(), - rs.GetDeletionCount()); + rs.GetDeletionCount(), + rs.GetGarbageBlockCount()); } //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/filestore/private/api/protos/tablet.proto b/cloud/filestore/private/api/protos/tablet.proto index 99b85becf7..41d1aacc8e 100644 --- a/cloud/filestore/private/api/protos/tablet.proto +++ b/cloud/filestore/private/api/protos/tablet.proto @@ -151,6 +151,8 @@ message TGetStorageStatsRequest uint32 CompactionRangeCountByCompactionScore = 3; // The same but for cleanup score. uint32 CompactionRangeCountByCleanupScore = 4; + // The same but for garbage score. + uint32 CompactionRangeCountByGarbageScore = 5; } message TGetStorageStatsResponse diff --git a/cloud/filestore/tools/analytics/libs/event-log/request_printer.cpp b/cloud/filestore/tools/analytics/libs/event-log/request_printer.cpp index bc562a2732..34598cbc32 100644 --- a/cloud/filestore/tools/analytics/libs/event-log/request_printer.cpp +++ b/cloud/filestore/tools/analytics/libs/event-log/request_printer.cpp @@ -260,7 +260,14 @@ TString PrintCompactionRanges( currentRange << PrintValue("blobsCount", range.GetBlobsCount()) << ", "; } if (range.HasDeletionsCount()) { - currentRange << PrintValue("deletionsCount", range.GetDeletionsCount()) << ", "; + currentRange << PrintValue( + "deletionsCount", + range.GetDeletionsCount()) << ", "; + } + if (range.HasGarbageBlocksCount()) { + currentRange << PrintValue( + "garbageBlocksCount", + range.GetGarbageBlocksCount()) << ", "; } if (currentRange.empty()) { From 511cbe1f57780a31b87490b2fd6e4938337b89dc Mon Sep 17 00:00:00 2001 From: qkrorlqr Date: Fri, 18 Oct 2024 15:45:18 +0000 Subject: [PATCH 2/3] issue-2137: calculating GarbageBlocksCount stats in CompactionMap - cleanup --- cloud/filestore/libs/storage/tablet/tablet_actor_addblob.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_addblob.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_addblob.cpp index 8e1f18fbe3..180d4c8c60 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_addblob.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_addblob.cpp @@ -315,8 +315,6 @@ class TAddBlobsExecutor TABLET_VERIFY(!args.MergedBlobs); TABLET_VERIFY(!args.UnalignedDataParts); - THashSet rangeIds; - for (const auto& blob: args.SrcBlobs) { const auto rangeId = Tablet.GetMixedRangeIndex(blob.Blocks); auto& stats = AccessCompactionStats(rangeId); From 6a82597071788564f0a542dba99d6ff91b358fee Mon Sep 17 00:00:00 2001 From: qkrorlqr Date: Fri, 18 Oct 2024 17:39:13 +0000 Subject: [PATCH 3/3] issue-2137: calculating GarbageBlocksCount for each affected rangeId only once --- .../libs/storage/tablet/tablet_actor_addblob.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_addblob.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_addblob.cpp index 180d4c8c60..0f73ce7ad5 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_addblob.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_addblob.cpp @@ -315,6 +315,8 @@ class TAddBlobsExecutor TABLET_VERIFY(!args.MergedBlobs); TABLET_VERIFY(!args.UnalignedDataParts); + THashSet rangeIds; + for (const auto& blob: args.SrcBlobs) { const auto rangeId = Tablet.GetMixedRangeIndex(blob.Blocks); auto& stats = AccessCompactionStats(rangeId); @@ -324,6 +326,8 @@ class TAddBlobsExecutor // needed. stats.BlobsCount = Max(1U, stats.BlobsCount) - 1; Tablet.DeleteMixedBlocks(db, blob.BlobId, blob.Blocks); + + rangeIds.insert(rangeId); } THashMap rangeId2AddedBlobsCount; @@ -334,6 +338,8 @@ class TAddBlobsExecutor if (Tablet.WriteMixedBlocks(db, blob.BlobId, blob.Blocks)) { ++rangeId2AddedBlobsCount[rangeId]; } + + rangeIds.insert(rangeId); } for (const auto& [rangeId, addedBlobsCount]: rangeId2AddedBlobsCount) { @@ -350,9 +356,7 @@ class TAddBlobsExecutor } // recalculating GarbageBlocksCount for each of the affected ranges - for (const auto& blob: args.SrcBlobs) { - const auto rangeId = Tablet.GetMixedRangeIndex(blob.Blocks); - + for (const auto rangeId: rangeIds) { AccessCompactionStats(rangeId).GarbageBlocksCount = Tablet.CalculateMixedIndexRangeGarbageBlockCount(rangeId); }