Merge pull request #32830 from vespa-engine/toregge/populate-metrics-…

…for-attribute-vector-size-on-disk Populate metrics for attribute vector size on disk.
vespa-engine · Nov 11, 2024 · 26a6054 · 26a6054
2 parents c9f97eb + 05d77d2
commit 26a6054
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 7 deletions.
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp
@@ -90,10 +90,15 @@ FlushableAttribute::Flusher::saveAttribute()
         if (_saver) {
             search::AttributeFileSaveTarget saveTarget(_fattr._tuneFileAttributes, fileHeaderContext);
             saveSuccess = _saver->save(saveTarget);
+            if (saveSuccess) {
+                _fattr._attr->set_size_on_disk(saveTarget.size_on_disk());
+            }
             _saver.reset();
         } else {
             saveSuccess = _saveTarget.writeToFile(_fattr._tuneFileAttributes, fileHeaderContext);
-            _fattr._attr->set_size_on_disk(_saveTarget.size_on_disk());
+            if (saveSuccess) {
+                _fattr._attr->set_size_on_disk(_saveTarget.size_on_disk());
+            }
         }
     }
     return saveSuccess;

diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp
@@ -78,10 +78,16 @@ DocumentMetaStoreFlushTarget::Flusher::saveDocumentMetaStore()
         _saver.reset();
         if (saveSuccess) {
             saveSuccess = memorySaveTarget.writeToFile(_dmsft._tuneFileAttributes, fileHeaderContext);
+            if (saveSuccess) {
+                _dmsft._dms->set_size_on_disk(memorySaveTarget.size_on_disk());
+            }
         }
     } else {
         search::AttributeFileSaveTarget saveTarget(_dmsft._tuneFileAttributes, fileHeaderContext);
         saveSuccess = _saver->save(saveTarget);
+        if (saveSuccess) {
+            _dmsft._dms->set_size_on_disk(saveTarget.size_on_disk());
+        }
         _saver.reset();
     }
     return saveSuccess;

diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp
@@ -71,6 +71,7 @@ DocumentDBTaggedMetrics::SubDBMetrics::DocumentStoreMetrics::~DocumentStoreMetri
 
 DocumentDBTaggedMetrics::AttributeMetrics::AttributeMetrics(MetricSet *parent)
     : MetricSet("attribute", {}, "Attribute vector metrics for this document db", parent),
+      diskUsage("disk_usage", {}, "Disk space usage in bytes", this),
       resourceUsage(this),
       totalMemoryUsage(this)
 {

diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h
@@ -84,6 +84,7 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet
             ~ResourceUsageMetrics() override;
         };
 
+        metrics::LongValueMetric diskUsage;
         ResourceUsageMetrics resourceUsage;
         MemoryUsageMetrics totalMemoryUsage;
 

diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp
@@ -94,10 +94,12 @@ struct TempAttributeMetric
 {
     MemoryUsage memoryUsage;
     uint64_t    bitVectors;
+    uint64_t    size_on_disk;
 
     TempAttributeMetric()
         : memoryUsage(),
-          bitVectors(0)
+          bitVectors(0),
+          size_on_disk(0)
     {}
 };
 
@@ -122,13 +124,15 @@ isNotReadySubDB(const IDocumentSubDB *subDb, const DocumentSubDBCollection &subD
 
 void
 fillTempAttributeMetrics(TempAttributeMetrics &metrics, const std::string &attrName,
-                         const MemoryUsage &memoryUsage, uint32_t bitVectors)
+                         const MemoryUsage &memoryUsage, uint32_t bitVectors, uint64_t size_on_disk)
 {
     metrics.total.memoryUsage.merge(memoryUsage);
     metrics.total.bitVectors += bitVectors;
+    metrics.total.size_on_disk += size_on_disk;
     TempAttributeMetric &m = metrics.attrs[attrName];
     m.memoryUsage.merge(memoryUsage);
     m.bitVectors += bitVectors;
+    m.size_on_disk += size_on_disk;
 }
 
 void
@@ -149,9 +153,10 @@ fillTempAttributeMetrics(TempAttributeMetrics &totalMetrics,
                 const search::attribute::Status &status = attr->getStatus();
                 MemoryUsage memoryUsage(status.getAllocated(), status.getUsed(), status.getDead(), status.getOnHold());
                 uint32_t bitVectors = status.getBitVectors();
-                fillTempAttributeMetrics(totalMetrics, attr->getName(), memoryUsage, bitVectors);
+                uint64_t size_on_disk = attr->size_on_disk();
+                fillTempAttributeMetrics(totalMetrics, attr->getName(), memoryUsage, bitVectors, size_on_disk);
                 if (subMetrics != nullptr) {
-                    fillTempAttributeMetrics(*subMetrics, attr->getName(), memoryUsage, bitVectors);
+                    fillTempAttributeMetrics(*subMetrics, attr->getName(), memoryUsage, bitVectors, size_on_disk);
                 }
             }
             auto imported = attrMgr->getImportedAttributes();
@@ -160,9 +165,9 @@ fillTempAttributeMetrics(TempAttributeMetrics &totalMetrics,
                 imported->getAll(i_list);
                 for (const auto& attr : i_list) {
                     auto memory_usage = attr->get_memory_usage();
-                    fillTempAttributeMetrics(totalMetrics,  attr->getName(), memory_usage, 0);
+                    fillTempAttributeMetrics(totalMetrics,  attr->getName(), memory_usage, 0, 0);
                     if (subMetrics != nullptr) {
-                        fillTempAttributeMetrics(*subMetrics,  attr->getName(), memory_usage, 0);
+                        fillTempAttributeMetrics(*subMetrics,  attr->getName(), memory_usage, 0, 0);
                     }
                 }
             }
@@ -177,6 +182,7 @@ updateAttributeMetrics(AttributeMetrics &metrics, const TempAttributeMetrics &tm
         auto entry = metrics.get_field_metrics_entry(attr.first);
         if (entry) {
             entry->memoryUsage.update(attr.second.memoryUsage);
+            entry->size_on_disk.set(attr.second.size_on_disk);
         }
     }
 }
@@ -192,6 +198,7 @@ updateAttributeMetrics(DocumentDBTaggedMetrics &metrics, const DocumentSubDBColl
     updateAttributeMetrics(metrics.ready.attributes, readyMetrics);
     updateAttributeMetrics(metrics.notReady.attributes, notReadyMetrics);
     updateMemoryUsageMetrics(metrics.attribute.totalMemoryUsage, totalMetrics.total.memoryUsage, totalStats);
+    updateDiskUsageMetric(metrics.attribute.diskUsage, totalMetrics.total.size_on_disk, totalStats);
 }
 
 void