feat(storage): add metric to monitor available split block tasks

broxus · Feb 7, 2025 · 9f72fdb · 9f72fdb
1 parent 23e6417
commit 9f72fdb
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 28 deletions.
diff --git a/scripts/gen-dashboard.py b/scripts/gen-dashboard.py
@@ -62,11 +62,11 @@ def generate_legend_format(labels: List[str]) -> str:
 
 
 def create_gauge_panel(
-    expr: Union[str, List[Union[str, Expr]], Expr],
-    title: str,
-    unit_format=UNITS.NUMBER_FORMAT,
-    labels=[],
-    legend_format: str | None = None,
+        expr: Union[str, List[Union[str, Expr]], Expr],
+        title: str,
+        unit_format=UNITS.NUMBER_FORMAT,
+        labels=[],
+        legend_format: str | None = None,
 ) -> Panel:
     if isinstance(expr, str):
         expr = [Expr(metric=expr, label_selectors=labels)]
@@ -95,12 +95,12 @@ def create_gauge_panel(
 
 
 def create_counter_panel(
-    expr: Union[str | Expr, List[Union[str, Expr]]],
-    title: str,
-    unit_format: str = UNITS.NUMBER_FORMAT,
-    labels_selectors: List[str] = [],
-    legend_format: str | None = None,
-    by_labels: list[str] = ["instance"],
+        expr: Union[str | Expr, List[Union[str, Expr]]],
+        title: str,
+        unit_format: str = UNITS.NUMBER_FORMAT,
+        labels_selectors: List[str] = [],
+        legend_format: str | None = None,
+        by_labels: list[str] = ["instance"],
 ) -> Panel:
     """
     Create a counter panel for visualization.
@@ -160,12 +160,12 @@ def create_counter_panel(
 
 
 def create_percent_panel(
-    metric1: str,
-    metric2: str,
-    title: str,
-    group_by_labels: List[str] = ["instance"],
-    label_selectors: List[str] = [],
-    unit_format: str = UNITS.PERCENT_FORMAT,
+        metric1: str,
+        metric2: str,
+        title: str,
+        group_by_labels: List[str] = ["instance"],
+        label_selectors: List[str] = [],
+        unit_format: str = UNITS.PERCENT_FORMAT,
 ) -> Panel:
     """
     create a panel showing the percentage of metric1 to metric2, grouped by specified labels.
@@ -198,10 +198,10 @@ def create_percent_panel(
 
 
 def create_heatmap_panel(
-    metric_name,
-    title,
-    unit_format=yaxis(UNITS.SECONDS),
-    labels=[],
+        metric_name,
+        title,
+        unit_format=yaxis(UNITS.SECONDS),
+        labels=[],
 ) -> Panel:
     return heatmap_panel(
         title,
@@ -220,10 +220,10 @@ def create_heatmap_panel(
 
 
 def create_heatmap_quantile_panel(
-    metric_name: str,
-    title: str,
-    unit_format: str = UNITS.NUMBER_FORMAT,
-    quantile: AcceptedQuantile = "0.95",
+        metric_name: str,
+        title: str,
+        unit_format: str = UNITS.NUMBER_FORMAT,
+        quantile: AcceptedQuantile = "0.95",
 ) -> Panel:
     """
     Create a heatmap quantile panel for the given metric.
@@ -260,11 +260,11 @@ def create_heatmap_quantile_panel(
 
 
 def create_row(
-    name: str, metrics, repeat: str | None = None, collapsed=True
+        name: str, metrics, repeat: str | None = None, collapsed=True
 ) -> RowPanel:
     layout = Layout(name, repeat=repeat, collapsed=collapsed)
     for i in range(0, len(metrics), 2):
-        chunk = metrics[i : i + 2]
+        chunk = metrics[i: i + 2]
         layout.row(chunk)
     return layout.row_panel
 
@@ -855,6 +855,12 @@ def storage() -> RowPanel:
             UNITS.BYTES,
             "0.999",
         ),
+        create_heatmap_quantile_panel(
+            "tycho_storage_split_block_available_permits",
+            "Split block available permits",
+            UNITS.NUMBER_FORMAT,
+            "0.999",
+        ),
         create_heatmap_panel(
             "tycho_storage_batch_write_time", "Time to write merge in write batch"
         ),
@@ -2392,7 +2398,6 @@ def templates() -> Templating:
     timezone="browser",
 ).auto_panel_ids()
 
-
 # open file as stream
 if len(sys.argv) > 1:
     stream = open(sys.argv[1], "w")

diff --git a/storage/src/store/block/mod.rs b/storage/src/store/block/mod.rs
@@ -1032,6 +1032,9 @@ impl BlockStorage {
         self.db.rocksdb().write(batch)?;
 
         // Start splitting block data
+        metrics::histogram!("tycho_storage_split_block_available_permits")
+            .record(self.split_block_semaphore.available_permits() as f64);
+
         let permit = self.split_block_semaphore.clone().acquire_owned().await?;
         let _handle = self.spawn_split_block_data(&id.block_id, data, permit);