Skip to content

Commit

Permalink
feat: pipeline gauge metrics (#2284)
Browse files Browse the repository at this point in the history 
Signed-off-by: veds-g <[email protected]>
  • Loading branch information
@veds-g
veds-g authored Dec 15, 2024
1 parent 883650c commit d75998f
Show file tree
Hide file tree
Showing 9 changed files with 397 additions and 166 deletions.
87 changes: 56 additions & 31 deletions config/advanced-install/namespaced-numaflow-server.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,33 +143,71 @@ data:
# example for local prometheus service
# url: http://prometheus-operated.monitoring.svc.cluster.local:9090
patterns:
- name: vertex_gauge
object: vertex
title: Vertex Pending Messages
description: This query is the total number of pending messages for the vertex
expr: |
sum($metric_name{$filters}) by ($dimension, period)
params:
- name: start_time
required: false
- name: end_time
required: false
metrics:
- metric_name: vertex_pending_messages
required_filters:
- namespace
- pipeline
- vertex
dimensions:
- name: pod
# expr: optional expression for prometheus query
# overrides the default expression
filters:
- name: pod
required: false
- name: period
required: false
- name: vertex
# expr: optional expression for prometheus query
# overrides the default expression
filters:
- name: period
required: false
- name: mono_vertex_gauge
object: mono-vertex
title: Pending Messages Lag
description: This query is the total number of pending messages for the mono vertex
expr: |
$metric_name{$filters}
sum($metric_name{$filters}) by ($dimension, period)
params:
- name: start_time
required: false
- name: end_time
required: false
metrics:
- metric_name: monovtx_pending
required_filters:
- namespace
- mvtx_name
dimensions:
- name: pod
filters:
- name: pod
required: false
- name: period
required: false
- name: mono-vertex
filters:
- name: period
required: false
- metric_name: monovtx_pending
required_filters:
- namespace
- mvtx_name
dimensions:
- name: pod
# expr: optional expression for prometheus query
# overrides the default expression
filters:
- name: pod
required: false
- name: period
required: false
- name: mono-vertex
# expr: optional expression for prometheus query
# overrides the default expression
filters:
- name: period
required: false
- name: mono_vertex_histogram
object: mono-vertex
title: Processing Time Latency
Expand All @@ -192,11 +230,7 @@ data:
- mvtx_name
dimensions:
- name: mono-vertex
# expr: optional expression for prometheus query
# overrides the default expression
- name: pod
# expr: optional expression for prometheus query
# overrides the default expression
filters:
- name: pod
required: false
Expand All @@ -206,15 +240,11 @@ data:
- mvtx_name
dimensions:
- name: mono-vertex
# expr: optional expression for prometheus query
# overrides the default expression
- name: pod
# expr: optional expression for prometheus query
# overrides the default expression
filters:
- name: pod
required: false
# Add histogram metrics similar to the pattern above
- name: vertex_throughput
object: vertex
title: Vertex Throughput and Message Rates
Expand All @@ -235,12 +265,11 @@ data:
- vertex
dimensions:
- name: vertex
# expr: optional expression for prometheus query
# overrides the default expression
- name: pod
filters:
- name: pod
required: false
- name: mono_vertex_throughput
object: mono-vertex
title: Mono-Vertex Throughput and Message Rates
Expand All @@ -260,11 +289,7 @@ data:
- mvtx_name
dimensions:
- name: mono-vertex
# expr: optional expression for prometheus query
# overrides the default expression
- name: pod
# expr: optional expression for prometheus query
# overrides the default expression
filters:
- name: pod
required: false
Expand Down
87 changes: 56 additions & 31 deletions config/advanced-install/numaflow-server.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,33 +150,71 @@ data:
# example for local prometheus service
# url: http://prometheus-operated.monitoring.svc.cluster.local:9090
patterns:
- name: vertex_gauge
object: vertex
title: Vertex Pending Messages
description: This query is the total number of pending messages for the vertex
expr: |
sum($metric_name{$filters}) by ($dimension, period)
params:
- name: start_time
required: false
- name: end_time
required: false
metrics:
- metric_name: vertex_pending_messages
required_filters:
- namespace
- pipeline
- vertex
dimensions:
- name: pod
# expr: optional expression for prometheus query
# overrides the default expression
filters:
- name: pod
required: false
- name: period
required: false
- name: vertex
# expr: optional expression for prometheus query
# overrides the default expression
filters:
- name: period
required: false
- name: mono_vertex_gauge
object: mono-vertex
title: Pending Messages Lag
description: This query is the total number of pending messages for the mono vertex
expr: |
$metric_name{$filters}
sum($metric_name{$filters}) by ($dimension, period)
params:
- name: start_time
required: false
- name: end_time
required: false
metrics:
- metric_name: monovtx_pending
required_filters:
- namespace
- mvtx_name
dimensions:
- name: pod
filters:
- name: pod
required: false
- name: period
required: false
- name: mono-vertex
filters:
- name: period
required: false
- metric_name: monovtx_pending
required_filters:
- namespace
- mvtx_name
dimensions:
- name: pod
# expr: optional expression for prometheus query
# overrides the default expression
filters:
- name: pod
required: false
- name: period
required: false
- name: mono-vertex
# expr: optional expression for prometheus query
# overrides the default expression
filters:
- name: period
required: false
- name: mono_vertex_histogram
object: mono-vertex
title: Processing Time Latency
Expand All @@ -199,11 +237,7 @@ data:
- mvtx_name
dimensions:
- name: mono-vertex
# expr: optional expression for prometheus query
# overrides the default expression
- name: pod
# expr: optional expression for prometheus query
# overrides the default expression
filters:
- name: pod
required: false
Expand All @@ -213,15 +247,11 @@ data:
- mvtx_name
dimensions:
- name: mono-vertex
# expr: optional expression for prometheus query
# overrides the default expression
- name: pod
# expr: optional expression for prometheus query
# overrides the default expression
filters:
- name: pod
required: false
# Add histogram metrics similar to the pattern above
- name: vertex_throughput
object: vertex
title: Vertex Throughput and Message Rates
Expand All @@ -242,12 +272,11 @@ data:
- vertex
dimensions:
- name: vertex
# expr: optional expression for prometheus query
# overrides the default expression
- name: pod
filters:
- name: pod
required: false
- name: mono_vertex_throughput
object: mono-vertex
title: Mono-Vertex Throughput and Message Rates
Expand All @@ -267,11 +296,7 @@ data:
- mvtx_name
dimensions:
- name: mono-vertex
# expr: optional expression for prometheus query
# overrides the default expression
- name: pod
# expr: optional expression for prometheus query
# overrides the default expression
filters:
- name: pod
required: false
Expand Down
Loading

0 comments on commit d75998f

Please sign in to comment.