install Prometheus as subchart

helm/supersonic/Chart.yaml

@@ -13,6 +13,10 @@ annotations:
 
 dependencies:
   - name: grafana
-    version: "8.9.0"  # Latest stable version as of now
+    version: "8.9.0"
     repository: https://grafana.github.io/helm-charts
     condition: grafana-legacy.enabled
+  - name: prometheus
+    version: "27.3.0"
+    repository: https://prometheus-community.github.io/helm-charts
+    condition: prometheus.enabled

helm/supersonic/templates/_prometheus.tpl

@@ -43,15 +43,23 @@ Get existing Prometheus service name (from any release)
 Get Prometheus URL (handles external, ingress, existing, and new instances)
 */}}
 {{- define "supersonic.prometheusUrl" -}}
-{{- if (not .Values.prometheus.external) -}}
+{{- if .Values.prometheus.external -}}
+{{- if .Values.prometheus.url -}}
+{{ .Values.prometheus.scheme }}://{{ .Values.prometheus.url }}
+{{- end -}}
+{{- else if .Values.prometheus.enabled -}}
+{{- if .Values.prometheus.ingress.enabled -}}
+https://{{ .Values.prometheus.ingress.hostName }}
+{{- else -}}
+http://{{ include "supersonic.prometheusName" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.prometheus.server.service.servicePort }}
+{{- end -}}
+{{- else -}}
 {{- if .Values.prometheus.ingress.enabled -}}
 https://{{ .Values.prometheus.ingress.hostName }}
 {{- else -}}
 http://{{ include "supersonic.prometheusName" . }}.{{ .Release.Namespace }}.svc.cluster.local:9090
 {{- end -}}
-{{- else if .Values.prometheus.url -}}
-{{ .Values.prometheus.scheme }}://{{ .Values.prometheus.url }}
-{{- end }}
+{{- end -}}
 {{- end }}
 
 {{/*

helm/supersonic/templates/grafana-datasources.yaml

@@ -0,0 +1,28 @@
+{{- if .Values.grafana.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-grafana-datasources
+  labels:
+    grafana_datasource: "1"
+data:
+  datasources.yaml: |
+    apiVersion: 1
+    datasources:
+      - name: prometheus
+        type: prometheus
+        access: proxy
+        isDefault: true
+        {{- if .Values.prometheus.external }}
+        url: {{ .Values.prometheus.scheme }}://{{ .Values.prometheus.url }}
+        {{- else if .Values.prometheus.enabled }}
+        url: http://{{ .Release.Name }}-prometheus-server.{{ .Release.Namespace }}.svc.cluster.local:9090
+        {{- else }}
+        url: http://{{ include "supersonic.prometheusName" . }}.{{ .Release.Namespace }}.svc.cluster.local:9090
+        {{- end }}
+        jsonData:
+          timeInterval: "5s"
+          {{- if and .Values.prometheus.external (eq .Values.prometheus.scheme "https") }}
+          tlsSkipVerify: true
+          {{- end }}
+{{- end }} 

helm/supersonic/templates/prometheus-configmap.yaml

@@ -1,4 +1,4 @@
-{{- if not .Values.prometheus.external }}
+{{- if and .Values.prometheus.enabled (not .Values.prometheus.external) }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -10,8 +10,8 @@ metadata:
 data:
   prometheus.yml: |
     global:
-      scrape_interval: 5s
-      evaluation_interval: 5s
+      scrape_interval: {{ .Values.prometheus.server.global.scrape_interval }}
+      evaluation_interval: {{ .Values.prometheus.server.global.evaluation_interval }}
 
     scrape_configs:
       - job_name: "{{ include "supersonic.tritonName" . }}"

helm/supersonic/values.yaml

@@ -170,6 +170,9 @@ prometheus:
   # -- Whether to use external Prometheus instance (true) or deploy internal one (false)
   external: true
 
+  # -- Enable or disable Prometheus deployment via subchart
+  enabled: false  # Set to true to deploy Prometheus via subchart
+
   # -- External Prometheus server url and port number (find in documentation of a given cluster or ask admins)
   # Only used when external=true
   url: ""
@@ -193,6 +196,56 @@ prometheus:
     ingressClassName: ""
     annotations: {}
 
+  # -- Prometheus Helm chart configuration (https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus)
+  server:
+    useExistingClusterRoleName: supersonic-prometheus-role
+    releaseNamespace: true
+    persistentVolume:
+      enabled: false
+    resources:
+      requests:
+        cpu: 500m
+        memory: 512Mi
+      limits:
+        cpu: 1
+        memory: 1Gi
+    retention: 15d
+    global:
+      scrape_interval: 5s
+      evaluation_interval: 5s
+    service:
+      enabled: true
+      servicePort: 9090
+    configMapOverrideName: prometheus-config
+    ingress:
+      enabled: true
+      hosts: []
+      ingressClassName: ""
+      annotations: {}
+      tls:
+        - hosts: []
+
+  serviceAccounts:
+    server:
+      create: false
+      name: supersonic-prometheus-sa
+
+  rbac:
+    create: false
+  alertmanager:
+    enabled: false
+  pushgateway:
+    enabled: false
+  kube-state-metrics:
+    enabled: false
+  prometheus-node-exporter:
+    enabled: false
+  prometheus-pushgateway:
+    enabled: false
+  configmapReload:
+    prometheus:
+      enabled: false
+
 ingress:
   enabled: false
   hostName: ""
@@ -244,6 +297,10 @@ grafana:
           type: prometheus
           access: proxy
           isDefault: true
+          url: http://supersonic-prometheus-server:9090
+          jsonData:
+            timeInterval: "5s"
+            tlsSkipVerify: true
 
   # -- Grafana dashboard providers configuration
   dashboardProviders:

values/values-nautilus-cms.yaml

@@ -11,8 +11,8 @@ triton:
       --exit-timeout-secs=60 \
       --backend-config=onnxruntime,enable-global-threadpool=1
   resources:
-    limits: { cpu: 2, memory: 4G, nvidia.com/a100: 1}
-    requests: { cpu: 2, memory: 4G, nvidia.com/a100: 1}
+    limits: { cpu: 2, memory: 4G, nvidia.com/gpu: 1}
+    requests: { cpu: 2, memory: 4G, nvidia.com/gpu: 1}
   modelRepository:
     enabled: true
     storageType: cvmfs-pvc
@@ -22,15 +22,21 @@ envoy:
   enabled: true
   loadBalancerPolicy: "ROUND_ROBIN"
 prometheus:
+  enabled: true
   external: false
-  url: prometheus.nrp-nautilus.io
-  port: 443
-  scheme: https
-  serverLoadThreshold: 100
-  ingress:
-    enabled: true
-    hostName: prometheus-cms.nrp-nautilus.io
-    ingressClassName: haproxy
+  server:
+    ingress:
+      enabled: true
+      hosts:
+        - prometheus-cms.nrp-nautilus.io
+      tls:
+        - hosts:
+            - prometheus-cms.nrp-nautilus.io
+      ingressClassName: haproxy
+      annotations:
+        haproxy-ingress.github.io/cors-enable: "true"
+        haproxy-ingress.github.io/proxy-body-size: "512m"
+        haproxy-ingress.github.io/timeout-http-request: "5m"
 
 autoscaler:
   enabled: true
@@ -67,15 +73,6 @@ grafana-legacy:
 grafana:
   enabled: true
 
-  datasources:
-    datasources.yaml:
-      datasources:
-        - name: prometheus
-          type: prometheus
-          access: proxy
-          url: https://prometheus.nrp-nautilus.io
-          isDefault: true
-
   ingress:
     enabled: true
     hosts: