feat: implement gauges, counters and histogram for ctx (#671)

* feat: implement gauges, counters and histogram for ctx

* chore: fix histogram labels in upstream pkg

* chore: add panic recovery for prom metrics

* chore: bump go to 1.22

* chore: update codeql action version

* chore: go mod tidy

* chore: update codeql action version

* chore: go version to 1.22.0

* chore: add table label to push_queue_create_handler_records counter

* chore: add counter label for upstream

.github/workflows/codeql.yml

@@ -44,9 +44,13 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
+      - uses: actions/setup-go@v5
+        with:
+          check-latest: true
+
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@0b21cf2492b6b02c465a3e5d7c473717ad7721ba # v3.23.1
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -56,7 +60,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@0b21cf2492b6b02c465a3e5d7c473717ad7721ba # v3.23.1
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -69,6 +73,6 @@ jobs:
       #   ./location_of_script_within_repo/buildscript.sh
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@0b21cf2492b6b02c465a3e5d7c473717ad7721ba # v3.23.1
+        uses: github/codeql-action/analyze@v3
         with:
           category: "/language:${{matrix.language}}"

.github/workflows/lint.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
-          go-version: 1.20.x
+          go-version: 1.22.x
       - name: golangci-lint
         uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0
         with:

.github/workflows/release.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
-          go-version: v1.20.x
+          go-version: v1.22.x
       - name: Create commits
         run: |
           # Sleep to let index refresh

.github/workflows/test.yaml

@@ -9,7 +9,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
-          go-version: 1.20.x
+          go-version: 1.22.x
       - name: Checkout code
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
@@ -31,7 +31,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
-          go-version: 1.20.x
+          go-version: 1.22.x
       - name: Check out main branch
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:

context/context.go

@@ -2,6 +2,8 @@ package context
 
 import (
 	gocontext "context"
+	"slices"
+	"strings"
 	"time"
 
 	commons "github.com/flanksource/commons/context"
@@ -11,9 +13,11 @@ import (
 	"github.com/flanksource/duty/types"
 	"github.com/flanksource/kommons"
 	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/samber/lo"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
+	"golang.org/x/exp/maps"
 	"gorm.io/gorm"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -319,62 +323,184 @@ func (k Context) Wrap(ctx gocontext.Context) Context {
 		WithNamespace(k.GetNamespace())
 }
 
+func stringSliceToMap(s []string) map[string]string {
+	m := make(map[string]string)
+	for i := 0; i < len(s)-1; i += 2 {
+		m[s[i]] = s[i+1]
+	}
+	return m
+}
+
 type Histogram struct {
-	Context Context
-	Name    string
-	Labels  map[string]string
+	Context   Context
+	Name      string
+	Histogram *prometheus.HistogramVec
+	Labels    map[string]string
 }
 
-type Counter struct {
-	Context Context
-	Name    string
-	Labels  map[string]string
+var ctxHistograms = make(map[string]*prometheus.HistogramVec)
+
+func (k Context) Histogram(name string, labels ...string) Histogram {
+	labelMap := stringSliceToMap(labels)
+	labelKeys := maps.Keys(labelMap)
+	slices.Sort(labelKeys)
+	key := strings.Join(append(labelKeys, name), ".")
+
+	if histo, exists := ctxHistograms[key]; exists {
+		return Histogram{
+			Context:   k,
+			Histogram: histo,
+			Name:      name,
+			Labels:    labelMap,
+		}
+	}
+
+	histo := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name: name,
+	}, labelKeys)
+
+	if err := prometheus.Register(histo); err != nil {
+		k.Errorf("error registering histogram[%s/%v]: %v", name, labels, err)
+	}
+
+	ctxHistograms[key] = histo
+
+	return Histogram{
+		Context:   k,
+		Histogram: histo,
+		Name:      name,
+		Labels:    stringSliceToMap(labels),
+	}
+}
+
+func (h *Histogram) Label(k, v string) Histogram {
+	h.Labels[k] = v
+	return *h
 }
 
 func (h Histogram) Record(duration time.Duration) {
-	// noop
+	defer func() {
+		if r := recover(); r != nil {
+			h.Context.Errorf("error observe to histogram[%s/%v]: %v", h.Name, h.Labels, r)
+		}
+	}()
+
+	h.Histogram.With(prometheus.Labels(h.Labels)).Observe(float64(duration))
 }
 
 func (h Histogram) Since(s time.Time) {
 	h.Record(time.Since(s))
 }
 
-func (h Histogram) Label(k, v string) Histogram {
-	h.Labels[k] = v
-	return h
+type Counter struct {
+	Context Context
+	Name    string
+	Labels  map[string]string
+	Counter *prometheus.CounterVec
+}
+
+var ctxCounters = make(map[string]*prometheus.CounterVec)
+
+func (k Context) Counter(name string, labels ...string) Counter {
+	labelMap := stringSliceToMap(labels)
+	labelKeys := maps.Keys(labelMap)
+	slices.Sort(labelKeys)
+	key := strings.Join(append(labelKeys, name), ".")
+
+	if counter, exists := ctxCounters[key]; exists {
+		return Counter{
+			Context: k,
+			Counter: counter,
+			Name:    name,
+			Labels:  labelMap,
+		}
+	}
+
+	counter := prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: name,
+	}, labelKeys)
+
+	if err := prometheus.Register(counter); err != nil {
+		k.Errorf("error registering counter[%s/%v]: %v", name, labels, err)
+	}
+
+	ctxCounters[key] = counter
+	return Counter{
+		Context: k,
+		Counter: counter,
+		Name:    name,
+		Labels:  labelMap,
+	}
 }
 
 func (c Counter) Add(count int) {
 	c.AddFloat(float64(count))
 }
 
-func (c Counter) Label(k, v string) Counter {
+func (c Counter) AddFloat(count float64) {
+	defer func() {
+		if r := recover(); r != nil {
+			c.Context.Errorf("error adding to counter[%s/%v]: %v", c.Name, c.Labels, r)
+		}
+	}()
+
+	c.Counter.With(prometheus.Labels(c.Labels)).Add(count)
+}
+
+func (c *Counter) Label(k, v string) Counter {
 	c.Labels[k] = v
-	return c
+	return *c
 }
-func (c Counter) AddFloat(count float64) {
-	// noop
+
+type Gauge struct {
+	Context Context
+	Name    string
+	Labels  map[string]string
+	Gauge   *prometheus.GaugeVec
 }
 
-func stringSliceToMap(s []string) map[string]string {
-	m := make(map[string]string)
-	for i := 0; i < len(s)-1; i += 2 {
-		m[s[i]] = s[i+1]
+var ctxGauges = make(map[string]*prometheus.GaugeVec)
+
+func (k Context) Gauge(name string, labels ...string) Gauge {
+	labelMap := stringSliceToMap(labels)
+	labelKeys := maps.Keys(labelMap)
+	slices.Sort(labelKeys)
+	key := strings.Join(append(labelKeys, name), ".")
+
+	if gauge, exists := ctxGauges[key]; exists {
+		return Gauge{
+			Context: k,
+			Gauge:   gauge,
+			Name:    name,
+			Labels:  labelMap,
+		}
 	}
-	return m
-}
-func (k Context) Histogram(name string, labels ...string) Histogram {
-	return Histogram{
-		Context: k,
-		Name:    name,
-		Labels:  stringSliceToMap(labels),
+
+	gauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: name,
+	}, labelKeys)
+
+	if err := prometheus.Register(gauge); err != nil {
+		k.Errorf("error registering gauge[%s/%v]: %v", name, labels, err)
 	}
-}
 
-func (k Context) Counter(name string, labels ...string) Counter {
-	return Counter{
+	ctxGauges[key] = gauge
+	return Gauge{
 		Context: k,
+		Gauge:   gauge,
 		Name:    name,
-		Labels:  stringSliceToMap(labels),
+		Labels:  labelMap,
 	}
 }
+
+func (g Gauge) Set(count float64) {
+	g.Gauge.With(prometheus.Labels(g.Labels)).Set(count)
+}
+
+func (g Gauge) Add(count float64) {
+	g.Gauge.With(prometheus.Labels(g.Labels)).Add(count)
+}
+
+func (g Gauge) Sub(count float64) {
+	g.Gauge.With(prometheus.Labels(g.Labels)).Sub(count)
+}

go.mod

@@ -1,6 +1,6 @@
 module github.com/flanksource/duty
 
-go 1.20
+go 1.22.0
 
 require (
 	ariga.io/atlas v0.14.2
@@ -31,6 +31,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.13.2
 	github.com/onsi/gomega v1.29.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/prometheus/client_golang v1.14.0
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rodaine/table v1.1.0
 	github.com/samber/lo v1.39.0
@@ -46,6 +47,7 @@ require (
 	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0
 	go.opentelemetry.io/otel/sdk v1.22.0
 	go.opentelemetry.io/otel/trace v1.22.0
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
 	golang.org/x/sync v0.6.0
 	google.golang.org/grpc v1.60.1
 	gorm.io/driver/postgres v1.5.3
@@ -146,7 +148,6 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
@@ -181,7 +182,6 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
-	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
 	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/oauth2 v0.15.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect