Add exemplars service

docker-compose.exemplars.yml

@@ -0,0 +1,17 @@
+services:
+  exemplars:
+    image: golang:latest
+    container_name: exemplars
+    working_dir: /home/go/app
+    volumes:
+      - ./exemplars:/home/go/app
+    command: "go run main.go"
+    ports:
+      - 8081:8081
+    environment:
+      - HOST=grafana-gateway
+      - PORT=3100
+    networks:
+      grafana:
+    depends_on:
+      - gateway

exemplars/go.mod

@@ -0,0 +1,17 @@
+module exemplars
+
+go 1.22.1
+
+require github.com/prometheus/client_golang v1.19.0
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/gorilla/websocket v1.5.1 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
+)

exemplars/go.sum

@@ -0,0 +1,24 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
+github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=

exemplars/main.go

@@ -0,0 +1,125 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/gorilla/websocket"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+type Stream struct {
+	Stream struct {
+		Traces string `json:"traces"`
+	} `json:"stream"`
+	Values [][]string `json:"values"`
+}
+
+type Streams struct {
+	Streams []Stream `json:"streams"`
+}
+
+func connectWebsocket(urlString string) *websocket.Conn {
+	log.Print("connecting to websocket")
+
+	c, _, err := websocket.DefaultDialer.Dial(urlString, http.Header{
+		"X-Scope-OrgId": []string{"tenant1"},
+	})
+	if err != nil {
+		log.Fatal("dial:", err)
+	}
+	log.Print("connected to websocket")
+
+	return c
+}
+
+func main() {
+	requestDurations := prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name:    "prometheus_exemplar",
+		Help:    "A histogram of the HTTP request durations in seconds.",
+		Buckets: prometheus.ExponentialBuckets(0.1, 1.5, 5),
+	})
+	// Create non-global registry.
+	registry := prometheus.NewRegistry()
+
+	// Add go runtime metrics and process collectors.
+	registry.MustRegister(
+		collectors.NewGoCollector(),
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+		requestDurations,
+	)
+
+	host, exists := os.LookupEnv("HOST")
+	if !exists || host == "" {
+		log.Fatal("No HOST defined, where am I pulling trace ID's from?")
+	}
+	log.Printf("successfully retrieved host %s", host)
+
+	port, exists := os.LookupEnv("PORT")
+	if !exists || port == "" {
+		log.Fatal("No PORT defined, where am I pulling trace ID's from?")
+	}
+	log.Printf("successfully retrieved port %s", port)
+
+	endpoint := "loki/api/v1/tail"
+	urlString := fmt.Sprintf("ws://%s:%s/%s?query=%%7Btraces%%3D%%7E%%22.%%2B%%22%%7D&limit=1000", host, port, endpoint)
+	log.Printf("url: %s", urlString)
+
+	time.Sleep(15 * time.Second)
+
+	c := connectWebsocket(urlString)
+	defer c.Close()
+
+	done := make(chan struct{})
+
+	go func() {
+		defer close(done)
+		for {
+			streams := Streams{}
+			err := c.ReadJSON(&streams)
+			if err != nil {
+				log.Println("read:", err)
+				if strings.Contains(err.Error(), "reached tail max duration limit") {
+					c.Close()
+					c = connectWebsocket(urlString)
+				} else {
+					return
+				}
+			}
+			for _, trace := range streams.Streams {
+				log.Print("parsing trace from stream")
+				splitString := strings.Split(trace.Values[0][1], " ")
+				traceID := ""
+				for _, kv := range splitString {
+					split := strings.Split(kv, "=")
+					if len(split) != 0 && split[0] == "tid" {
+						traceID = split[1]
+						log.Println("traceID:", traceID)
+					}
+				}
+				now := time.Now()
+				requestDurations.(prometheus.ExemplarObserver).ObserveWithExemplar(
+					time.Since(now).Seconds(), prometheus.Labels{"traceID": traceID},
+				)
+				log.Printf("pushed metric for traceID: %s", traceID)
+			}
+		}
+	}()
+
+	// Expose /metrics HTTP endpoint using the created custom registry.
+	http.Handle(
+		"/metrics", promhttp.HandlerFor(
+			registry,
+			promhttp.HandlerOpts{
+				EnableOpenMetrics: true,
+			}),
+	)
+	// To test: curl -H 'Accept: application/openmetrics-text' localhost:8081/metrics
+	log.Fatalln(http.ListenAndServe(":8081", nil))
+}