Skip to content

Commit

Permalink
replace go-gin-prometheus for better control over metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
DoctorVin committed Jun 10, 2024
1 parent 38165ca commit 71fc2ba
Show file tree
Hide file tree
Showing 6 changed files with 76 additions and 17 deletions.
4 changes: 1 addition & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ require (
github.com/lib/pq v1.10.9
github.com/pkg/errors v0.9.1
github.com/pressly/goose/v3 v3.15.0 // indirect
github.com/prometheus/client_golang v1.19.0 // indirect
github.com/prometheus/client_golang v1.19.0
github.com/spf13/cobra v1.8.0
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.18.2
Expand All @@ -24,7 +24,6 @@ require (
github.com/volatiletech/randomize v0.0.1
github.com/volatiletech/sqlboiler/v4 v4.16.2
github.com/volatiletech/strmangle v0.0.6
github.com/zsais/go-gin-prometheus v0.1.0
go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0
go.opentelemetry.io/otel v1.25.0
go.opentelemetry.io/otel/exporters/jaeger v1.16.0 // indirect
Expand Down Expand Up @@ -99,7 +98,6 @@ require (
github.com/prometheus/procfs v0.12.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
Expand Down
5 changes: 0 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -617,8 +617,6 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
Expand Down Expand Up @@ -690,8 +688,6 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
github.com/zsais/go-gin-prometheus v0.1.0 h1:bkLv1XCdzqVgQ36ScgRi09MA2UC1t3tAB6nsfErsGO4=
github.com/zsais/go-gin-prometheus v0.1.0/go.mod h1:Slirjzuz8uM8Cw0jmPNqbneoqcUtY2GGjn2bEd4NRLY=
go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU=
Expand Down Expand Up @@ -997,7 +993,6 @@ golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
17 changes: 8 additions & 9 deletions internal/httpsrv/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
ginzap "github.com/gin-contrib/zap"
"github.com/gin-gonic/gin"
"github.com/jmoiron/sqlx"
ginprometheus "github.com/zsais/go-gin-prometheus"
"go.hollow.sh/toolbox/events"
"go.hollow.sh/toolbox/ginauth"
"go.hollow.sh/toolbox/ginjwt"
Expand All @@ -20,6 +19,7 @@ import (
"go.uber.org/zap/zapcore"
"gocloud.dev/secrets"

"github.com/metal-toolbox/fleetdb/internal/metrics"
fleetdbapi "github.com/metal-toolbox/fleetdb/pkg/api/v1"
)

Expand Down Expand Up @@ -66,7 +66,11 @@ func (s *Server) setup() *gin.Engine {
MaxAge: corsMaxAge,
}))

p := ginprometheus.NewPrometheus("gin")
r.Use(func(c *gin.Context) {
start := time.Now()
c.Next() // call the rest of the handler chain
metrics.APICallEpilog(start, c.FullPath(), c.Writer.Status())
})

v1Rtr := fleetdbapi.Router{
DB: s.DB,
Expand All @@ -76,18 +80,13 @@ func (s *Server) setup() *gin.Engine {
EventStream: s.EventStream,
}

// Remove any params from the URL string to keep the number of labels down
p.ReqCntURLLabelMappingFn = func(c *gin.Context) string {
return c.FullPath()
}

p.Use(r)

r.Use(ginzap.GinzapWithConfig(s.Logger.With(zap.String("component", "httpsrv")), &ginzap.Config{
TimeFormat: time.RFC3339,
UTC: true,
Context: ginzap.Fn(func(c *gin.Context) []zapcore.Field {
return []zapcore.Field{
zap.String("path", c.Request.URL.Path),
zap.String("query", c.Request.URL.RawQuery),
zap.String("jwt_subject", ginjwt.GetSubject(c)),
zap.String("jwt_user", ginjwt.GetUser(c)),
}
Expand Down
3 changes: 3 additions & 0 deletions internal/inventory/device_components.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/volatiletech/sqlboiler/v4/queries/qm"

"github.com/metal-toolbox/fleetdb/internal/dbtools"
"github.com/metal-toolbox/fleetdb/internal/metrics"
"github.com/metal-toolbox/fleetdb/internal/models"
)

Expand Down Expand Up @@ -129,6 +130,7 @@ func retrieveComponentAttributes(ctx context.Context, exec boil.ContextExecutor,
return nil, nil
case nil:
default:
metrics.DBError("fetch component attributes")
return nil, err
}

Expand All @@ -151,6 +153,7 @@ func retrieveVersionedAttribute(ctx context.Context, exec boil.ContextExecutor,

fwr, err := models.VersionedAttributes(mods...).One(ctx, exec)
if err != nil {
metrics.DBError("fetch versioned attributes")
return nil, err
}
return fwr.Data, nil
Expand Down
3 changes: 3 additions & 0 deletions internal/inventory/device_view.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/volatiletech/sqlboiler/v4/queries/qm"

"github.com/metal-toolbox/fleetdb/internal/dbtools"
"github.com/metal-toolbox/fleetdb/internal/metrics"
"github.com/metal-toolbox/fleetdb/internal/models"
)

Expand Down Expand Up @@ -153,6 +154,7 @@ func (dv *DeviceView) FromDatastore(ctx context.Context, exec boil.ContextExecut
switch err {
case nil:
default:
metrics.DBError("fetching attributes")
return errors.Wrap(err, "fetching attributes")
}

Expand Down Expand Up @@ -190,6 +192,7 @@ func (dv *DeviceView) FromDatastore(ctx context.Context, exec boil.ContextExecut
case sql.ErrNoRows:
// just skip it, status is optional
default:
metrics.DBError("fetch status VA")
return errors.Wrap(err, "fetching versioned attibutes")
}

Expand Down
61 changes: 61 additions & 0 deletions internal/metrics/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package metrics

import (
"strconv"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)

var (
apiLatencySec *prometheus.HistogramVec
dbErrorCount *prometheus.CounterVec
)

const (
appName = "fleetdb"
apiSub = "api"
dbSub = "database"
)

func init() {
apiLatencySec = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: appName,
Subsystem: apiSub,
Name: "latency_seconds",
Help: "api latency measurements in seconds",
// XXX: These buckets will likely need some tuning
Buckets: []float64{0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0},
},
[]string{
"code",
"endpoint",
},
)

dbErrorCount = promauto.NewCounterVec(
prometheus.CounterOpts{
Namespace: appName,
Subsystem: dbSub,
Name: "error_count",
Help: "total count of database errors",
},
[]string{
"operation",
},
)
}

// APICallEpilog observes the response and elapsed time of a call to a given endpoint
func APICallEpilog(start time.Time, endpoint string, responseCode int) {
code := strconv.Itoa(responseCode)
elapsed := time.Since(start).Seconds()
apiLatencySec.WithLabelValues(endpoint, code).Observe(elapsed)
}

// DBError observes errors arising from an attempt to read or write data to the remote database
func DBError(op string) {
dbErrorCount.WithLabelValues(op).Inc()
}

0 comments on commit 71fc2ba

Please sign in to comment.