Add prometheus metric (#22)

* 🌱 Add prometheus metric * Update pkg/faultdetector/metrics.go Co-authored-by: Himanshu Nagda <[email protected]> * ♻️ Create RegisterHandler on HTTP server - Implement RegisterHandler - Refector app to registerHandler for prometheus - Move FaultDetectorMetric into FaultDetector file --------- Co-authored-by: Himanshu Nagda <[email protected]>
LiskHQ · Feb 2, 2024 · d20b8e2 · d20b8e2
1 parent 8092a20
commit d20b8e2
Show file tree

Hide file tree

Showing 5 changed files with 88 additions and 2 deletions.
diff --git a/cmd/main.go b/cmd/main.go
@@ -12,11 +12,15 @@ import (
 	"strings"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/LiskHQ/op-fault-detector/pkg/api"
 	"github.com/LiskHQ/op-fault-detector/pkg/config"
 	"github.com/LiskHQ/op-fault-detector/pkg/faultdetector"
 	"github.com/LiskHQ/op-fault-detector/pkg/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/viper"
 )
 
@@ -35,6 +39,12 @@ type App struct {
 func NewApp(logger log.Logger) (*App, error) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
+	reg := prometheus.NewRegistry()
+	// Adding Go process related metric
+	reg.MustRegister(
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+		collectors.NewGoCollector(),
+	)
 
 	configFilepath := flag.String("config", "./config.yaml", "Path to the config file")
 	flag.Parse()
@@ -54,6 +64,7 @@ func NewApp(logger log.Logger) (*App, error) {
 		errorChan,
 		&wg,
 		config.FaultDetectorConfig,
+		reg,
 	)
 	if err != nil {
 		logger.Errorf("Failed to create fault detector service.")
@@ -63,6 +74,9 @@ func NewApp(logger log.Logger) (*App, error) {
 	// Start API Server
 	apiServer := api.NewHTTPServer(ctx, logger, &wg, config, errorChan)
 
+	// Register metrics endpoint on the apiServer
+	apiServer.RegisterHandler("GET", "/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg, ProcessStartTime: time.Now()}))
+
 	return &App{
 		ctx:           ctx,
 		logger:        logger,

diff --git a/go.mod b/go.mod
@@ -15,9 +15,11 @@ require (
 
 require (
 	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.10.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
 	github.com/bytedance/sonic v1.10.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
 	github.com/chenzhuoyu/iasm v0.9.1 // indirect
 	github.com/consensys/bavard v0.1.13 // indirect
@@ -44,12 +46,17 @@ require (
 	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_golang v1.18.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect

diff --git a/go.sum b/go.sum
@@ -159,6 +159,8 @@ github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWV
 github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/pointerstructure v1.2.1 h1:ZhBBeX8tSlRpu/FFhXH4RC4OJzFlqsQhoHZAz4x7TIw=
@@ -182,12 +184,20 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
 github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
+github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
 github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
 github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
 github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
 github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
+github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
 github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
 github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw=
 github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=

diff --git a/pkg/api/server.go b/pkg/api/server.go
@@ -18,6 +18,7 @@ import (
 // HTTPServer embeds the http.Server along with the various other properties.
 type HTTPServer struct {
 	server    *http.Server
+	router    *gin.Engine
 	ctx       context.Context
 	logger    log.Logger
 	wg        *sync.WaitGroup
@@ -45,6 +46,10 @@ func (w *HTTPServer) Stop() error {
 	return err
 }
 
+func (w *HTTPServer) RegisterHandler(httpMethod string, relativePath string, h http.Handler) {
+	w.router.Handle(httpMethod, relativePath, gin.WrapH(h))
+}
+
 func getGinModeFromSysLogLevel(sysLogLevel string) string {
 	ginMode := gin.DebugMode // Default mode
 
@@ -66,6 +71,7 @@ func NewHTTPServer(ctx context.Context, logger log.Logger, wg *sync.WaitGroup, c
 
 	// Register handlers for routes without any base path
 	logger.Debug("Registering handlers for non-versioned endpoints.")
+
 	routes.RegisterHandlers(logger, router)
 
 	// Register handlers for routes following the base path
@@ -84,6 +90,7 @@ func NewHTTPServer(ctx context.Context, logger log.Logger, wg *sync.WaitGroup, c
 			Handler:           router,
 			ReadHeaderTimeout: 10 * time.Second,
 		},
+		router,
 		ctx,
 		logger,
 		wg,

diff --git a/pkg/faultdetector/faultdetector.go b/pkg/faultdetector/faultdetector.go
@@ -9,6 +9,7 @@ import (
 	"github.com/LiskHQ/op-fault-detector/pkg/chain"
 	"github.com/LiskHQ/op-fault-detector/pkg/config"
 	"github.com/LiskHQ/op-fault-detector/pkg/log"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 const (
@@ -21,6 +22,7 @@ type FaultDetector struct {
 	logger                 log.Logger
 	errorChan              chan error
 	wg                     *sync.WaitGroup
+	metrics                *faultDetectorMetrics
 	l1RpcApi               *chain.ChainAPIClient
 	l2RpcApi               *chain.ChainAPIClient
 	oracleContractAccessor *chain.OracleAccessor
@@ -31,8 +33,38 @@ type FaultDetector struct {
 	quitTickerChan         chan struct{}
 }
 
+type faultDetectorMetrics struct {
+	highestOutputIndex   prometheus.Gauge
+	stateMismatch        prometheus.Gauge
+	apiConnectionFailure prometheus.Gauge
+}
+
+// NewFaultDetectorMetrics returns [FaultDetectorMetrics] with initialized metrics and registering to prometheus registry.
+func newFaultDetectorMetrics(reg prometheus.Registerer) *faultDetectorMetrics {
+	m := &faultDetectorMetrics{
+		highestOutputIndex: prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Name: "fault_detector_highest_output_index",
+				Help: "The highest current output index",
+			}),
+		stateMismatch: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "fault_detector_is_state_mismatch",
+			Help: "0 when state is matched, 1 when mismatch",
+		}),
+		apiConnectionFailure: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "fault_detector_api_connection_failure",
+			Help: "Number of times API call failed",
+		}),
+	}
+	reg.MustRegister(m.highestOutputIndex)
+	reg.MustRegister(m.stateMismatch)
+	reg.MustRegister(m.apiConnectionFailure)
+
+	return m
+}
+
 // NewFaultDetector will return [FaultDetector] with the initialized providers and configuration.
-func NewFaultDetector(ctx context.Context, logger log.Logger, errorChan chan error, wg *sync.WaitGroup, faultDetectorConfig *config.FaultDetectorConfig) (*FaultDetector, error) {
+func NewFaultDetector(ctx context.Context, logger log.Logger, errorChan chan error, wg *sync.WaitGroup, faultDetectorConfig *config.FaultDetectorConfig, metricRegistry *prometheus.Registry) (*FaultDetector, error) {
 	// Initialize API Providers
 	l1RpcApi, err := chain.GetAPIClient(ctx, faultDetectorConfig.L1RPCEndpoint, logger)
 	if err != nil {
@@ -65,8 +97,20 @@ func NewFaultDetector(ctx context.Context, logger log.Logger, errorChan chan err
 		return nil, err
 	}
 
+	finalizedPeriodSeconds, err := oracleContractAccessor.FinalizationPeriodSeconds()
+	if err != nil {
+		logger.Errorf("Failed to query `FinalizationPeriodSeconds` from Oracle contract accessor, error: %w", err)
+		return nil, err
+	}
+
+	metrics := newFaultDetectorMetrics(metricRegistry)
 	// TODO: Calculate from findFirstUnfinalizedOutputIndex(context, OracleContractAccessor, L1Provider, faultProofWindow, logger)
 
+	// Set after findFirstUnfinalizedOutputIndex
+	metrics.highestOutputIndex.Set(1)
+	// Initially set state mismatch to 0
+	metrics.stateMismatch.Set(0)
+
 	faultDetector := &FaultDetector{
 		ctx:                    ctx,
 		logger:                 logger,
@@ -75,9 +119,10 @@ func NewFaultDetector(ctx context.Context, logger log.Logger, errorChan chan err
 		l1RpcApi:               l1RpcApi,
 		l2RpcApi:               l2RpcApi,
 		oracleContractAccessor: oracleContractAccessor,
-		faultProofWindow:       uint64(2), // TODO
+		faultProofWindow:       finalizedPeriodSeconds.Uint64(),
 		currentOutputIndex:     uint64(2), // TODO
 		diverged:               false,
+		metrics:                metrics,
 	}
 
 	return faultDetector, nil
@@ -109,6 +154,9 @@ func (fd *FaultDetector) Stop() {
 
 // TODO: Implement checkFault to check for faults
 func (fd *FaultDetector) checkFault() {
+	// TODO: Increment or set in different scenarios
+	fd.metrics.highestOutputIndex.Inc()
+	fd.metrics.stateMismatch.Dec()
 	// TODO: The below log need to be removed/updated after full implementation
 	fd.logger.Infof("Connected to L1 and L2 chains, the currentOutputIndex is %d", fd.currentOutputIndex)
 }