Merge remote-tracking branch 'origin' into 7-implement-start-fault-de…

…tector-method

cmd/main.go

@@ -12,11 +12,15 @@ import (
 	"strings"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/LiskHQ/op-fault-detector/pkg/api"
 	"github.com/LiskHQ/op-fault-detector/pkg/config"
 	"github.com/LiskHQ/op-fault-detector/pkg/faultdetector"
 	"github.com/LiskHQ/op-fault-detector/pkg/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/viper"
 )
 
@@ -33,6 +37,13 @@ type App struct {
 
 // NewApp returns [App] with all the initialized services and variables.
 func NewApp(ctx context.Context, logger log.Logger) (*App, error) {
+	reg := prometheus.NewRegistry()
+	// Adding Go process related metric
+	reg.MustRegister(
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+		collectors.NewGoCollector(),
+	)
+
 	configFilepath := flag.String("config", "./config.yaml", "Path to the config file")
 	flag.Parse()
 	config, err := getAppConfig(logger, *configFilepath)
@@ -51,6 +62,7 @@ func NewApp(ctx context.Context, logger log.Logger) (*App, error) {
 		errorChan,
 		&wg,
 		config.FaultDetectorConfig,
+		reg,
 	)
 
 	if err != nil {
@@ -61,6 +73,9 @@ func NewApp(ctx context.Context, logger log.Logger) (*App, error) {
 	// Start API Server
 	apiServer := api.NewHTTPServer(ctx, logger, &wg, config, errorChan)
 
+	// Register metrics endpoint on the apiServer
+	apiServer.RegisterHandler("GET", "/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg, ProcessStartTime: time.Now()}))
+
 	return &App{
 		ctx:           ctx,
 		logger:        logger,

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/ethereum/go-ethereum v1.13.10
 	github.com/gin-gonic/gin v1.9.1
 	github.com/magiconair/properties v1.8.7
+	github.com/prometheus/client_golang v1.18.0
 	github.com/spf13/viper v1.18.2
 	github.com/stretchr/testify v1.8.4
 	go.uber.org/multierr v1.11.0
@@ -50,7 +51,6 @@ require (
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/google/uuid v1.5.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
@@ -64,7 +64,7 @@ require (
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -73,10 +73,9 @@ require (
 	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.17.0 // indirect
-	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
-	github.com/prometheus/common v0.44.0 // indirect
-	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rivo/uniseg v0.4.3 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect

go.sum

@@ -127,8 +127,6 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk=
 github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
@@ -192,8 +190,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
 github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
-github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/pointerstructure v1.2.1 h1:ZhBBeX8tSlRpu/FFhXH4RC4OJzFlqsQhoHZAz4x7TIw=
@@ -233,14 +231,14 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
-github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
-github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
-github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
-github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
-github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
-github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
-github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
+github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
+github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
+github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw=
 github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -338,7 +336,6 @@ golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
 golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

pkg/api/server.go

@@ -18,6 +18,7 @@ import (
 // HTTPServer embeds the http.Server along with the various other properties.
 type HTTPServer struct {
 	server    *http.Server
+	router    *gin.Engine
 	ctx       context.Context
 	logger    log.Logger
 	wg        *sync.WaitGroup
@@ -45,6 +46,10 @@ func (w *HTTPServer) Stop() error {
 	return err
 }
 
+func (w *HTTPServer) RegisterHandler(httpMethod string, relativePath string, h http.Handler) {
+	w.router.Handle(httpMethod, relativePath, gin.WrapH(h))
+}
+
 func getGinModeFromSysLogLevel(sysLogLevel string) string {
 	ginMode := gin.DebugMode // Default mode
 
@@ -66,6 +71,7 @@ func NewHTTPServer(ctx context.Context, logger log.Logger, wg *sync.WaitGroup, c
 
 	// Register handlers for routes without any base path
 	logger.Debug("Registering handlers for non-versioned endpoints.")
+
 	routes.RegisterHandlers(logger, router)
 
 	// Register handlers for routes following the base path
@@ -84,6 +90,7 @@ func NewHTTPServer(ctx context.Context, logger log.Logger, wg *sync.WaitGroup, c
 			Handler:           router,
 			ReadHeaderTimeout: 10 * time.Second,
 		},
+		router,
 		ctx,
 		logger,
 		wg,

pkg/faultdetector/faultdetector.go

@@ -11,6 +11,7 @@ import (
 	"github.com/LiskHQ/op-fault-detector/pkg/encoding"
 	"github.com/LiskHQ/op-fault-detector/pkg/log"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 const (
@@ -24,6 +25,7 @@ type FaultDetector struct {
 	logger                 log.Logger
 	errorChan              chan error
 	wg                     *sync.WaitGroup
+	metrics                *faultDetectorMetrics
 	l1RpcApi               *chain.ChainAPIClient
 	l2RpcApi               *chain.ChainAPIClient
 	oracleContractAccessor *chain.OracleAccessor
@@ -34,8 +36,38 @@ type FaultDetector struct {
 	quitTickerChan         chan struct{}
 }
 
+type faultDetectorMetrics struct {
+	highestOutputIndex   prometheus.Gauge
+	stateMismatch        prometheus.Gauge
+	apiConnectionFailure prometheus.Gauge
+}
+
+// NewFaultDetectorMetrics returns [FaultDetectorMetrics] with initialized metrics and registering to prometheus registry.
+func newFaultDetectorMetrics(reg prometheus.Registerer) *faultDetectorMetrics {
+	m := &faultDetectorMetrics{
+		highestOutputIndex: prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Name: "fault_detector_highest_output_index",
+				Help: "The highest current output index",
+			}),
+		stateMismatch: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "fault_detector_is_state_mismatch",
+			Help: "0 when state is matched, 1 when mismatch",
+		}),
+		apiConnectionFailure: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "fault_detector_api_connection_failure",
+			Help: "Number of times API call failed",
+		}),
+	}
+	reg.MustRegister(m.highestOutputIndex)
+	reg.MustRegister(m.stateMismatch)
+	reg.MustRegister(m.apiConnectionFailure)
+
+	return m
+}
+
 // NewFaultDetector will return [FaultDetector] with the initialized providers and configuration.
-func NewFaultDetector(ctx context.Context, logger log.Logger, errorChan chan error, wg *sync.WaitGroup, faultDetectorConfig *config.FaultDetectorConfig) (*FaultDetector, error) {
+func NewFaultDetector(ctx context.Context, logger log.Logger, errorChan chan error, wg *sync.WaitGroup, faultDetectorConfig *config.FaultDetectorConfig, metricRegistry *prometheus.Registry) (*FaultDetector, error) {
 	// Initialize API Providers
 	l1RpcApi, err := chain.GetAPIClient(ctx, faultDetectorConfig.L1RPCEndpoint, logger)
 	if err != nil {
@@ -69,20 +101,21 @@ func NewFaultDetector(ctx context.Context, logger log.Logger, errorChan chan err
 		return nil, err
 	}
 
-	faultProofWindow, err := oracleContractAccessor.FinalizationPeriodSeconds()
+	finalizedPeriodSeconds, err := oracleContractAccessor.FinalizationPeriodSeconds()
 	if err != nil {
-		logger.Errorf("Failed to query FinalizationPeriodSecond", err)
+		logger.Errorf("Failed to query `FinalizationPeriodSeconds` from Oracle contract accessor, error: %w", err)
 		return nil, err
 	}
-	logger.Infof("Fault proof window is set to %d.", faultProofWindow)
+
+	logger.Infof("Fault proof window is set to %d.", finalizedPeriodSeconds)
 
 	var currentOutputIndex uint64
 	if int64(faultDetectorConfig.Startbatchindex) == -1 {
 		logger.Infof("Finding appropriate starting unfinalized batch....")
 		// firstUnfinalized, _ := FindFirstUnfinalizedOutputIndex(
 		// 	ctx,
 		// 	logger,
-		// 	encoding.MustConvertBigIntToUint64(faultProofWindow),
+		// 	encoding.MustConvertBigIntToUint64(finalizedPeriodSeconds),
 		// 	oracleContractAccessor,
 		// 	l2RpcApi,
 		// )
@@ -102,6 +135,10 @@ func NewFaultDetector(ctx context.Context, logger log.Logger, errorChan chan err
 	}
 	logger.Infof("Starting unfinalized batch index is set to %d.", currentOutputIndex)
 
+	metrics := newFaultDetectorMetrics(metricRegistry)
+	// Initially set state mismatch to 0
+	metrics.stateMismatch.Set(0)
+
 	faultDetector := &FaultDetector{
 		ctx:                    ctx,
 		logger:                 logger,
@@ -110,9 +147,10 @@ func NewFaultDetector(ctx context.Context, logger log.Logger, errorChan chan err
 		l1RpcApi:               l1RpcApi,
 		l2RpcApi:               l2RpcApi,
 		oracleContractAccessor: oracleContractAccessor,
-		faultProofWindow:       encoding.MustConvertBigIntToUint64(faultProofWindow),
+		faultProofWindow:       finalizedPeriodSeconds.Uint64(),
 		currentOutputIndex:     currentOutputIndex,
 		diverged:               false,
+		metrics:                metrics,
 	}
 
 	return faultDetector, nil
@@ -211,6 +249,10 @@ func (fd *FaultDetector) checkFault() {
 		return
 	}
 
+	// TODO: Increment or set in different scenarios
+	fd.metrics.highestOutputIndex.Inc()
+	fd.metrics.stateMismatch.Set(0)
+
 	// Time taken to execute each batch in milliseconds.
 	elapsedTime := time.Since(startTime).Milliseconds()
 	fd.logger.Infof("Successfully checked current batch with index %d --> ok, time taken %dms.", fd.currentOutputIndex, elapsedTime)