Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
@YukiNodereal
YukiNodereal committed Aug 12, 2024
1 parent 9eb3d32 commit 780b2b6
Show file tree
Hide file tree
Showing 2 changed files with 107 additions and 165 deletions.
209 changes: 98 additions & 111 deletions checks/dbshard.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,162 +4,149 @@ import (
"fmt"
"github.com/bnb-chain/gnfd-qa-test-monitor/abci"
"github.com/bnb-chain/gnfd-qa-test-monitor/utils"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/tidwall/gjson"
"strings"
)

var MainNetSpHost = []string{
"greenfield-sp.bnbchain.org",
"greenfield-sp.defibit.io",
"greenfield-sp.ninicoin.io",
"greenfield-sp.nariox.org",
"greenfield-sp.lumibot.org",
"greenfield-sp.voltbot.io",
"greenfield-sp.nodereal.io",
}

var TestNetSpHost = []string{
"gnfd-testnet-sp1.bnbchain.org",
"gnfd-testnet-sp2.bnbchain.org",
"gnfd-testnet-sp3.bnbchain.org",
"gnfd-testnet-sp4.bnbchain.org",
"gnfd-testnet-sp1.nodereal.io",
"gnfd-testnet-sp2.nodereal.io",
"gnfd-testnet-sp3.nodereal.io",
}

type Code uint32

const (
OK Code = iota
GetBlockHeightErr
GetObjectTotalCountErr
GetObjectSealCountErr
CheckObjectTotalCountErr
CheckObjectSealCountErr
)

func CheckDbShard(checkHeight int64, spHost string) Code {
xmlResult, err := abci.BsDBInfoBlockHeight(spHost, checkHeight)
const (
TestNetRpc = "https://gnfd-testnet-fullnode-tendermint-us.bnbchain.org:443"
MainNetRpc = "https://greenfield-chain.bnbchain.org:443"
)

var (
MainNetSpHosts = []string{
"greenfield-sp.bnbchain.org",
"greenfield-sp.defibit.io",
"greenfield-sp.ninicoin.io",
"greenfield-sp.nariox.org",
"greenfield-sp.lumibot.org",
"greenfield-sp.voltbot.io",
"greenfield-sp.nodereal.io",
}

TestNetSpHosts = []string{
"gnfd-testnet-sp1.bnbchain.org",
"gnfd-testnet-sp2.bnbchain.org",
"gnfd-testnet-sp3.bnbchain.org",
"gnfd-testnet-sp4.bnbchain.org",
"gnfd-testnet-sp1.nodereal.io",
"gnfd-testnet-sp2.nodereal.io",
"gnfd-testnet-sp3.nodereal.io",
}
)

func CheckDbShard(checkEnv, checkRpc string, checkSpHosts []string) {
checkBlockMetrics := promauto.NewGauge(prometheus.GaugeOpts{Name: fmt.Sprintf("%v_sp_db_shard_check_block_height", checkEnv)})
lastChainHeight, err := abci.LastBlockHeight(checkRpc)
calcHeight := lastChainHeight / 3600 * 3600
checkBlockMetrics.Set(float64(calcHeight))
if err != nil {
checkBlockMetrics.Set(float64(GetBlockHeightErr))
return
}

checkSpErrCodeMetrics := make([]prometheus.Gauge, len(checkSpHosts))
objCountArr := make([][]gjson.Result, len(checkSpHosts))
sealObjCountArr := make([][]gjson.Result, len(checkSpHosts))
isErr := false
for i, spHost := range checkSpHosts {
metricsSpHost := strings.Replace(spHost, "-", "_", -1)
metricsSpHost = strings.Replace(metricsSpHost, ".", "_", -1)
checkSpErrCodeMetrics[i] = promauto.NewGauge(prometheus.GaugeOpts{Name: fmt.Sprintf("%v_sp_db_shard_error_code_%v", checkEnv, metricsSpHost)})

objCount, sealCount, errCode := getSpDbData(spHost, calcHeight)
if errCode != OK {
checkSpErrCodeMetrics[i].Set(float64(errCode))
isErr = true
}
objCountArr[i] = objCount
sealObjCountArr[i] = sealCount
}

if isErr {
return
}

spIndex, errCode := checkDbData(objCountArr, sealObjCountArr)
if errCode != OK {
checkSpErrCodeMetrics[spIndex].Set(float64(errCode))
return
}

for _, metric := range checkSpErrCodeMetrics {
metric.Set(float64(OK))
}
}

func getSpDbData(spHost string, height int64) (objCount, objSealCount []gjson.Result, errCode Code) {
xmlResult, err := abci.BsDBInfoBlockHeight(spHost, height)
if err != nil {
fmt.Println(err)
return nil, nil, GetBlockHeightErr
}

var resObjectCount, resObjectSealCount []gjson.Result
objectResString := utils.GetXmlPath(xmlResult, "GfSpGetBsDBInfoResponse/ObjectTotalCount")
if objectResString == "" {
fmt.Printf("sp: %v, ObjectTotalCount error\n", spHost)
return GetObjectTotalCountErr
return nil, nil, GetObjectTotalCountErr
} else {
objectTotalCount := gjson.Parse(objectResString).Array()
resObjectCount = objectTotalCount
objCount = objectTotalCount
}

objectSealResString := utils.GetXmlPath(xmlResult, "GfSpGetBsDBInfoResponse/ObjectSealCount")
if objectSealResString == "" {
fmt.Printf("sp: %v, ObjectSealCount error\n", spHost)
return GetObjectSealCountErr
return nil, nil, GetObjectSealCountErr
} else {
ObjectSealCount := gjson.Parse(objectSealResString).Array()
resObjectSealCount = ObjectSealCount
objSealCount = ObjectSealCount
}

// check sp object count
return objCount, objSealCount, OK
}

func checkDbData(spObjCounts, spObjSealCounts [][]gjson.Result) (spIndex int, errCode Code) {
for i := 0; i < 64; i++ {
sumObject := int64(0)
sumSp1 := int64(0)
for _, objectCount := range resObjectCount {
sumObject = sumObject + objectCount.Int()
for _, objectCount := range spObjCounts {
sumObject = sumObject + objectCount[i].Int()
sumSp1++
}
sumSealedObject := int64(0)
sumSp2 := int64(0)
for _, sealObjectCount := range resObjectSealCount {
sumSealedObject = sumSealedObject + sealObjectCount.Int()
for _, sealObjectCount := range spObjSealCounts {
sumSealedObject = sumSealedObject + sealObjectCount[i].Int()
sumSp2++
}

objectAverage := sumObject / sumSp1
sealObjectAverage := sumSealedObject / sumSp2
for _, eachValue := range resObjectCount {
if objectAverage != eachValue.Int() {
return CheckObjectTotalCountErr
for spIndex, eachValue := range spObjCounts {
if objectAverage != eachValue[i].Int() {
return spIndex, CheckObjectTotalCountErr
}
}
for _, eachValue := range resObjectSealCount {
if sealObjectAverage != eachValue.Int() {
return CheckObjectSealCountErr

sealObjectAverage := sumSealedObject / sumSp2
for _, eachValue := range spObjSealCounts {
if sealObjectAverage != eachValue[i].Int() {
return spIndex, CheckObjectSealCountErr
}
}
}

return OK
return 0, OK
}

//func CheckSpDbObjectNumsPeriod1(spHostArray []string) (block int64, errCode Code) {
// chainHeight, err := abci.LastBlockHeight()
// if err != nil {
// fmt.Println(err)
// }
// fmt.Printf("height: %d\n", chainHeight)
// calcHeight := chainHeight / 3600 * 3600
// fmt.Printf("calcHeight: %d\n", calcHeight)
//
// resObjectCount := make(map[string][]gjson.Result)
// resObjectSealCount := make(map[string][]gjson.Result)
//
// // get everyone sp object count
// for _, spHost := range spHostArray {
// xmlResult, err := abci.BsDBInfoBlockHeight(spHost, calcHeight)
// if err != nil {
// fmt.Println(err)
// }
//
// objectResString := utils.GetXmlPath(xmlResult, "GfSpGetBsDBInfoResponse/ObjectTotalCount")
// if objectResString == "" {
// fmt.Printf("sp: %v, ObjectTotalCount error\n", spHost)
// return calcHeight, GetObjectTotalCountErr
// } else {
// ObjectTotalCount := gjson.Parse(objectResString).Array()
// resObjectCount[spHost] = ObjectTotalCount
// }
//
// objectSealResString := utils.GetXmlPath(xmlResult, "GfSpGetBsDBInfoResponse/ObjectSealCount")
// if objectSealResString == "" {
// fmt.Printf("sp: %v, ObjectSealCount error\n", spHost)
// return calcHeight, GetObjectSealCountErr
// } else {
// ObjectSealCount := gjson.Parse(objectSealResString).Array()
// resObjectSealCount[spHost] = ObjectSealCount
// }
// }
//
// // check sp object count
// for i := 0; i < 64; i++ {
// sumObject := int64(0)
// sumSp1 := int64(0)
// for _, objectCount := range resObjectCount {
// sumObject = sumObject + objectCount[i].Int()
// sumSp1++
// }
// sumSealedObject := int64(0)
// sumSp2 := int64(0)
// for _, sealObjectCount := range resObjectSealCount {
// sumSealedObject = sumSealedObject + sealObjectCount[i].Int()
// sumSp2++
// }
//
// objectAverage := sumObject / sumSp1
// sealObjectAverage := sumSealedObject / sumSp2
// for _, eachValue := range resObjectCount {
// if objectAverage != eachValue[i].Int() {
// return calcHeight, CheckObjectTotalCountErr
// }
// }
// for _, eachValue := range resObjectSealCount {
// if sealObjectAverage != eachValue[i].Int() {
// return calcHeight, CheckObjectSealCountErr
// }
// }
// }
//
// return calcHeight, OK
//}
63 changes: 9 additions & 54 deletions main.go
View file
Original file line number Diff line number Diff line change
@@ -1,75 +1,30 @@
package main

import (
"fmt"
"github.com/bnb-chain/gnfd-qa-test-monitor/abci"
"github.com/bnb-chain/gnfd-qa-test-monitor/checks"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
"net/http"
"strings"
"time"
)

func recordMetrics() {
testNetRpc := "https://gnfd-testnet-fullnode-tendermint-us.bnbchain.org:443"
mainNetRpc := "https://greenfield-chain.bnbchain.org:443"

checkTestNetBlock := promauto.NewGauge(prometheus.GaugeOpts{Name: "testnet_sp_db_shard_check_block_height"})
checkMainNetBlock := promauto.NewGauge(prometheus.GaugeOpts{Name: "mainnet_sp_db_shard_check_block_height"})

checkTestNetSpErrCodes := make([]prometheus.Gauge, len(checks.TestNetSpHost))
for i, spHost := range checks.TestNetSpHost {
spHost = strings.Replace(spHost, "-", "_", -1)
spHost = strings.Replace(spHost, ".", "_", -1)
name := fmt.Sprintf("testnet_sp_db_shard_error_code_%v", spHost)
checkTestNetSpErrCodes[i] = promauto.NewGauge(prometheus.GaugeOpts{Name: name})
}

checkMainNetSpErrCodes := make([]prometheus.Gauge, len(checks.TestNetSpHost))
for i, spHost := range checks.TestNetSpHost {
spHost = strings.Replace(spHost, "-", "_", -1)
spHost = strings.Replace(spHost, ".", "_", -1)
name := fmt.Sprintf("mainnet_sp_db_shard_error_code_%v", spHost)
checkMainNetSpErrCodes[i] = promauto.NewGauge(prometheus.GaugeOpts{Name: name})
}

func recordMetricsV() {
go func() {
for {
// check TestNet
testNetChainHeight, err := abci.LastBlockHeight(testNetRpc)
if err != nil {
fmt.Println(err)
time.Sleep(time.Minute * 10)
}
testNetCalcHeight := testNetChainHeight / 3600 * 3600
checkTestNetBlock.Set(float64(testNetCalcHeight))
for i, spHost := range checks.TestNetSpHost {
errCode := checks.CheckDbShard(testNetCalcHeight, spHost)
checkTestNetSpErrCodes[i].Set(float64(errCode))
}

// check MainNet
mainNetChainHeight, err := abci.LastBlockHeight(mainNetRpc)
if err != nil {
fmt.Println(err)
time.Sleep(time.Minute * 10)
}
mainNetCalcHeight := mainNetChainHeight / 3600 * 3600
checkMainNetBlock.Set(float64(mainNetCalcHeight))
for i, spHost := range checks.MainNetSpHost {
errCode := checks.CheckDbShard(mainNetCalcHeight, spHost)
checkMainNetSpErrCodes[i].Set(float64(errCode))
}
checks.CheckDbShard("testnet", checks.TestNetRpc, checks.TestNetSpHosts)
time.Sleep(time.Minute * 10)
}
}()

go func() {
for {
checks.CheckDbShard("mainnet", checks.MainNetRpc, checks.MainNetSpHosts)
time.Sleep(time.Minute * 10)
}
}()
}

func main() {
recordMetrics()
recordMetricsV()
http.Handle("/metrics", promhttp.Handler())
err := http.ListenAndServe(":24367", nil)
if err != nil {
Expand Down

0 comments on commit 780b2b6

Please sign in to comment.