Skip to content

Commit

Permalink
ci(scripts/join): improve join nightly metrics (#2299)
Browse files Browse the repository at this point in the history
Show cpu and net rx container stats.

issue: #2294
  • Loading branch information
corverroos authored Oct 28, 2024
1 parent e4ead65 commit 4bbce22
Showing 1 changed file with 62 additions and 1 deletion.
63 changes: 62 additions & 1 deletion scripts/join/join_test.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package join_test

import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"math"
Expand Down Expand Up @@ -111,6 +113,8 @@ func TestJoinOmega(t *testing.T) {
case <-ticker.C:
haloStatus, err := retry(ctx, haloStatus)
require.NoError(t, err)
stats, err := retry(ctx, getContainerStats)
require.NoError(t, err)

// CometBFT RPC errors while syncing, so best effort fetch
var haloSynced bool
Expand All @@ -136,10 +140,14 @@ func TestJoinOmega(t *testing.T) {
"cstatus", haloStatus,
"csynced", haloSynced,
"cheight", haloHeight,
"ccpu", stats.HaloCPU,
"crx", stats.HaloRX,
"esynced", execSynced,
"eheight", execHeight,
"etarget", execTarget,
"duration", time.Since(t0).Truncate(time.Second),
"ecpu", stats.EVMCPU,
"erx", stats.EVMRX,
"duration", time.Since(t0).Truncate(10*time.Second),
"eta", eta,
"bps", bps,
)
Expand Down Expand Up @@ -241,3 +249,56 @@ func getGitCommit7(t *testing.T) string {

return string(out)[0:7]
}

type stats struct {
HaloCPU string
HaloRX string
EVMCPU string
EVMRX string
}

type statsJSON struct {
BlockIO string `json:"BlockIO"`
CPUPerc string `json:"CPUPerc"`
Container string `json:"Container"`
ID string `json:"ID"`
MemPerc string `json:"MemPerc"`
MemUsage string `json:"MemUsage"`
Name string `json:"Name"`
NetIO string `json:"NetIO"`
PIDs string `json:"PIDs"`
}

// getContainerStats returns halo and omni_evm CPU and network RX stats.
func getContainerStats(ctx context.Context) (stats, error) {
out, err := exec.CommandContext(ctx, "docker", "stats", "--format=json", "--no-stream", "--no-trunc").CombinedOutput()
if err != nil {
return stats{}, errors.Wrap(err, "docker stats")
}

var resp stats
for _, line := range bytes.Split(out, []byte("\n")) {
line = bytes.TrimSpace(line)
if string(line) == "" {
continue
}

var s statsJSON
err := json.Unmarshal(line, &s)
if err != nil {
return stats{}, errors.Wrap(err, "json unmarshal")
}

cpu := strings.TrimSpace(s.CPUPerc)
rx := strings.TrimSpace(strings.Split(s.NetIO, "/")[0])
if s.Name == "halo" {
resp.HaloCPU = cpu
resp.HaloRX = rx
} else if s.Name == "omni_evm" {
resp.EVMCPU = cpu
resp.EVMRX = rx
}
}

return resp, nil
}

0 comments on commit 4bbce22

Please sign in to comment.