[ingester] add stats for containerhit

deepflowio · Oct 17, 2023 · 9b7d8a3 · 9b7d8a3
1 parent 4621dad
commit 9b7d8a3
Show file tree

Hide file tree

Showing 7 changed files with 43 additions and 10 deletions.
diff --git a/cli/go.mod b/cli/go.mod
@@ -4,8 +4,8 @@ go 1.18
 
 require (
 	github.com/bitly/go-simplejson v0.5.0
-	github.com/deepflowio/deepflow/message v0.0.0-20230927071530-6d75b95973e7
-	github.com/deepflowio/deepflow/server v0.0.0-20231009085033-3a5edd2556eb
+	github.com/deepflowio/deepflow/message v0.0.0-20231009120220-2f68ce239be3
+	github.com/deepflowio/deepflow/server v0.0.0-20231017074137-795433a9f8d6
 	github.com/golang/protobuf v1.5.3
 	github.com/mattn/go-runewidth v0.0.14
 	github.com/olekukonko/tablewriter v0.0.5

diff --git a/cli/go.sum b/cli/go.sum
@@ -64,10 +64,10 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/deepflowio/deepflow/message v0.0.0-20230927071530-6d75b95973e7 h1:rrvioA174n8n6g0Jk24vtb+8o1dpvT4fP8ASQfSPJEc=
-github.com/deepflowio/deepflow/message v0.0.0-20230927071530-6d75b95973e7/go.mod h1:e+1lUMMlycCvFRKvlwt/y/0vxJnF8wVss3GyR1ARXY0=
-github.com/deepflowio/deepflow/server v0.0.0-20231009085033-3a5edd2556eb h1:eVjiez/siYxkgtfeqaXWsY9uN0IiOwxKzlZOhvlebRQ=
-github.com/deepflowio/deepflow/server v0.0.0-20231009085033-3a5edd2556eb/go.mod h1:ycwHI3PEI6Sx5R3euwdmkQhZud8HReZyfVnv9PeJhtc=
+github.com/deepflowio/deepflow/message v0.0.0-20231009120220-2f68ce239be3 h1:sR2fwjKCZvK3iP5WCfaoscD6mrPohlXf2r/JSRG3b18=
+github.com/deepflowio/deepflow/message v0.0.0-20231009120220-2f68ce239be3/go.mod h1:e+1lUMMlycCvFRKvlwt/y/0vxJnF8wVss3GyR1ARXY0=
+github.com/deepflowio/deepflow/server v0.0.0-20231017074137-795433a9f8d6 h1:aV4DG5qAMckvxLs+uTH0epzkb/MIQ+PHUcIvTTDXNXA=
+github.com/deepflowio/deepflow/server v0.0.0-20231017074137-795433a9f8d6/go.mod h1:wHHRLzFXOh8dwThssCb4WvfHOwV2mg0VZTuH+ED+7CM=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=

diff --git a/server/ingester/ingesterctl/cmd/cmd.go b/server/ingester/ingesterctl/cmd/cmd.go
@@ -63,9 +63,13 @@ func RegisterIngesterCommand(root *cobra.Command) {
 		Use:   "otlp",
 		Short: "otlp exporter debug commands",
 	}
+	profileCmd := &cobra.Command{
+		Use:   "profile",
+		Short: "profile debug commands",
+	}
 
 	root.AddCommand(ingesterCmd)
-	ingesterCmd.AddCommand(dropletCmd, flowMetricsCmd, flowLogCmd, prometheusCmd, otlpCmd)
+	ingesterCmd.AddCommand(dropletCmd, flowMetricsCmd, flowLogCmd, prometheusCmd, otlpCmd, profileCmd)
 	ingesterCmd.AddCommand(profiler.RegisterProfilerCommand())
 	ingesterCmd.AddCommand(debug.RegisterLogLevelCommand())
 	ingesterCmd.AddCommand(RegisterTimeConvertCommand())
@@ -102,6 +106,8 @@ func RegisterIngesterCommand(root *cobra.Command) {
 	otlpCmd.AddCommand(debug.ClientRegisterSimple(ingesterctl.CMD_OTLP_EXPORTER, debug.CmdHelper{"stats", "show otlp exporter stats"}, nil))
 	otlpCmd.AddCommand(debug.ClientRegisterSimple(ingesterctl.CMD_EXPORTER_PLATFORMDATA, debug.CmdHelper{"platformData", "show otlp platformData"}, nil))
 
+	profileCmd.AddCommand(debug.ClientRegisterSimple(ingesterctl.CMD_PLATFORMDATA_PROFILE, debug.CmdHelper{"platformData [filter]", "show profile platform data statistics"}, nil))
+
 	root.GenBashCompletionFile("/usr/share/bash-completion/completions/deepflow-ctl")
 }
 

diff --git a/server/ingester/ingesterctl/const.go b/server/ingester/ingesterctl/const.go
@@ -51,6 +51,7 @@ const (
 	CMD_L7_FLOW_LOG
 	CMD_OTLP_EXPORTER
 	CMD_EXPORTER_PLATFORMDATA
+	CMD_PLATFORMDATA_PROFILE
 )
 
 const (

diff --git a/server/ingester/profile/dbwriter/profile.go b/server/ingester/profile/dbwriter/profile.go
@@ -381,6 +381,7 @@ func (p *InProcessProfile) fillResource(vtapID uint32, containerID string, platf
 
 func (p *InProcessProfile) fillPodInfo(vtapID uint32, containerID string, platformData *grpc.PlatformInfoTable) {
 	if containerID == "" {
+		log.Debugf("%s-%s uploaded empty containerID by vtapID: %d", p.AppService, p.ProfileEventType, vtapID)
 		return
 	}
 	podInfo := platformData.QueryPodContainerInfo(vtapID, containerID)

diff --git a/server/ingester/profile/profile/profile.go b/server/ingester/profile/profile/profile.go
@@ -26,6 +26,7 @@ import (
 	"github.com/deepflowio/deepflow/server/ingester/profile/dbwriter"
 	"github.com/deepflowio/deepflow/server/ingester/profile/decoder"
 	"github.com/deepflowio/deepflow/server/libs/datatype"
+	"github.com/deepflowio/deepflow/server/libs/debug"
 	"github.com/deepflowio/deepflow/server/libs/grpc"
 	"github.com/deepflowio/deepflow/server/libs/queue"
 	libqueue "github.com/deepflowio/deepflow/server/libs/queue"
@@ -71,6 +72,7 @@ func NewProfiler(msgType datatype.MessageType, config *config.Config, platformDa
 			if err != nil {
 				return nil, err
 			}
+			debug.ServerRegisterSimple(ingesterctl.CMD_PLATFORMDATA_PROFILE, platformDatas[i])
 		}
 		profileWriter, err := dbwriter.NewProfileWriter(datatype.MESSAGE_TYPE_PROFILE, i, config)
 		if err != nil {

diff --git a/server/libs/grpc/grpc_platformdata.go b/server/libs/grpc/grpc_platformdata.go
@@ -171,6 +171,7 @@ type PlatformInfoTable struct {
 	podNameInfos       map[string][]*PodInfo
 	vtapIdInfos        map[uint32]*VtapInfo
 	containerInfos     map[string][]*PodInfo
+	containerHitCount  map[string]*uint64
 	containerMissCount map[string]*uint64
 
 	peerConnections map[int32][]int32
@@ -412,6 +413,7 @@ func NewPlatformInfoTable(ips []net.IP, port, index, rpcMaxMsgSize int, moduleNa
 		vtapIdInfos:        make(map[uint32]*VtapInfo),
 		containerInfos:     make(map[string][]*PodInfo),
 		containerMissCount: make(map[string]*uint64),
+		containerHitCount:  make(map[string]*uint64),
 		peerConnections:    make(map[int32][]int32),
 		ctlIP:              nodeIP,
 		counter:            &Counter{},
@@ -858,8 +860,17 @@ func (t *PlatformInfoTable) String() string {
 		fmt.Fprintf(sb, "  %-15d  %d\n", epcID, *missCount)
 	}
 
+	if len(t.containerHitCount) > 0 {
+		sb.WriteString("\n9 *containerID         hitcount  (使用containerID匹配pod信息成功的统计)\n")
+		sb.WriteString("--------------------------------\n")
+	}
+
+	for containerID, hitCount := range t.containerHitCount {
+		fmt.Fprintf(sb, "  %-20s  %d\n", containerID, *hitCount)
+	}
+
 	if len(t.containerMissCount) > 0 {
-		sb.WriteString("\n9 *containerID         miss  (使用containerID无法匹配到pod信息的统计)\n")
+		sb.WriteString("\n10 *containerID         miss  (使用containerID无法匹配到pod信息的统计)\n")
 		sb.WriteString("--------------------------------\n")
 	}
 
@@ -958,6 +969,8 @@ func (t *PlatformInfoTable) updatePlatformData(platformData *trident.PlatformDat
 
 	t.epcIDBaseInfos = newEpcIDBaseInfos
 	t.epcIDBaseMissCount = make(map[int32]*uint64)
+	t.containerHitCount = make(map[string]*uint64)
+	t.containerMissCount = make(map[string]*uint64)
 
 	t.netnsIdInfos = newNetnsIdInfos
 	t.epcIDPodInfos = newEpcIDPodInfos
@@ -1021,6 +1034,9 @@ func (t *PlatformInfoTable) ReloadSlave() error {
 		t.epcIDBaseInfos = masterTable.epcIDBaseInfos
 		t.epcIDBaseMissCount = make(map[int32]*uint64)
 
+		t.containerHitCount = make(map[string]*uint64)
+		t.containerMissCount = make(map[string]*uint64)
+
 		t.versionPlatformData = newVersion
 		t.otherRegionCount = 0
 	}
@@ -1496,9 +1512,16 @@ func (t *PlatformInfoTable) QueryPodContainerInfo(vtapID uint32, containerID str
 		atomic.AddInt64(&t.counter.ContainerTotalCount, 1)
 		// assume containerid will not repeat in one cluster
 		if containerInfos, ok := t.containerInfos[containerID]; ok {
-			atomic.AddInt64(&t.counter.ContainerHitCount, int64(len(containerInfos)))
 			for _, podInfo := range containerInfos {
 				if podInfo.PodClusterId == podClusterId {
+					hitCount, ok := t.containerHitCount[containerID]
+					if !ok {
+						hitCount = new(uint64)
+						t.containerHitCount[containerID] = hitCount
+					}
+					atomic.AddUint64(hitCount, 1)
+					atomic.AddInt64(&t.counter.ContainerHitCount, 1)
+
 					return podInfo
 				}
 			}
@@ -1510,7 +1533,7 @@ func (t *PlatformInfoTable) QueryPodContainerInfo(vtapID uint32, containerID str
 				t.containerMissCount[containerID] = missCount
 			}
 			atomic.AddUint64(missCount, 1)
-			atomic.AddInt64(&t.counter.ContainerMissCount, int64(len(containerInfos)))
+			atomic.AddInt64(&t.counter.ContainerMissCount, 1)
 		}
 	}
 	return nil