diff --git a/server/ingester/ingesterctl/cmd/cmd.go b/server/ingester/ingesterctl/cmd/cmd.go index 8dc815fa055..8532b12ed5d 100644 --- a/server/ingester/ingesterctl/cmd/cmd.go +++ b/server/ingester/ingesterctl/cmd/cmd.go @@ -63,9 +63,13 @@ func RegisterIngesterCommand(root *cobra.Command) { Use: "otlp", Short: "otlp exporter debug commands", } + profileCmd := &cobra.Command{ + Use: "profile", + Short: "profile debug commands", + } root.AddCommand(ingesterCmd) - ingesterCmd.AddCommand(dropletCmd, flowMetricsCmd, flowLogCmd, prometheusCmd, otlpCmd) + ingesterCmd.AddCommand(dropletCmd, flowMetricsCmd, flowLogCmd, prometheusCmd, otlpCmd, profileCmd) ingesterCmd.AddCommand(profiler.RegisterProfilerCommand()) ingesterCmd.AddCommand(debug.RegisterLogLevelCommand()) ingesterCmd.AddCommand(RegisterTimeConvertCommand()) @@ -102,6 +106,8 @@ func RegisterIngesterCommand(root *cobra.Command) { otlpCmd.AddCommand(debug.ClientRegisterSimple(ingesterctl.CMD_OTLP_EXPORTER, debug.CmdHelper{"stats", "show otlp exporter stats"}, nil)) otlpCmd.AddCommand(debug.ClientRegisterSimple(ingesterctl.CMD_EXPORTER_PLATFORMDATA, debug.CmdHelper{"platformData", "show otlp platformData"}, nil)) + profileCmd.AddCommand(debug.ClientRegisterSimple(ingesterctl.CMD_PLATFORMDATA_PROFILE, debug.CmdHelper{"platformData [filter]", "show profile platform data statistics"}, nil)) + root.GenBashCompletionFile("/usr/share/bash-completion/completions/deepflow-ctl") } diff --git a/server/ingester/ingesterctl/const.go b/server/ingester/ingesterctl/const.go index 675d5750f84..fbcab950585 100644 --- a/server/ingester/ingesterctl/const.go +++ b/server/ingester/ingesterctl/const.go @@ -51,6 +51,7 @@ const ( CMD_L7_FLOW_LOG CMD_OTLP_EXPORTER CMD_EXPORTER_PLATFORMDATA + CMD_PLATFORMDATA_PROFILE ) const ( diff --git a/server/ingester/profile/dbwriter/profile.go b/server/ingester/profile/dbwriter/profile.go index abd0e9193a8..530a04e4e18 100644 --- a/server/ingester/profile/dbwriter/profile.go +++ b/server/ingester/profile/dbwriter/profile.go @@ -381,6 +381,7 @@ func (p *InProcessProfile) fillResource(vtapID uint32, containerID string, platf func (p *InProcessProfile) fillPodInfo(vtapID uint32, containerID string, platformData *grpc.PlatformInfoTable) { if containerID == "" { + log.Debugf("%s-%s uploaded empty containerID by vtapID: %d", p.AppService, p.ProfileEventType, vtapID) return } podInfo := platformData.QueryPodContainerInfo(vtapID, containerID) diff --git a/server/ingester/profile/decoder/decoder_test.go b/server/ingester/profile/decoder/decoder_test.go new file mode 100644 index 00000000000..8c70eb9a320 --- /dev/null +++ b/server/ingester/profile/decoder/decoder_test.go @@ -0,0 +1,32 @@ +package decoder + +import ( + "encoding/hex" + "fmt" + "testing" + + "github.com/deepflowio/deepflow/server/libs/zerodoc/pb" +) + +func TestDecoder(t *testing.T) { + data := []byte{ + 40, 99, 90, 253, 4, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 59, 91, 47, 117, 115, 114, 47, 108, 111, 99, 97, 108, 47, 98, 105, 110, 47, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 93, 59, 91, 47, 117, 115, 114, 47, 108, 111, 99, 97, 108, 47, 98, 105, 110, 47, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 93, 59, 91, 47, 117, 115, 114, 47, 108, 111, 99, 97, 108, 47, 98, 105, 110, 47, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 93, 59, 91, 47, 117, 115, 114, 47, 108, 111, 99, 97, 108, 47, 98, 105, 110, 47, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 93, 59, 91, 47, 117, 115, 114, 47, 108, 111, 99, 97, 108, 47, 98, 105, 110, 47, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 93, 59, 91, 47, 117, 115, 114, 47, 108, 111, 99, 97, 108, 47, 98, 105, 110, 47, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 93, 59, 91, 47, 117, 115, 114, 47, 108, 111, 99, 97, 108, 47, 98, 105, 110, 47, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 93, 59, 91, 47, 117, 115, 114, 47, 108, 111, 99, 97, 108, 47, 98, 105, 110, 47, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 93, 59, 91, 47, 117, 115, 114, 47, 108, 111, 99, 97, 108, 47, 98, 105, 110, 47, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 93, 59, 91, 47, 117, 115, 114, 47, 108, 111, 99, 97, 108, 47, 98, 105, 110, 47, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 93, 59, 91, 47, 117, 115, 114, 47, 108, 111, 99, 97, 108, 47, 98, 105, 110, 47, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 93, 59, 91, 47, 117, 115, 114, 47, 108, 111, 99, 97, 108, 47, 98, 105, 110, 47, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 93, 59, 91, 47, 117, 115, 114, 47, 108, 111, 99, 97, 108, 47, 98, 105, 110, 47, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 93, 59, 91, 47, 117, 115, 114, 47, 108, 111, 99, 97, 108, 47, 98, 105, 110, 47, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 93, 59, 91, 107, 93, 32, 101, 110, 116, 114, 121, 95, 83, 89, 83, 67, 65, 76, 76, 95, 54, 52, 95, 97, 102, 116, 101, 114, 95, 104, 119, 102, 114, 97, 109, 101, 59, 91, 107, 93, 32, 100, 111, 95, 115, 121, 115, 99, 97, 108, 108, 95, 54, 52, 59, 91, 107, 93, 32, 95, 95, 120, 54, 52, 95, 115, 121, 115, 95, 102, 117, 116, 101, 120, 59, 91, 107, 93, 32, 100, 111, 95, 102, 117, 116, 101, 120, 59, 91, 107, 93, 32, 102, 117, 116, 101, 120, 95, 119, 97, 105, 116, 59, 91, 107, 93, 32, 104, 114, 116, 105, 109, 101, 114, 95, 99, 97, 110, 99, 101, 108, 59, 91, 107, 93, 32, 104, 114, 116, 105, 109, 101, 114, 95, 116, 114, 121, 95, 116, 111, 95, 99, 97, 110, 99, 101, 108, 46, 112, 97, 114, 116, 46, 48, 59, 91, 107, 93, 32, 95, 95, 108, 111, 99, 107, 95, 116, 101, 120, 116, 95, 115, 116, 97, 114, 116, 160, 1, 192, 249, 210, 135, 242, 168, 168, 199, 23, 168, 1, 1, 176, 1, 240, 241, 164, 145, 177, 49, 184, 1, 239, 15, 192, 1, 145, 17, 202, 1, 14, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 210, 1, 14, 107, 117, 98, 101, 45, 97, 112, 105, 115, 101, 114, 118, 101, 114, 216, 1, 169, 221, 3, 224, 1, 211, 172, 2, 232, 1, 255, 31, 240, 1, 1, 248, 1, 152, 129, 128, 128, 15, 130, 2, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + } + fmt.Println(string(data)) +} + +func TestDecoder2(t *testing.T) { + data := "28635a0e6a6176613b5b756e6b6e6f776e5da001e69b97b491cea6c717a80101b801ae0ac001b71dca01046a617661d201046a617661d801d6d602e001f2ffffff0fe801ff1ff00101f801bf8d80800f8202413264366432363439626634303635313664373365653333396439373731373233303662643730623962373064343162383633336337303239353235613965306400c7000000" + reqByte, err := hex.DecodeString(data) + if err != nil { + fmt.Println(err) + } + var req pb.Profile + if err := req.Unmarshal(reqByte); err != nil { + fmt.Printf("[%s]", string(req.ContainerId)) + // 2d6d2649bf406516d73ee339d977172306bd70b9b70d41b8633c7029525a9e0d + // 2d6d2649bf406516d73ee339d977172306bd70b9b70d41b8633c7029525a9e0d + return + } + fmt.Println(string(data)) +} diff --git a/server/ingester/profile/profile/profile.go b/server/ingester/profile/profile/profile.go index ae62ac54ca7..e3934fddd2d 100644 --- a/server/ingester/profile/profile/profile.go +++ b/server/ingester/profile/profile/profile.go @@ -26,6 +26,7 @@ import ( "github.com/deepflowio/deepflow/server/ingester/profile/dbwriter" "github.com/deepflowio/deepflow/server/ingester/profile/decoder" "github.com/deepflowio/deepflow/server/libs/datatype" + "github.com/deepflowio/deepflow/server/libs/debug" "github.com/deepflowio/deepflow/server/libs/grpc" "github.com/deepflowio/deepflow/server/libs/queue" libqueue "github.com/deepflowio/deepflow/server/libs/queue" @@ -71,6 +72,7 @@ func NewProfiler(msgType datatype.MessageType, config *config.Config, platformDa if err != nil { return nil, err } + debug.ServerRegisterSimple(ingesterctl.CMD_PLATFORMDATA_PROFILE, platformDatas[i]) } profileWriter, err := dbwriter.NewProfileWriter(datatype.MESSAGE_TYPE_PROFILE, i, config) if err != nil { diff --git a/server/libs/grpc/grpc_platformdata.go b/server/libs/grpc/grpc_platformdata.go index 27b29b7d361..5a2aad01113 100644 --- a/server/libs/grpc/grpc_platformdata.go +++ b/server/libs/grpc/grpc_platformdata.go @@ -171,6 +171,7 @@ type PlatformInfoTable struct { podNameInfos map[string][]*PodInfo vtapIdInfos map[uint32]*VtapInfo containerInfos map[string][]*PodInfo + containerHitCount map[string]*uint64 containerMissCount map[string]*uint64 peerConnections map[int32][]int32 @@ -412,6 +413,7 @@ func NewPlatformInfoTable(ips []net.IP, port, index, rpcMaxMsgSize int, moduleNa vtapIdInfos: make(map[uint32]*VtapInfo), containerInfos: make(map[string][]*PodInfo), containerMissCount: make(map[string]*uint64), + containerHitCount: make(map[string]*uint64), peerConnections: make(map[int32][]int32), ctlIP: nodeIP, counter: &Counter{}, @@ -858,8 +860,17 @@ func (t *PlatformInfoTable) String() string { fmt.Fprintf(sb, " %-15d %d\n", epcID, *missCount) } + if len(t.containerHitCount) > 0 { + sb.WriteString("\n9 *containerID hitcount (使用containerID匹配pod信息成功的统计)\n") + sb.WriteString("--------------------------------\n") + } + + for containerID, hitCount := range t.containerHitCount { + fmt.Fprintf(sb, " %-20s %d\n", containerID, *hitCount) + } + if len(t.containerMissCount) > 0 { - sb.WriteString("\n9 *containerID miss (使用containerID无法匹配到pod信息的统计)\n") + sb.WriteString("\n10 *containerID miss (使用containerID无法匹配到pod信息的统计)\n") sb.WriteString("--------------------------------\n") } @@ -958,6 +969,8 @@ func (t *PlatformInfoTable) updatePlatformData(platformData *trident.PlatformDat t.epcIDBaseInfos = newEpcIDBaseInfos t.epcIDBaseMissCount = make(map[int32]*uint64) + t.containerHitCount = make(map[string]*uint64) + t.containerMissCount = make(map[string]*uint64) t.netnsIdInfos = newNetnsIdInfos t.epcIDPodInfos = newEpcIDPodInfos @@ -1021,6 +1034,9 @@ func (t *PlatformInfoTable) ReloadSlave() error { t.epcIDBaseInfos = masterTable.epcIDBaseInfos t.epcIDBaseMissCount = make(map[int32]*uint64) + t.containerHitCount = make(map[string]*uint64) + t.containerMissCount = make(map[string]*uint64) + t.versionPlatformData = newVersion t.otherRegionCount = 0 } @@ -1496,9 +1512,16 @@ func (t *PlatformInfoTable) QueryPodContainerInfo(vtapID uint32, containerID str atomic.AddInt64(&t.counter.ContainerTotalCount, 1) // assume containerid will not repeat in one cluster if containerInfos, ok := t.containerInfos[containerID]; ok { - atomic.AddInt64(&t.counter.ContainerHitCount, int64(len(containerInfos))) for _, podInfo := range containerInfos { if podInfo.PodClusterId == podClusterId { + hitCount, ok := t.containerHitCount[containerID] + if !ok { + hitCount = new(uint64) + t.containerHitCount[containerID] = hitCount + } + atomic.AddUint64(hitCount, 1) + atomic.AddInt64(&t.counter.ContainerHitCount, 1) + return podInfo } } @@ -1510,7 +1533,7 @@ func (t *PlatformInfoTable) QueryPodContainerInfo(vtapID uint32, containerID str t.containerMissCount[containerID] = missCount } atomic.AddUint64(missCount, 1) - atomic.AddInt64(&t.counter.ContainerMissCount, int64(len(containerInfos))) + atomic.AddInt64(&t.counter.ContainerMissCount, 1) } } return nil