[ingester] l7_flow_log store syscall_coroutine_0/syscall_coroutine_1

deepflowio · Oct 16, 2023 · aa524d4 · aa524d4
1 parent 76b61af
commit aa524d4
Show file tree

Hide file tree

Showing 8 changed files with 28 additions and 5 deletions.
diff --git a/server/go.mod b/server/go.mod
@@ -35,7 +35,7 @@ require (
 	github.com/cornelk/hashmap v1.0.8
 	github.com/deckarep/golang-set v1.8.0
 	github.com/deckarep/golang-set/v2 v2.1.0
-	github.com/deepflowio/deepflow/message v0.0.0-20230927071530-6d75b95973e7
+	github.com/deepflowio/deepflow/message v0.0.0-20231009120220-2f68ce239be3
 	github.com/deepflowio/deepflow/server/controller/cloud/kubernetes_gather/expand v0.0.0-00010101000000-000000000000
 	github.com/deepflowio/deepflow/server/controller/cloud/platform v0.0.0-00010101000000-000000000000
 	github.com/deepflowio/deepflow/server/controller/db/mysql/migrator v0.0.0-00010101000000-000000000000

diff --git a/server/go.sum b/server/go.sum
@@ -153,8 +153,8 @@ github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsP
 github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo=
 github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI=
 github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
-github.com/deepflowio/deepflow/message v0.0.0-20230927071530-6d75b95973e7 h1:rrvioA174n8n6g0Jk24vtb+8o1dpvT4fP8ASQfSPJEc=
-github.com/deepflowio/deepflow/message v0.0.0-20230927071530-6d75b95973e7/go.mod h1:e+1lUMMlycCvFRKvlwt/y/0vxJnF8wVss3GyR1ARXY0=
+github.com/deepflowio/deepflow/message v0.0.0-20231009120220-2f68ce239be3 h1:sR2fwjKCZvK3iP5WCfaoscD6mrPohlXf2r/JSRG3b18=
+github.com/deepflowio/deepflow/message v0.0.0-20231009120220-2f68ce239be3/go.mod h1:e+1lUMMlycCvFRKvlwt/y/0vxJnF8wVss3GyR1ARXY0=
 github.com/deepflowio/tempopb v0.0.0-20230215110519-15853baf3a79 h1:erRwXyZiUZxxZX/Q1QHesZXgxjiunZUFy+ggCRimkD4=
 github.com/deepflowio/tempopb v0.0.0-20230215110519-15853baf3a79/go.mod h1:h2rkZ319TExIUGuK8a2dlcGd8qc6wdhsrcpXWaJQaQE=
 github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE=

diff --git a/server/ingester/ckissu/ckissu.go b/server/ingester/ckissu/ckissu.go
@@ -950,6 +950,15 @@ var ColumnDrops635 = []*ColumnDrops{
 	},
 }
 
+var ColumnAdd64 = []*ColumnAdds{
+	&ColumnAdds{
+		Dbs:         []string{"flow_log"},
+		Tables:      []string{"l7_flow_log", "l7_flow_log_local"},
+		ColumnNames: []string{"syscall_coroutine_0", "syscall_coroutine_1"},
+		ColumnType:  ckdb.UInt64,
+	},
+}
+
 func getTables(connect *sql.DB, db, tableName string) ([]string, error) {
 	sql := fmt.Sprintf("SHOW TABLES IN %s", db)
 	rows, err := connect.Query(sql)
@@ -1330,7 +1339,7 @@ func NewCKIssu(cfg *config.Config) (*Issu, error) {
 		datasourceInfo: make(map[string]*DatasourceInfo),
 	}
 
-	allVersionAdds := [][]*ColumnAdds{ColumnAdd610, ColumnAdd611, ColumnAdd612, ColumnAdd613, ColumnAdd615, ColumnAdd618, ColumnAdd620, ColumnAdd623, ColumnAdd625, ColumnAdd626, ColumnAdd633, ColumnAdd635}
+	allVersionAdds := [][]*ColumnAdds{ColumnAdd610, ColumnAdd611, ColumnAdd612, ColumnAdd613, ColumnAdd615, ColumnAdd618, ColumnAdd620, ColumnAdd623, ColumnAdd625, ColumnAdd626, ColumnAdd633, ColumnAdd635, ColumnAdd64}
 	i.columnAdds = []*ColumnAdd{}
 	for _, versionAdd := range allVersionAdds {
 		for _, adds := range versionAdd {

diff --git a/server/ingester/common/const.go b/server/ingester/common/const.go
@@ -17,6 +17,6 @@
 package common
 
 const (
-	CK_VERSION             = "v6.3.8.2" // 用于表示clickhouse的表版本号
+	CK_VERSION             = "v6.3.4.0" // 用于表示clickhouse的表版本号
 	DEFAULT_PCAP_DATA_PATH = "/var/lib/pcap"
 )
diff --git a/server/ingester/flow_log/log_data/l7_flow_log.go b/server/ingester/flow_log/log_data/l7_flow_log.go
@@ -81,6 +81,8 @@ type L7Base struct {
 	SyscallTraceIDResponse uint64
 	SyscallThread0         uint32
 	SyscallThread1         uint32
+	SyscallCoroutine0      uint64
+	SyscallCoroutine1      uint64
 	SyscallCapSeq0         uint32
 	SyscallCapSeq1         uint32
 }
@@ -130,6 +132,8 @@ func L7BaseColumns() []*ckdb.Column {
 		ckdb.NewColumn("syscall_trace_id_response", ckdb.UInt64).SetComment("SyscallTraceID-响应"),
 		ckdb.NewColumn("syscall_thread_0", ckdb.UInt32).SetComment("Syscall线程-请求"),
 		ckdb.NewColumn("syscall_thread_1", ckdb.UInt32).SetComment("Syscall线程-响应"),
+		ckdb.NewColumn("syscall_coroutine_0", ckdb.UInt64).SetComment("Request Syscall Coroutine"),
+		ckdb.NewColumn("syscall_coroutine_1", ckdb.UInt64).SetComment("Response Syscall Coroutine"),
 		ckdb.NewColumn("syscall_cap_seq_0", ckdb.UInt32).SetComment("Syscall序列号-请求"),
 		ckdb.NewColumn("syscall_cap_seq_1", ckdb.UInt32).SetComment("Syscall序列号-响应"),
 	)
@@ -177,6 +181,8 @@ func (f *L7Base) WriteBlock(block *ckdb.Block) {
 		f.SyscallTraceIDResponse,
 		f.SyscallThread0,
 		f.SyscallThread1,
+		f.SyscallCoroutine0,
+		f.SyscallCoroutine1,
 		f.SyscallCapSeq0,
 		f.SyscallCapSeq1)
 }
@@ -538,6 +544,8 @@ func (b *L7Base) Fill(log *pb.AppProtoLogsData, platformData *grpc.PlatformInfoT
 	b.SyscallTraceIDResponse = l.SyscallTraceIdResponse
 	b.SyscallThread0 = l.SyscallTraceIdThread_0
 	b.SyscallThread1 = l.SyscallTraceIdThread_1
+	b.SyscallCoroutine0 = l.SyscallCoroutine_0
+	b.SyscallCoroutine1 = l.SyscallCoroutine_1
 	b.SyscallCapSeq0 = l.SyscallCapSeq_0
 	b.SyscallCapSeq1 = l.SyscallCapSeq_1
 

diff --git a/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log b/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log
@@ -90,6 +90,8 @@ syscall_trace_id_request  , syscall_trace_id_request  , syscall_trace_id_request
 syscall_trace_id_response , syscall_trace_id_response , syscall_trace_id_response  , int            ,                       , Tracing Info      , 111
 syscall_thread_0          , syscall_thread_0          , syscall_thread_0           , int            ,                       , Tracing Info      , 111
 syscall_thread_1          , syscall_thread_1          , syscall_thread_1           , int            ,                       , Tracing Info      , 111
+syscall_coroutine_0       , syscall_coroutine_0       , syscall_coroutine_0        , int            ,                       , Tracing Info      , 111
+syscall_coroutine_1       , syscall_coroutine_1       , syscall_coroutine_1        , int            ,                       , Tracing Info      , 111
 syscall_cap_seq_0         , syscall_cap_seq_0         , syscall_cap_seq_0          , int            ,                       , Tracing Info      , 111
 syscall_cap_seq_1         , syscall_cap_seq_1         , syscall_cap_seq_1          , int            ,                       , Tracing Info      , 111
 

diff --git a/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log.ch b/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log.ch
@@ -90,6 +90,8 @@ syscall_trace_id_request  , 请求 Syscall TraceID     ,
 syscall_trace_id_response , 响应 Syscall TraceID     ,
 syscall_thread_0          , 请求 Syscall 线程        ,
 syscall_thread_1          , 响应 Syscall 线程        ,
+syscall_coroutine_0       , 请求 Syscall 协程        ,
+syscall_coroutine_1       , 响应 Syscall 协程        ,
 syscall_cap_seq_0         , 请求 Syscall 序号        ,
 syscall_cap_seq_1         , 响应 Syscall 序号        ,
 

diff --git a/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log.en b/server/querier/db_descriptions/clickhouse/tag/flow_log/l7_flow_log.en
@@ -90,6 +90,8 @@ syscall_trace_id_request  , Req Syscall TraceID           ,
 syscall_trace_id_response , Resp Syscall TraceID          ,
 syscall_thread_0          , Req Syscall Thread            ,
 syscall_thread_1          , Resp Syscall Thread           ,
+syscall_coroutine_0       , Req Syscall Coroutine         ,
+syscall_coroutine_1       , Resp Syscall Coroutine        ,
 syscall_cap_seq_0         , Req Syscall CapSeq            ,
 syscall_cap_seq_1         , Resp Syscall CapSeq           ,