Skip to content

Commit

Permalink
Merge branch 'main' into support-debian
Browse files Browse the repository at this point in the history
  • Loading branch information
@yinjiping
yinjiping authored Oct 16, 2023
2 parents f127f94 + 3d11340 commit 2a8f3b2
Show file tree
Hide file tree
Showing 30 changed files with 461 additions and 223 deletions.
Binary file added agent/resources/test/flow_generator/mongo/mongo.pcap
View file
Binary file not shown.
30 changes: 30 additions & 0 deletions agent/resources/test/flow_generator/mongo/mongo.result
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 0, response_id: 0, op_code: 2004, op_code_name: "OP_QUERY", request: "{ \"isMaster\": 1, \"speculativeAuthenticate\": { \"saslStart\": 1, \"mechanism\": \"SCRAM-SHA-256\", \"payload\": Binary(0x0, biwsbj1hZG1pbixyPW5mdGVQaVovV1NuMUZrNjF5QWpFV29xbThaL0Y2MGc5), \"db\": \"admin\" }, \"saslSupportedMechs\": \"admin.admin\", \"client\": { \"application\": { \"name\": \"MongoDB Shell\" }, \"driver\": { \"name\": \"MongoDB Internal Client\", \"version\": \"4.4.25\" }, \"os\": { \"type\": \"Linux\", \"name\": \"CentOS Linux release 7.9.2009 (Core)\", \"architecture\": \"x86_64\", \"version\": \"Kernel 3.10.0-1160.80.1.el7.x86_64\" } } }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 0, response_id: 60, op_code: 1, op_code_name: "OP_REPLY", request: "", response: "{ \"ismaster\": true, \"topologyVersion\": { \"processId\": ObjectId(\"652213ba46c335fa2820b0dc\"), \"counter\": 0 }, \"maxBsonObjectSize\": 16777216, \"maxMessageSizeBytes\": 48000000, \"maxWriteBatchSize\": 100000, \"localTime\": DateTime(\"2023-10-08 2:46:22.212 +00:00:00\"), \"logicalSessionTimeoutMinutes\": 30, \"connectionId\": 3, \"minWireVersion\": 0, \"maxWireVersion\": 9, \"readOnly\": false, \"saslSupportedMechs\": [\"SCRAM-SHA-1\", \"SCRAM-SHA-256\"], \"speculativeAuthenticate\": { \"conversationId\": 1, \"done\": false, \"payload\": Binary(0x0, cj1uZnRlUGlaL1dTbjFGazYxeUFqRVdvcW04Wi9GNjBnOWJMZUpWOExOL3JQUUVtWERkYjZMTjJVb1puZlRidnZnLHM9dEpLa0drajNQcUNpc1dsdkN0L0gyWDZDVm5NOG5GVlV4UG1vQkE9PSxpPTE1MDAw) }, \"ok\": 1 }", response_code: 0, exception: "", rrt: 1053 } is_mongo: false
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 1, response_id: 0, op_code: 2013, op_code_name: "OP_MSG", request: "{ \"saslContinue\": 1, \"payload\": Binary(0x0, Yz1iaXdzLHI9bmZ0ZVBpWi9XU24xRms2MXlBakVXb3FtOFovRjYwZzliTGVKVjhMTi9yUFFFbVhEZGI2TE4yVW9abmZUYnZ2ZyxwPWhBVFRhMkhFWEw1VkRMRWFVdVM4OG84cGNIZmpRK1ZRRklkcnFwQjR1cXM9), \"conversationId\": 1, \"$db\": \"admin\" }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 1, response_id: 61, op_code: 2013, op_code_name: "OP_MSG", request: "", response: "{ \"conversationId\": 1, \"done\": false, \"payload\": Binary(0x0, dj1nT0psRVhyMTdXblV0UThqcDMvUlQ5bDhvRDZRN01GWDlGS3FUelRhdHpjPQ==), \"ok\": 1 }", response_code: 0, exception: "", rrt: 325 } is_mongo: false
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 2, response_id: 0, op_code: 2013, op_code_name: "OP_MSG", request: "{ \"saslContinue\": 1, \"payload\": Binary(0x0, ), \"conversationId\": 1, \"$db\": \"admin\" }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 2, response_id: 62, op_code: 2013, op_code_name: "OP_MSG", request: "", response: "{ \"conversationId\": 1, \"done\": true, \"payload\": Binary(0x0, ), \"ok\": 1 }", response_code: 0, exception: "", rrt: 338 } is_mongo: false
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 3, response_id: 0, op_code: 2013, op_code_name: "OP_MSG", request: "{ \"whatsmyuri\": 1, \"$db\": \"admin\" }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 3, response_id: 63, op_code: 2013, op_code_name: "OP_MSG", request: "", response: "{ \"you\": \"10.50.1.138:43250\", \"ok\": 1 }", response_code: 0, exception: "", rrt: 128 } is_mongo: false
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 4, response_id: 0, op_code: 2013, op_code_name: "OP_MSG", request: "{ \"buildinfo\": 1, \"$db\": \"admin\" }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 4, response_id: 64, op_code: 2013, op_code_name: "OP_MSG", request: "", response: "{}", response_code: 0, exception: "", rrt: 196 } is_mongo: false
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 5, response_id: 0, op_code: 2013, op_code_name: "OP_MSG", request: "{ \"getLog\": \"startupWarnings\", \"lsid\": { \"id\": Binary(0x4, uU9EjcLlRI+tnzaqrJqWqQ==) }, \"$db\": \"admin\" }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 5, response_id: 65, op_code: 2013, op_code_name: "OP_MSG", request: "", response: "{ \"totalLinesWritten\": 3, \"log\": [\"{\"t\":{\"$date\":\"2023-10-08T10:28:11.902+08:00\"},\"s\":\"W\", \"c\":\"CONTROL\", \"id\":22120, \"ctx\":\"initandlisten\",\"msg\":\"Access control is not enabled for the database. Read and write access to data and configuration is unrestricted\",\"tags\":[\"startupWarnings\"]}\", \"{\"t\":{\"$date\":\"2023-10-08T10:28:11.902+08:00\"},\"s\":\"W\", \"c\":\"CONTROL\", \"id\":22178, \"ctx\":\"initandlisten\",\"msg\":\"/sys/kernel/mm/transparent_hugepage/enabled is 'always'. We suggest setting it to 'never'\",\"tags\":[\"startupWarnings\"]}\", \"{\"t\":{\"$date\":\"2023-10-08T10:28:11.902+08:00\"},\"s\":\"W\", \"c\":\"CONTROL\", \"id\":22181, \"ctx\":\"initandlisten\",\"msg\":\"/sys/kernel/mm/transparent_hugepage/defrag is 'always'. We suggest setting it to 'never'\",\"tags\":[\"startupWarnings\"]}\"], \"ok\": 1 }", response_code: 0, exception: "", rrt: 182 } is_mongo: false
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 6, response_id: 0, op_code: 2013, op_code_name: "OP_MSG", request: "{ \"isMaster\": 1, \"forShell\": 1, \"lsid\": { \"id\": Binary(0x4, uU9EjcLlRI+tnzaqrJqWqQ==) }, \"$db\": \"test\" }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 6, response_id: 66, op_code: 2013, op_code_name: "OP_MSG", request: "", response: "{ \"ismaster\": true, \"topologyVersion\": { \"processId\": ObjectId(\"652213ba46c335fa2820b0dc\"), \"counter\": 0 }, \"maxBsonObjectSize\": 16777216, \"maxMessageSizeBytes\": 48000000, \"maxWriteBatchSize\": 100000, \"localTime\": DateTime(\"2023-10-08 2:46:22.3 +00:00:00\"), \"logicalSessionTimeoutMinutes\": 30, \"connectionId\": 3, \"minWireVersion\": 0, \"maxWireVersion\": 9, \"readOnly\": false, \"ok\": 1 }", response_code: 0, exception: "", rrt: 174 } is_mongo: false
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 7, response_id: 0, op_code: 2013, op_code_name: "OP_MSG", request: "{ \"buildInfo\": 1, \"lsid\": { \"id\": Binary(0x4, uU9EjcLlRI+tnzaqrJqWqQ==) }, \"$db\": \"test\" }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 7, response_id: 67, op_code: 2013, op_code_name: "OP_MSG", request: "", response: "{}", response_code: 0, exception: "", rrt: 139 } is_mongo: false
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 8, response_id: 0, op_code: 2013, op_code_name: "OP_MSG", request: "{ \"getCmdLineOpts\": 1, \"lsid\": { \"id\": Binary(0x4, uU9EjcLlRI+tnzaqrJqWqQ==) }, \"$db\": \"admin\" }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 8, response_id: 68, op_code: 2013, op_code_name: "OP_MSG", request: "", response: "{ \"argv\": [\"/usr/bin/mongod\", \"-f\", \"/etc/mongod.conf\"], \"parsed\": { \"config\": \"/etc/mongod.conf\", \"net\": { \"bindIp\": \"0.0.0.0\", \"port\": 27017 }, \"processManagement\": { \"timeZoneInfo\": \"/usr/share/zoneinfo\" }, \"storage\": { \"dbPath\": \"/var/lib/mongo\", \"journal\": { \"enabled\": true } }, \"systemLog\": { \"destination\": \"file\", \"logAppend\": true, \"path\": \"/var/log/mongodb/mongod.log\" } }, \"ok\": 1 }", response_code: 0, exception: "", rrt: 135 } is_mongo: false
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 9, response_id: 0, op_code: 2013, op_code_name: "OP_MSG", request: "{ \"buildInfo\": 1, \"$db\": \"test\" }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 9, response_id: 69, op_code: 2013, op_code_name: "OP_MSG", request: "", response: "{}", response_code: 0, exception: "", rrt: 207 } is_mongo: false
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 10, response_id: 0, op_code: 2013, op_code_name: "OP_MSG", request: "{ \"isMaster\": 1, \"forShell\": 1, \"$db\": \"test\" }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 10, response_id: 70, op_code: 2013, op_code_name: "OP_MSG", request: "", response: "{ \"ismaster\": true, \"topologyVersion\": { \"processId\": ObjectId(\"652213ba46c335fa2820b0dc\"), \"counter\": 0 }, \"maxBsonObjectSize\": 16777216, \"maxMessageSizeBytes\": 48000000, \"maxWriteBatchSize\": 100000, \"localTime\": DateTime(\"2023-10-08 2:46:22.306 +00:00:00\"), \"logicalSessionTimeoutMinutes\": 30, \"connectionId\": 3, \"minWireVersion\": 0, \"maxWireVersion\": 9, \"readOnly\": false, \"ok\": 1 }", response_code: 0, exception: "", rrt: 143 } is_mongo: false
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 11, response_id: 0, op_code: 2013, op_code_name: "OP_MSG", request: "{ \"replSetGetStatus\": 1, \"forShell\": 1, \"$db\": \"admin\" }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 11, response_id: 71, op_code: 2013, op_code_name: "OP_MSG", request: "", response: "", response_code: 76, exception: "not running with --replSet", rrt: 571 } is_mongo: false
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 12, response_id: 0, op_code: 2013, op_code_name: "OP_MSG", request: "{ \"getLog\": \"startupWarnings\", \"lsid\": { \"id\": Binary(0x4, uU9EjcLlRI+tnzaqrJqWqQ==) }, \"$db\": \"admin\" }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 12, response_id: 72, op_code: 2013, op_code_name: "OP_MSG", request: "", response: "{ \"totalLinesWritten\": 3, \"log\": [\"{\"t\":{\"$date\":\"2023-10-08T10:28:11.902+08:00\"},\"s\":\"W\", \"c\":\"CONTROL\", \"id\":22120, \"ctx\":\"initandlisten\",\"msg\":\"Access control is not enabled for the database. Read and write access to data and configuration is unrestricted\",\"tags\":[\"startupWarnings\"]}\", \"{\"t\":{\"$date\":\"2023-10-08T10:28:11.902+08:00\"},\"s\":\"W\", \"c\":\"CONTROL\", \"id\":22178, \"ctx\":\"initandlisten\",\"msg\":\"/sys/kernel/mm/transparent_hugepage/enabled is 'always'. We suggest setting it to 'never'\",\"tags\":[\"startupWarnings\"]}\", \"{\"t\":{\"$date\":\"2023-10-08T10:28:11.902+08:00\"},\"s\":\"W\", \"c\":\"CONTROL\", \"id\":22181, \"ctx\":\"initandlisten\",\"msg\":\"/sys/kernel/mm/transparent_hugepage/defrag is 'always'. We suggest setting it to 'never'\",\"tags\":[\"startupWarnings\"]}\"], \"ok\": 1 }", response_code: 0, exception: "", rrt: 334 } is_mongo: false
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 13, response_id: 0, op_code: 2013, op_code_name: "OP_MSG", request: "{ \"isMaster\": 1, \"forShell\": 1, \"$db\": \"test\" }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 13, response_id: 73, op_code: 2013, op_code_name: "OP_MSG", request: "", response: "{ \"ismaster\": true, \"topologyVersion\": { \"processId\": ObjectId(\"652213ba46c335fa2820b0dc\"), \"counter\": 0 }, \"maxBsonObjectSize\": 16777216, \"maxMessageSizeBytes\": 48000000, \"maxWriteBatchSize\": 100000, \"localTime\": DateTime(\"2023-10-08 2:46:26.793 +00:00:00\"), \"logicalSessionTimeoutMinutes\": 30, \"connectionId\": 3, \"minWireVersion\": 0, \"maxWireVersion\": 9, \"readOnly\": false, \"ok\": 1 }", response_code: 0, exception: "", rrt: 189 } is_mongo: false
MongoDBInfo { msg_type: Request, is_tls: false, req_len: 0, resp_len: 0, request_id: 14, response_id: 0, op_code: 2013, op_code_name: "OP_MSG", request: "{ \"endSessions\": [{ \"id\": Binary(0x4, uU9EjcLlRI+tnzaqrJqWqQ==) }], \"$db\": \"admin\" }", response: "", response_code: 0, exception: "", rrt: 0 } is_mongo: true
MongoDBInfo { msg_type: Response, is_tls: false, req_len: 0, resp_len: 0, request_id: 14, response_id: 74, op_code: 2013, op_code_name: "OP_MSG", request: "", response: "{ \"ok\": 1 }", response_code: 0, exception: "", rrt: 786 } is_mongo: false
59 changes: 27 additions & 32 deletions agent/src/collector/l7_quadruple_generator.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ impl SubQuadGen {
flow: flow.clone(),
l7_protocol: meter.l7_protocol,
endpoint_hash,
endpoint: meter.endpoint.clone(),
endpoint: meter.endpoint,
is_active_host0,
is_active_host1,
time_in_second: tagged_flow.flow.flow_stat_time,
Expand Down Expand Up @@ -549,8 +549,8 @@ impl L7QuadrupleGenerator {
) {
let mut second_inject = false;
let mut minute_inject = false;
if let Some(s) = self.second_quad_gen.as_mut() {
if config.vtap_flow_1s_enabled {
if config.vtap_flow_1s_enabled {
if let Some(s) = self.second_quad_gen.as_mut() {
second_inject = s.move_window(time_in_second);
}
}
Expand All @@ -565,7 +565,11 @@ impl L7QuadrupleGenerator {

let endpoint_hash = hash_endpoint(&l7_stats.endpoint) as u32;

let app_meter = Self::generate_app_meter(&l7_stats, config.l7_metrics_enabled);
let app_meter = if config.l7_metrics_enabled {
Self::generate_app_meter(&l7_stats)
} else {
AppMeter::default()
};

if second_inject {
self.second_quad_gen.as_mut().unwrap().inject_app_meter(
Expand All @@ -588,12 +592,7 @@ impl L7QuadrupleGenerator {
}
}

fn generate_app_meter(l7_stats: &L7Stats, l7_metrics_enabled: bool) -> AppMeter {
let mut app_meter = AppMeter::default();

if !l7_metrics_enabled {
return app_meter;
}
fn generate_app_meter(l7_stats: &L7Stats) -> AppMeter {
let (close_type, direction_score) = if let Some(tagged_flow) = &l7_stats.flow {
(
tagged_flow.flow.close_type,
Expand All @@ -609,7 +608,7 @@ impl L7QuadrupleGenerator {
SignalSource::Packet | SignalSource::EBPF | SignalSource::XFlow,
) => {
// only L7Protocol is Unknown or Other and SignalSource != Otel will execute the following logic
app_meter = AppMeter {
AppMeter {
traffic: AppTraffic {
request: (close_type != CloseType::ForcedReport) as u32,
response: (close_type != CloseType::ForcedReport) as u32,
Expand All @@ -618,28 +617,24 @@ impl L7QuadrupleGenerator {
..Default::default()
}
}
(_, _) => {
app_meter = AppMeter {
traffic: AppTraffic {
request: stats.request_count,
response: stats.response_count,
direction_score: direction_score,
},
latency: AppLatency {
rrt_max: stats.rrt_max,
rrt_sum: stats.rrt_sum as u64,
rrt_count: stats.rrt_count,
},
anomaly: AppAnomaly {
client_error: stats.err_client_count,
server_error: stats.err_server_count,
timeout: stats.err_timeout,
},
};
}
(_, _) => AppMeter {
traffic: AppTraffic {
request: stats.request_count,
response: stats.response_count,
direction_score: direction_score,
},
latency: AppLatency {
rrt_max: stats.rrt_max,
rrt_sum: stats.rrt_sum as u64,
rrt_count: stats.rrt_count,
},
anomaly: AppAnomaly {
client_error: stats.err_client_count,
server_error: stats.err_server_count,
timeout: stats.err_timeout,
},
},
}

app_meter
}

fn handler_routine(&mut self) {
Expand Down
10 changes: 3 additions & 7 deletions agent/src/flow_generator/flow_map.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ use super::{

use crate::{
common::{
ebpf::EbpfType,
endpoint::{
EndpointData, EndpointDataPov, EndpointInfo, EPC_FROM_DEEPFLOW, EPC_FROM_INTERNET,
},
Expand Down Expand Up @@ -1009,12 +1008,6 @@ impl FlowMap {

fn init_flow(&mut self, config: &Config, meta_packet: &mut MetaPacket) -> Box<FlowNode> {
let flow_config = config.flow;
match meta_packet.ebpf_type {
EbpfType::GoHttp2Uprobe | EbpfType::GoHttp2UprobeData => {}
_ => {
meta_packet.lookup_key.direction = PacketDirection::ClientToServer;
}
}

let mut tagged_flow = TaggedFlow::default();
let lookup_key = &meta_packet.lookup_key;
Expand Down Expand Up @@ -1574,6 +1567,9 @@ impl FlowMap {
node.flow_state = FlowState::Established;
// opening timeout
node.timeout = config.flow.flow_timeout.opening;
if let Some(meta_flow_log) = node.meta_flow_log.as_mut() {
let _ = meta_flow_log.parse_l3(meta_packet);
}
node
}

Expand Down
Loading

0 comments on commit 2a8f3b2

Please sign in to comment.