From 0f3c45b165017653eaa3fb00294f464ac80aca58 Mon Sep 17 00:00:00 2001 From: gaoyixiang1 <1739037263@qq.com> Date: Thu, 18 Jan 2024 18:36:53 +0800 Subject: [PATCH 1/5] =?UTF-8?q?C++=E7=AC=A6=E5=8F=B7=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Stack_Analyser/libbpf/stack_analyzer.cc | 609 +++++++++++------- 1 file changed, 377 insertions(+), 232 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/libbpf/stack_analyzer.cc b/eBPF_Supermarket/Stack_Analyser/libbpf/stack_analyzer.cc index e36129134..6c4a761c5 100644 --- a/eBPF_Supermarket/Stack_Analyser/libbpf/stack_analyzer.cc +++ b/eBPF_Supermarket/Stack_Analyser/libbpf/stack_analyzer.cc @@ -22,6 +22,9 @@ #include #include #include +#include +#include +#include #include "rapidjson/document.h" #include "rapidjson/filewritestream.h" @@ -29,7 +32,9 @@ #include "symbol.h" #include "clipp.h" -extern "C" { + +extern "C" +{ #include #include #include @@ -49,15 +54,41 @@ extern "C" { // 模板用来统一调用多个类有同样但未被抽象的接口 // 虚函数用来规范接口来被统一调用 -class StackCollector { + + +std::string demangle(const char *symbol) { + size_t size = 0; + int status = 0; + char *demangled = abi::__cxa_demangle(symbol, NULL, &size, &status); + + if (status == 0 && demangled != NULL) { + std::string result; + for(size_t i=0;i *sortedCountList(void) { - if (value_fd < 0) { + std::vector *sortedCountList(void) + { + if (value_fd < 0) + { return NULL; } std::vector *D = new std::vector(); - for (psid prev = {0}, id; !bpf_map_get_next_key(value_fd, &prev, &id); prev = id) { + for (psid prev = {0}, id; !bpf_map_get_next_key(value_fd, &prev, &id); prev = id) + { bpf_map_lookup_elem(value_fd, &id, data_buf); CountItem d(id.pid, id.ksid, id.usid, data_value()); D->insert(std::lower_bound(D->begin(), D->end(), d), d); @@ -93,8 +128,8 @@ class StackCollector { int comm_fd = -1; // pid-进程名表的文件描述符 int trace_fd = -1; // 栈id-栈轨迹表的文件描述符 - void *data_buf = NULL; // 用于存储单个指标值的缓冲区 - + void *data_buf = NULL; // 用于存储单个指标值的缓冲区 + bool showDelta = true; /// @brief 将缓冲区的数据解析为特定值 @@ -107,31 +142,33 @@ class StackCollector { /// @return 字符串 virtual std::string data_str(uint64_t f) { return "value:" + std::to_string(f); }; - #define declareEBPF(eBPFName) \ +#define declareEBPF(eBPFName) \ struct eBPFName *skel = NULL; public: - std::string name; // 标识类名 + std::string name; // 标识类名 + + int pid = -1; // 用于设置ebpf程序跟踪的pid + int cpu = -1; // 用于设置ebpf程序跟踪的cpu + int err = 0; // 用于保存错误代码 - int pid = -1; // 用于设置ebpf程序跟踪的pid - int cpu = -1; // 用于设置ebpf程序跟踪的cpu - int err = 0; // 用于保存错误代码 - - bool ustack = true; // 是否跟踪用户栈 - bool kstack = true; // 是否跟踪内核栈 - uint64_t min = 0; - uint64_t max = __UINT64_MAX__; // 设置采集指标最大值，最小值 + bool ustack = true; // 是否跟踪用户栈 + bool kstack = true; // 是否跟踪内核栈 + uint64_t min = 0; + uint64_t max = __UINT64_MAX__; // 设置采集指标最大值，最小值 - bool clear = false; // 清除已输出的指标积累量 + bool clear = false; // 清除已输出的指标积累量 int self_pid; - StackCollector() { + StackCollector() + { self_pid = getpid(); data_buf = new uint64_t(0); }; - virtual ~StackCollector() { + virtual ~StackCollector() + { delete (uint64_t *)data_buf; }; @@ -139,52 +176,60 @@ class StackCollector { /// @param 无 /// @return 成功则返回0，否则返回负数 virtual int load(void) = 0; - #define defaultLoad \ - int load(void) override { \ - StackProgLoadOpen( \ - skel->bss->apid = pid \ - ); \ - return 0; \ +#define defaultLoad \ + int load(void) override \ + { \ + StackProgLoadOpen( \ + skel->bss->apid = pid); \ + return 0; \ }; /// @brief 将ebpf程序挂载到跟踪点上 /// @param 无 /// @return 成功则返回0，否则返回负数 virtual int attach(void) = 0; - #define defaultAttach \ - int attach(void) override { \ - err = skel->attach(skel); \ - CHECK_ERR(err, "Failed to attach BPF skeleton");\ - return 0; \ +#define defaultAttach \ + int attach(void) override \ + { \ + err = skel->attach(skel); \ + CHECK_ERR(err, "Failed to attach BPF skeleton"); \ + return 0; \ }; /// @brief 断开ebpf的跟踪点和处理函数间的连接 /// @param 无 virtual void detach(void) = 0; - #define defaultDetach \ - void detach(void) override {\ - if (skel) { \ - skel->detach(skel); \ - } \ +#define defaultDetach \ + void detach(void) override \ + { \ + if (skel) \ + { \ + skel->detach(skel); \ + } \ }; /// @brief 卸载ebpf程序 /// @param 无 virtual void unload(void) = 0; - #define defaultUnload \ - void unload(void) override {\ - if (skel) { \ - skel->destroy(skel);\ - } \ - skel = NULL; \ +#define defaultUnload \ + void unload(void) override \ + { \ + if (skel) \ + { \ + skel->destroy(skel); \ + } \ + skel = NULL; \ }; /// @brief 清除count map的数据 /// @param 无 - void check_clear_count(void) { - if(!showDelta) return; + void check_clear_count(void) + { + if (!showDelta) + return; uint c = MAX_ENTRIES; - for (psid prev = {0}, id; c && !bpf_map_get_next_key(value_fd, &prev, &id); c--, prev = id) { + for (psid prev = {0}, id; c && !bpf_map_get_next_key(value_fd, &prev, &id); c--, prev = id) + { bpf_map_delete_elem(value_fd, &id); } } @@ -194,7 +239,8 @@ class StackCollector { void print_list(void) { auto D = sortedCountList(); - for (auto id : *D) { + for (auto id : *D) + { printf("pid:%-6d\tusid:%-6d\tksid:%-6d\t%s\n", id.pid, id.usid, id.ksid, data_str(id.val).c_str()); } delete D; @@ -210,11 +256,12 @@ class StackCollector { CHECK_ERR_VALUE(comm_fd < 0, nullptr, "comm map open failure"); // std::filebuf DataFileBuf; // const std::string DataFileName = name + "_stack_data.log"; - // CHECK_ERR(DataFileBuf.open(DataFileName, std::ios::app) == nullptr, "data file open failed"); + // CHECK_ERR(DataFileBuf.open(DataFileName, std::ios::app) == nullptr, "data file open failed"); // std::ostream DataText(&DataFileBuf); auto DataTextP = new std::ostringstream(); auto &DataText = *DataTextP; - for (psid prev = {}, id; !bpf_map_get_next_key(value_fd, &prev, &id); prev = id) { + for (psid prev = {}, id; !bpf_map_get_next_key(value_fd, &prev, &id); prev = id) + { { char cmd[COMM_LEN]; bpf_map_lookup_elem(comm_fd, &id.pid, cmd); @@ -222,26 +269,49 @@ class StackCollector { } symbol sym; uint64_t ip[MAX_STACKS]; - if (id.usid >= 0) { + if (id.usid >= 0) + { bpf_map_lookup_elem(trace_fd, &id.usid, ip); std::string *s = 0, symbol; elf_file file; - uint64_t *p = ip + MAX_STACKS -1; - for(; !*p; p--); - for (; p >= ip; p--) { + uint64_t *p = ip + MAX_STACKS - 1; + for (; !*p; p--) + ; + for (; p >= ip; p--) + { uint64_t &addr = *p; sym.reset(addr); - if (g_symbol_parser.find_symbol_in_cache(id.pid, addr, symbol)) { + if (g_symbol_parser.find_symbol_in_cache(id.pid, addr, symbol)) + { s = &symbol; + if ((*s)[0] == '_' && (*s)[1] == 'Z')//代表是C++符号，则调用demangle解析 + { + *s = demangle(symbol.c_str()); + } DataText << *s << ';'; - } else if (g_symbol_parser.get_symbol_info(id.pid, sym, file) && - g_symbol_parser.find_elf_symbol(sym, file, id.pid, id.pid)) { + } + else if (g_symbol_parser.get_symbol_info(id.pid, sym, file) && + g_symbol_parser.find_elf_symbol(sym, file, id.pid, id.pid)) + { std::stringstream ss(""); ss << "+0x" << std::hex << (addr - sym.ip); + if (sym.name[0] == '_' && sym.name[1] == 'Z')//代表是C++符号，则调用demangle解析 + { + sym.name = demangle(sym.name.c_str()); + } + // int sym_len = sym.name.length(); + // if (sym.name[sym_len-1] == ')' && sym.name[sym_len-2] == '(')//代表函数名加了括号 + // { + // DataText << ""; + // }else{ + // sym.name+="()"; + // } sym.name += ss.str(); DataText << sym.name << ';'; g_symbol_parser.putin_symbol_cache(id.pid, addr, sym.name); - } else { + } + else + { std::stringstream ss(""); ss << "0x" << std::hex << addr; auto addr_str = ss.str(); @@ -249,20 +319,39 @@ class StackCollector { g_symbol_parser.putin_symbol_cache(id.pid, addr, addr_str); } } - } else { + } + else + { DataText << "[MISSING USER STACK];"; } DataText << "---------;"; - if (id.ksid >= 0) { + if (id.ksid >= 0) + { bpf_map_lookup_elem(trace_fd, &id.ksid, ip); uint64_t *p = ip + MAX_STACKS - 1; - for(; !*p; p--); - for (; p >= ip; p--) { + for (; !*p; p--) + ; + for (; p >= ip; p--) + { uint64_t &addr = *p; sym.reset(addr); - if (g_symbol_parser.find_kernel_symbol(sym)) { + if (g_symbol_parser.find_kernel_symbol(sym)) + { + if (sym.name[0] == '_' && sym.name[1] == 'Z')//代表是C++符号，则调用demangle解析 + { + sym.name = demangle(sym.name.c_str()); + } + // int sym_len = sym.name.length(); + // if (sym.name[sym_len-1] == ')' && sym.name[sym_len-2] == '(')//代表函数名加了括号 + // { + // DataText << ""; + // }else{ + // sym.name+="()"; + // } DataText << sym.name << ';'; - } else { + } + else + { std::stringstream ss(""); ss << "0x" << std::hex << addr; auto addr_str = ss.str(); @@ -270,7 +359,9 @@ class StackCollector { g_symbol_parser.putin_symbol_cache(pid, addr, addr_str); } } - } else { + } + else + { DataText << "[MISSING KERNEL STACK];"; } bpf_map_lookup_elem(value_fd, &id, data_buf); @@ -280,7 +371,8 @@ class StackCollector { } }; -class OnCPUStackCollector : public StackCollector { +class OnCPUStackCollector : public StackCollector +{ private: declareEBPF(on_cpu_count_bpf); const char *online_cpus_file = "/sys/devices/system/cpu/online"; @@ -292,7 +384,8 @@ class OnCPUStackCollector : public StackCollector { public: unsigned long long freq = 49; - OnCPUStackCollector() { + OnCPUStackCollector() + { name = "on_cpu"; err = parse_cpu_mask_file(online_cpus_file, &online_mask, &num_online_cpus); CHECK_ERR_EXIT(err, "Fail to get online CPU numbers"); @@ -302,19 +395,19 @@ class OnCPUStackCollector : public StackCollector { std::string data_str(uint64_t f) override { return "counts:" + std::to_string(f); }; - int load(void) override { + int load(void) override + { FILE *fp = popen("cat /proc/kallsyms | grep \" avenrun\"", "r"); CHECK_ERR(!fp, "Failed to draw flame graph"); unsigned long *load_a; fscanf(fp, "%p", &load_a); pclose(fp); StackProgLoadOpen( - skel->bss->load_a = load_a - ) - return 0; + skel->bss->load_a = load_a) return 0; }; - int attach(void) override { + int attach(void) override + { attr = { .type = PERF_TYPE_SOFTWARE, // hardware event can't be used .size = sizeof(attr), @@ -324,13 +417,16 @@ class OnCPUStackCollector : public StackCollector { .freq = 1, // use freq instead of period }; pefds = (int *)malloc(num_cpus * sizeof(int)); - for (int i = 0; i < num_cpus; i++) { + for (int i = 0; i < num_cpus; i++) + { pefds[i] = -1; } links = (struct bpf_link **)calloc(num_cpus, sizeof(struct bpf_link *)); - for (int cpu = 0; cpu < num_cpus; cpu++) { + for (int cpu = 0; cpu < num_cpus; cpu++) + { /* skip offline/not present CPUs */ - if (cpu >= num_online_cpus || !online_mask[cpu]) { + if (cpu >= num_online_cpus || !online_mask[cpu]) + { continue; } /* Set up performance monitoring on a CPU/Core */ @@ -344,17 +440,23 @@ class OnCPUStackCollector : public StackCollector { return 0; } - void detach(void) override { - if (links) { - for (int cpu = 0; cpu < num_cpus; cpu++) { + void detach(void) override + { + if (links) + { + for (int cpu = 0; cpu < num_cpus; cpu++) + { bpf_link__destroy(links[cpu]); } free(links); links = NULL; } - if (pefds) { - for (int i = 0; i < num_cpus; i++) { - if (pefds[i] >= 0) { + if (pefds) + { + for (int i = 0; i < num_cpus; i++) + { + if (pefds[i] >= 0) + { close(pefds[i]); } } @@ -366,20 +468,24 @@ class OnCPUStackCollector : public StackCollector { defaultUnload; }; -class OffCPUStackCollector : public StackCollector{ +class OffCPUStackCollector : public StackCollector +{ private: declareEBPF(off_cpu_count_bpf); + protected: std::string data_str(uint64_t f) override { return "time(ms):" + std::to_string(f); }; defaultLoad; defaultAttach; defaultDetach; defaultUnload; + public: OffCPUStackCollector() { name = "off-cpu"; }; }; -class MemoryStackCollector : public StackCollector { +class MemoryStackCollector : public StackCollector +{ private: declareEBPF(mem_count_bpf); @@ -388,15 +494,22 @@ class MemoryStackCollector : public StackCollector { public: char *object = (char *)"libc.so.6"; - - MemoryStackCollector() { kstack = false; name = "memory"; showDelta = false; }; - int load(void) override { + MemoryStackCollector() + { + kstack = false; + name = "memory"; + showDelta = false; + }; + + int load(void) override + { StackProgLoadOpen(); return 0; }; - int attach(void) override { + int attach(void) override + { ATTACH_UPROBE_CHECKED(skel, malloc, malloc_enter); ATTACH_URETPROBE_CHECKED(skel, malloc, malloc_exit); ATTACH_UPROBE_CHECKED(skel, calloc, calloc_enter); @@ -414,12 +527,14 @@ class MemoryStackCollector : public StackCollector { return 0; }; - void detach(void) override { + void detach(void) override + { skel->detach(skel); - #define destoryBPFLinkIfExist(name) \ - if(skel->links.name) { \ - bpf_link__destroy(skel->links.name); \ - } +#define destoryBPFLinkIfExist(name) \ + if (skel->links.name) \ + { \ + bpf_link__destroy(skel->links.name); \ + } destoryBPFLinkIfExist(malloc_enter); destoryBPFLinkIfExist(malloc_exit); destoryBPFLinkIfExist(calloc_enter); @@ -433,21 +548,25 @@ class MemoryStackCollector : public StackCollector { }; defaultUnload; - }; -class IOStackCollector : public StackCollector { +class IOStackCollector : public StackCollector +{ private: declareEBPF(io_count_bpf); + protected: - std::string data_str(uint64_t f) override { + std::string data_str(uint64_t f) override + { const std::string IOScale[] = {"counts", "size(B)", "aver(B/1)"}; return IOScale[DataType] + ":" + std::to_string(f); }; - uint64_t data_value() override { + uint64_t data_value() override + { io_tuple *p = (io_tuple *)data_buf; - switch (DataType) { + switch (DataType) + { case AVE: return p->size / p->count; case SIZE: @@ -462,33 +581,37 @@ class IOStackCollector : public StackCollector { public: io_mod DataType = io_mod::COUNT; - IOStackCollector() { + IOStackCollector() + { delete (uint64_t *)data_buf; data_buf = new io_tuple{0}; name = "io"; }; - - ~IOStackCollector() override { + ~IOStackCollector() override + { delete (io_tuple *)data_buf; }; defaultLoad; defaultAttach; defaultDetach; - defaultUnload; - + defaultUnload; }; -class ReadaheadStackCollector : public StackCollector { +class ReadaheadStackCollector : public StackCollector +{ private: declareEBPF(pre_count_bpf); + protected: - std::string data_str(uint64_t f) override { - return "rest_pages:" + std::to_string(f); + std::string data_str(uint64_t f) override + { + return "rest_pages:" + std::to_string(f); }; - uint64_t data_value() override { + uint64_t data_value() override + { ra_tuple *p = (ra_tuple *)data_buf; return p->expect - p->truth; }; @@ -499,121 +622,124 @@ class ReadaheadStackCollector : public StackCollector { defaultDetach; defaultUnload; - ReadaheadStackCollector() { + ReadaheadStackCollector() + { delete (uint64_t *)data_buf; data_buf = new ra_tuple{0}; name = "readahead"; showDelta = false; }; - ~ReadaheadStackCollector() override { + ~ReadaheadStackCollector() override + { delete (ra_tuple *)data_buf; } }; -namespace MainConfig { - int run_time = __INT_MAX__; // 运行时间 - unsigned delay = 5; // 设置输出间隔 - display_t d_mode = display_t::NO_OUTPUT; // 设置显示模式 +namespace MainConfig +{ + int run_time = __INT_MAX__; // 运行时间 + unsigned delay = 5; // 设置输出间隔 + display_t d_mode = display_t::NO_OUTPUT; // 设置显示模式 std::string command = ""; int32_t target_pid = -1; std::string server_address = "127.0.0.1:12345"; }; -std::vector StackCollectorList; -void endCollect(void) { +std::vector StackCollectorList; +void endCollect(void) +{ signal(SIGINT, SIG_IGN); - for(auto Item : StackCollectorList) { - if(MainConfig::run_time > 0) { + for (auto Item : StackCollectorList) + { + if (MainConfig::run_time > 0) + { Item->format(); } Item->detach(); Item->unload(); } - if (MainConfig::command.length()) { + if (MainConfig::command.length()) + { kill(MainConfig::target_pid, SIGTERM); } } uint64_t optbuff; -int main(int argc, char *argv[]) { - auto MainOption = ( - ( - ((clipp::option("-p", "--pid") & clipp::value("pid of sampled process, default -1 for all", MainConfig::target_pid)) % "set pid of process to monitor") | - ((clipp::option("-c", "--command") & clipp::value("to be sampled command to run, default none", MainConfig::command)) % "set command for monitoring the whole life") - ), - (clipp::option("-d", "--delay") & clipp::value("delay time(seconds) to output, default 5", MainConfig::delay)) % "set the interval to output", - clipp::option("-l", "--realtime-list").set(MainConfig::d_mode, LIST_OUTPUT) % "output in console, default false", - clipp::option("-t", "--timeout") & clipp::value("run time, default nearly infinite", MainConfig::run_time) % "set the total simpling time", - clipp::option("-s", "--server") & clipp::value("server address, default 127.0.0.1:12345", MainConfig::server_address) % "set the server address" - ); - - auto SubOption = ( - clipp::option("-U", "--user-stack-only").call([]{ - StackCollectorList.back()->kstack = false; - }) % "only sample user stacks", - clipp::option("-K", "--kernel-stack-only").call([]{ - StackCollectorList.back()->ustack = false; - }) % "only sample kernel stacks", - (clipp::option("-m", "--max-value") & clipp::value("max threshold of sampled value", optbuff).call([]{ - StackCollectorList.back()->max = optbuff; - })) % "set the max threshold of sampled value", - (clipp::option("-n", "--min-value") & clipp::value("min threshold of sampled value", optbuff).call([]{ - StackCollectorList.back()->min = optbuff; - })) % "set the min threshold of sampled value" - ); - - auto OnCpuOption = clipp::option("on-cpu").call([]{ - StackCollectorList.push_back(new OnCPUStackCollector()); - }) % "sample the call stacks of on-cpu processes" & ( - clipp::option("-F", "--frequency") & clipp::value("sampling frequency", optbuff).call([]{ - static_cast(StackCollectorList.back())->freq = optbuff; - }) % "sampling at a set frequency", - SubOption - ); - - auto OffCpuOption = clipp::option("off-cpu").call([]{ - StackCollectorList.push_back(new OffCPUStackCollector()); - }) % "sample the call stacks of off-cpu processes" & SubOption; - - auto MemoryOption = clipp::option("mem").call([]{ - StackCollectorList.push_back(new MemoryStackCollector()); - }) % "sample the memory usage of call stacks" & SubOption; - - auto IOOption = clipp::option("io").call([]{ - StackCollectorList.push_back(new IOStackCollector()); - }) % "sample the IO data volume of call stacks" & ( - (clipp::option("--mod") & ( - clipp::option("count").call([]{ - static_cast(StackCollectorList.back())->DataType = COUNT; - }) % "Counting the number of I/O operations" | - clipp::option("ave").call([]{ - static_cast(StackCollectorList.back())->DataType = AVE; - }) % "Counting the ave of I/O operations" | - clipp::option("size").call([]{ - static_cast(StackCollectorList.back())->DataType = SIZE; - }) % "Counting the size of I/O operations" - )) % "set the statistic mod", - SubOption - ); - - auto ReadaheadOption = clipp::option("ra").call([]{ - StackCollectorList.push_back(new ReadaheadStackCollector()); - }) % "sample the readahead hit rate of call stacks" & SubOption; - - auto cli = ( - MainOption, - clipp::option("-v", "--version").call([] { - std::cout << "verion 2.0\n\n"; - }) % "show version", - OnCpuOption, - OffCpuOption, - MemoryOption, - IOOption, - ReadaheadOption - ) % "statistic call trace relate with some metrics"; - - if (!clipp::parse(argc, argv, cli)) { +int main(int argc, char *argv[]) +{ + auto MainOption = (( + ((clipp::option("-p", "--pid") & clipp::value("pid of sampled process, default -1 for all", MainConfig::target_pid)) % "set pid of process to monitor") | + ((clipp::option("-c", "--command") & clipp::value("to be sampled command to run, default none", MainConfig::command)) % "set command for monitoring the whole life")), + (clipp::option("-d", "--delay") & clipp::value("delay time(seconds) to output, default 5", MainConfig::delay)) % "set the interval to output", + clipp::option("-l", "--realtime-list").set(MainConfig::d_mode, LIST_OUTPUT) % "output in console, default false", + clipp::option("-t", "--timeout") & clipp::value("run time, default nearly infinite", MainConfig::run_time) % "set the total simpling time", + clipp::option("-s", "--server") & clipp::value("server address, default 127.0.0.1:12345", MainConfig::server_address) % "set the server address"); + + auto SubOption = (clipp::option("-U", "--user-stack-only").call([] + { StackCollectorList.back()->kstack = false; }) % + "only sample user stacks", + clipp::option("-K", "--kernel-stack-only").call([] + { StackCollectorList.back()->ustack = false; }) % + "only sample kernel stacks", + (clipp::option("-m", "--max-value") & clipp::value("max threshold of sampled value", optbuff).call([] + { StackCollectorList.back()->max = optbuff; })) % + "set the max threshold of sampled value", + (clipp::option("-n", "--min-value") & clipp::value("min threshold of sampled value", optbuff).call([] + { StackCollectorList.back()->min = optbuff; })) % + "set the min threshold of sampled value"); + + auto OnCpuOption = clipp::option("on-cpu").call([] + { StackCollectorList.push_back(new OnCPUStackCollector()); }) % + "sample the call stacks of on-cpu processes" & + (clipp::option("-F", "--frequency") & clipp::value("sampling frequency", optbuff).call([] + { static_cast(StackCollectorList.back())->freq = optbuff; }) % + "sampling at a set frequency", + SubOption); + + auto OffCpuOption = clipp::option("off-cpu").call([] + { StackCollectorList.push_back(new OffCPUStackCollector()); }) % + "sample the call stacks of off-cpu processes" & + SubOption; + + auto MemoryOption = clipp::option("mem").call([] + { StackCollectorList.push_back(new MemoryStackCollector()); }) % + "sample the memory usage of call stacks" & + SubOption; + + auto IOOption = clipp::option("io").call([] + { StackCollectorList.push_back(new IOStackCollector()); }) % + "sample the IO data volume of call stacks" & + ((clipp::option("--mod") & (clipp::option("count").call([] + { static_cast(StackCollectorList.back())->DataType = COUNT; }) % + "Counting the number of I/O operations" | + clipp::option("ave").call([] + { static_cast(StackCollectorList.back())->DataType = AVE; }) % + "Counting the ave of I/O operations" | + clipp::option("size").call([] + { static_cast(StackCollectorList.back())->DataType = SIZE; }) % + "Counting the size of I/O operations")) % + "set the statistic mod", + SubOption); + + auto ReadaheadOption = clipp::option("ra").call([] + { StackCollectorList.push_back(new ReadaheadStackCollector()); }) % + "sample the readahead hit rate of call stacks" & + SubOption; + + auto cli = (MainOption, + clipp::option("-v", "--version").call([] + { std::cout << "verion 2.0\n\n"; }) % + "show version", + OnCpuOption, + OffCpuOption, + MemoryOption, + IOOption, + ReadaheadOption) % + "statistic call trace relate with some metrics"; + + if (!clipp::parse(argc, argv, cli)) + { std::cout << clipp::make_man_page(cli, argv[0]) << '\n'; return 0; } @@ -621,32 +747,42 @@ int main(int argc, char *argv[]) { uint64_t eventbuff = 1; int child_exec_event_fd = eventfd(0, EFD_CLOEXEC); CHECK_ERR(child_exec_event_fd < 0, "failed to create event fd"); - if(MainConfig::command.length()) { + if (MainConfig::command.length()) + { MainConfig::target_pid = fork(); - switch(MainConfig::target_pid) { - case -1: { - std::cout << "command create failed." << std::endl; - return -1; - } case 0: { - const auto bytes = read(child_exec_event_fd, &eventbuff, sizeof(eventbuff)); - CHECK_ERR( bytes < 0, "failed to read from fd %ld", bytes) - else CHECK_ERR(bytes != sizeof(eventbuff), "read unexpected size %ld", bytes); - printf("child exec %s\n", MainConfig::command.c_str()); - CHECK_ERR_EXIT(execl("/bin/bash", "bash", "-c", MainConfig::command.c_str(), NULL), "failed to execute child command"); - break; - } default: { - printf("create child %d\n", MainConfig::target_pid); - break; - } + switch (MainConfig::target_pid) + { + case -1: + { + std::cout << "command create failed." << std::endl; + return -1; + } + case 0: + { + const auto bytes = read(child_exec_event_fd, &eventbuff, sizeof(eventbuff)); + CHECK_ERR(bytes < 0, "failed to read from fd %ld", bytes) + else CHECK_ERR(bytes != sizeof(eventbuff), "read unexpected size %ld", bytes); + printf("child exec %s\n", MainConfig::command.c_str()); + CHECK_ERR_EXIT(execl("/bin/bash", "bash", "-c", MainConfig::command.c_str(), NULL), "failed to execute child command"); + break; + } + default: + { + printf("create child %d\n", MainConfig::target_pid); + break; + } } } - for(auto Item = StackCollectorList.begin(); Item != StackCollectorList.end(); ) { + for (auto Item = StackCollectorList.begin(); Item != StackCollectorList.end();) + { (*Item)->pid = MainConfig::target_pid; - if ((*Item)->load()) { + if ((*Item)->load()) + { goto err; } - if ((*Item)->attach()) { + if ((*Item)->attach()) + { goto err; } Item++; @@ -658,26 +794,31 @@ int main(int argc, char *argv[]) { Item = StackCollectorList.erase(Item); } - if(MainConfig::command.length()) { + if (MainConfig::command.length()) + { printf("wake up child\n"); write(child_exec_event_fd, &eventbuff, sizeof(eventbuff)); } printf("display mode: %d\n", MainConfig::d_mode); - // 创建 socket + // 创建 socket bool ToRemote = true; int clientSocket = socket(AF_INET, SOCK_STREAM, 0); - if (clientSocket == -1) { + if (clientSocket == -1) + { std::cerr << "Error creating socket" << std::endl; // return -1; ToRemote = false; - } else { + } + else + { // 服务器地址信息 sockaddr_in serverAddress; serverAddress.sin_family = AF_INET; auto ColonPos = MainConfig::server_address.find(':'); - if(ColonPos < 0) { + if (ColonPos < 0) + { std::cerr << "server address err" << std::endl; return 0; } @@ -686,7 +827,8 @@ int main(int argc, char *argv[]) { serverAddress.sin_port = htons(std::stoi(PortAddr)); inet_pton(AF_INET, IPAddr.c_str(), &serverAddress.sin_addr); // 连接到服务器 - if (connect(clientSocket, (struct sockaddr*)&serverAddress, sizeof(serverAddress)) == -1) { + if (connect(clientSocket, (struct sockaddr *)&serverAddress, sizeof(serverAddress)) == -1) + { std::cerr << "Error connecting to server" << std::endl; close(clientSocket); // return -1; @@ -694,32 +836,36 @@ int main(int argc, char *argv[]) { } } - - for(; MainConfig::run_time > 0 && (MainConfig::target_pid < 0 || !kill(MainConfig::target_pid, 0)); MainConfig::run_time -= MainConfig::delay) { - sleep(MainConfig::delay); // 模拟实时性 + for (; MainConfig::run_time > 0 && (MainConfig::target_pid < 0 || !kill(MainConfig::target_pid, 0)); MainConfig::run_time -= MainConfig::delay) + { + sleep(MainConfig::delay); // 模拟实时性 time_t timep; ::time(&timep); printf("%s", ctime(&timep)); - for(auto Item : StackCollectorList) { + for (auto Item : StackCollectorList) + { Item->detach(); // if(MainConfig::d_mode == display_t::LIST_OUTPUT) { // Item->print_list(); // } auto StreamData = Item->format(); - if(!StreamData) { + if (!StreamData) + { continue; } auto dataToSend = StreamData->str(); - if(ToRemote) { + if (ToRemote) + { // 发送数据到服务器 struct diy_header AHeader = { - .len = dataToSend.size() - }; + .len = dataToSend.size()}; strcpy(AHeader.name, Item->name.c_str()); send(clientSocket, &AHeader, sizeof(AHeader), 0); send(clientSocket, dataToSend.c_str(), AHeader.len, 0); - } else { + } + else + { Item->print_list(); std::ofstream fout; fout.open(Item->name + "_stack_data.txt", std::ios::out | std::ios::app); @@ -730,7 +876,6 @@ int main(int argc, char *argv[]) { Item->attach(); } - } // 关闭连接 close(clientSocket); From 7335b077579030d4344556ff994dabc1e5571fea Mon Sep 17 00:00:00 2001 From: gaoyixiang1 <1739037263@qq.com> Date: Wed, 24 Jan 2024 19:42:42 +0800 Subject: [PATCH 2/5] fix conflicts --- .../Stack_Analyser/libbpf/stack_analyzer.cc | 569 ++++-------------- 1 file changed, 130 insertions(+), 439 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/libbpf/stack_analyzer.cc b/eBPF_Supermarket/Stack_Analyser/libbpf/stack_analyzer.cc index 92c9b7263..b6dc50f2b 100644 --- a/eBPF_Supermarket/Stack_Analyser/libbpf/stack_analyzer.cc +++ b/eBPF_Supermarket/Stack_Analyser/libbpf/stack_analyzer.cc @@ -20,16 +20,11 @@ #include #include #include -#include -#include #include -#include -#include #include "symbol.h" #include "clipp.h" - extern "C" { #include @@ -46,7 +41,39 @@ extern "C" #include "bpf/pre_count.skel.h" } -std::string GetLocalDateTime(void) { +std::string demangleCppSym(std::string symbol) +{ + size_t size = 0; + int status = 0; + char *demangled = abi::__cxa_demangle(symbol.c_str(), NULL, &size, &status); + + if (status == 0 && demangled != NULL) + // 去除参数列表及括号 + { + char *func_name = demangled; + for (auto i = size - 1; i; i--) + { + if (demangled[i] == ' ') + { + for (char *p = demangled + i; *p; p++) + { + *p = p[1]; + } + } + } + std::string FuncName(func_name); + free(demangled); + return FuncName; + } + else + // 解码失败，返回原始符号 + { + return symbol; + } +} + +std::string getLocalDateTime(void) +{ auto t = time(NULL); auto localTm = localtime(&t); char buff[32]; @@ -57,29 +84,6 @@ std::string GetLocalDateTime(void) { // 模板用来统一调用多个类有同样但未被抽象的接口 // 虚函数用来规范接口来被统一调用 - - -std::string demangle(const char *symbol) { - size_t size = 0; - int status = 0; - char *demangled = abi::__cxa_demangle(symbol, NULL, &size, &status); - - if (status == 0 && demangled != NULL) { - std::string result; - for(size_t i=0;i *D = new std::vector(); - for (psid prev = {0}, id; !bpf_map_get_next_key(value_fd, &prev, &id); prev = id) - { - bpf_map_lookup_elem(value_fd, &id, data_buf); - CountItem d(id.pid, id.ksid, id.usid, data_value()); - auto keys = new psid[MAX_ENTRIES]; - auto vals = new char[MAX_ENTRIES*count_size]; + auto vals = new char[MAX_ENTRIES * count_size]; uint32_t count = MAX_ENTRIES; psid next_key; int err; - if(showDelta) { + if (showDelta) + { err = bpf_map_lookup_and_delete_batch(value_fd, NULL, &next_key, keys, vals, &count, NULL); - } else { + } + else + { err = bpf_map_lookup_batch(value_fd, NULL, &next_key, keys, vals, &count, NULL); } - if(err == EFAULT) { + if (err == EFAULT) + { return NULL; } auto D = new std::vector(); - for(uint32_t i = 0; i < count; i++) { - CountItem d(keys[i].pid, keys[i].ksid, keys[i].usid, data_value(vals + count_size*i)); + for (uint32_t i = 0; i < count; i++) + { + CountItem d(keys[i].pid, keys[i].ksid, keys[i].usid, data_value(vals + count_size * i)); D->insert(std::lower_bound(D->begin(), D->end(), d), d); } delete[] keys; @@ -152,8 +154,6 @@ class StackCollector int comm_fd = -1; // pid-进程名表的文件描述符 int trace_fd = -1; // 栈id-栈轨迹表的文件描述符 - void *data_buf = NULL; // 用于存储单个指标值的缓冲区 - size_t count_size = sizeof(uint32_t); bool showDelta = true; @@ -161,12 +161,12 @@ class StackCollector /// @brief 将缓冲区的数据解析为特定值 /// @param 无 /// @return 解析出的值 - virtual uint64_t data_value(void *data) { return *(uint32_t *)data; }; + virtual double data_value(void *data) { return *(uint32_t *)data; }; /// @brief 为特定值添加注解 /// @param f 特定值 /// @return 字符串 - virtual std::string data_str(uint64_t f) = 0; + virtual std::string data_str(void) = 0; #define declareEBPF(eBPFName) \ struct eBPFName *skel = NULL; @@ -190,14 +190,6 @@ class StackCollector StackCollector() { self_pid = getpid(); - - data_buf = new uint64_t(0); - }; - - virtual ~StackCollector() - { - delete (uint64_t *)data_buf; - }; /// @brief 负责ebpf程序的加载、参数设置和打开操作 @@ -249,198 +241,74 @@ class StackCollector skel = NULL; \ }; - - /// @brief 清除count map的数据 - /// @param 无 - void check_clear_count(void) - { - if (!showDelta) - return; - uint c = MAX_ENTRIES; - for (psid prev = {0}, id; c && !bpf_map_get_next_key(value_fd, &prev, &id); c--, prev = id) - { - bpf_map_delete_elem(value_fd, &id); - } - } - - /// @brief 打印count列表 - /// @param 无 - void print_list(void) - { - auto D = sortedCountList(); - for (auto id : *D) - { - printf("pid:%-6d\tusid:%-6d\tksid:%-6d\t%s\n", id.pid, id.usid, id.ksid, data_str(id.val).c_str()); - } - delete D; - } - - /// @brief 将表中的栈数据保存为火焰图 - /// @param 无 - /// @return 表未成功打开则返回负数 - std::ostringstream *format(void) + operator std::string() { - CHECK_ERR_VALUE(value_fd < 0, nullptr, "count map open failure"); - CHECK_ERR_VALUE(trace_fd < 0, nullptr, "trace map open failure"); - CHECK_ERR_VALUE(comm_fd < 0, nullptr, "comm map open failure"); - // std::filebuf DataFileBuf; - // const std::string DataFileName = name + "_stack_data.log"; - // CHECK_ERR(DataFileBuf.open(DataFileName, std::ios::app) == nullptr, "data file open failed"); - // std::ostream DataText(&DataFileBuf); - auto DataTextP = new std::ostringstream(); - auto &DataText = *DataTextP; - for (psid prev = {}, id; !bpf_map_get_next_key(value_fd, &prev, &id); prev = id) - { - { - char cmd[COMM_LEN]; - bpf_map_lookup_elem(comm_fd, &id.pid, cmd); - DataText << std::string(cmd) << ':' << std::to_string(id.pid) << ';'; - } - symbol sym; - uint64_t ip[MAX_STACKS]; - if (id.usid >= 0) - { - bpf_map_lookup_elem(trace_fd, &id.usid, ip); - std::string *s = 0, symbol; - elf_file file; - uint64_t *p = ip + MAX_STACKS - 1; - for (; !*p; p--) - ; - for (; p >= ip; p--) - { - uint64_t &addr = *p; - sym.reset(addr); - if (g_symbol_parser.find_symbol_in_cache(id.pid, addr, symbol)) - { - s = &symbol; - if ((*s)[0] == '_' && (*s)[1] == 'Z')//代表是C++符号，则调用demangle解析 - { - *s = demangle(symbol.c_str()); - } - DataText << *s << ';'; - } - else if (g_symbol_parser.get_symbol_info(id.pid, sym, file) && - g_symbol_parser.find_elf_symbol(sym, file, id.pid, id.pid)) - { - std::stringstream ss(""); - ss << "+0x" << std::hex << (addr - sym.ip); - if (sym.name[0] == '_' && sym.name[1] == 'Z')//代表是C++符号，则调用demangle解析 - { - sym.name = demangle(sym.name.c_str()); - } - // int sym_len = sym.name.length(); - // if (sym.name[sym_len-1] == ')' && sym.name[sym_len-2] == '(')//代表函数名加了括号 - // { - // DataText << ""; - // }else{ - // sym.name+="()"; - // } - sym.name += ss.str(); - DataText << sym.name << ';'; - g_symbol_parser.putin_symbol_cache(id.pid, addr, sym.name); - } - else - { - std::stringstream ss(""); - ss << "0x" << std::hex << addr; - auto addr_str = ss.str(); - DataText << addr_str << ';'; - g_symbol_parser.putin_symbol_cache(id.pid, addr, addr_str); - } - } - } - else - { - DataText << "[MISSING USER STACK];"; - } - DataText << "---------;"; - if (id.ksid >= 0) - { - bpf_map_lookup_elem(trace_fd, &id.ksid, ip); - uint64_t *p = ip + MAX_STACKS - 1; - for (; !*p; p--) - ; - for (; p >= ip; p--) - { - uint64_t &addr = *p; - sym.reset(addr); - if (g_symbol_parser.find_kernel_symbol(sym)) - { - if (sym.name[0] == '_' && sym.name[1] == 'Z')//代表是C++符号，则调用demangle解析 - { - sym.name = demangle(sym.name.c_str()); - } - // int sym_len = sym.name.length(); - // if (sym.name[sym_len-1] == ')' && sym.name[sym_len-2] == '(')//代表函数名加了括号 - // { - // DataText << ""; - // }else{ - // sym.name+="()"; - // } - DataText << sym.name << ';'; - } - else - { - std::stringstream ss(""); - ss << "0x" << std::hex << addr; - auto addr_str = ss.str(); - DataText << addr_str << ';'; - g_symbol_parser.putin_symbol_cache(pid, addr, addr_str); - } - } - } - else - { - DataText << "[MISSING KERNEL STACK];"; - - operator std::string() { std::ostringstream oss; - oss << "time:"; { - oss << GetLocalDateTime() << '\n'; + oss << "time:"; + { + oss << getLocalDateTime() << '\n'; } std::map> traces; - oss << "counts:\n"; { + oss << "counts:\n"; + { auto D = sortedCountList(); - if(!D) return oss.str(); - oss << "pid\tusid\tksid\t" << data_str(1).c_str() << '\n'; + if (!D) + return oss.str(); + oss << "pid\tusid\tksid\t" << data_str() << '\n'; uint64_t trace[MAX_STACKS], *p; - for (auto id : *D) { + for (auto id : *D) + { oss << id.pid << '\t' << id.usid << '\t' << id.ksid << '\t' << id.val << '\n'; - if(id.usid > 0 && traces.find(id.usid) == traces.end()) { + if (id.usid > 0 && traces.find(id.usid) == traces.end()) + { bpf_map_lookup_elem(trace_fd, &id.usid, trace); - for(p = trace + MAX_STACKS - 1; !*p; p--); - for (; p >= trace; p--) { + for (p = trace + MAX_STACKS - 1; !*p; p--) + ; + for (; p >= trace; p--) + { uint64_t &addr = *p; symbol sym; sym.reset(addr); elf_file file; - std::string symbol; - if (g_symbol_parser.find_symbol_in_cache(id.pid, addr, symbol)); + if (g_symbol_parser.find_symbol_in_cache(id.pid, addr, sym.name)) + ; else if (g_symbol_parser.get_symbol_info(id.pid, sym, file) && - g_symbol_parser.find_elf_symbol(sym, file, id.pid, id.pid)) { + g_symbol_parser.find_elf_symbol(sym, file, id.pid, id.pid)) + { + if (sym.name[0] == '_' && sym.name[1] == 'Z') + // 代表是C++符号，则调用demangle解析 + { + sym.name = demangleCppSym(sym.name); + } std::stringstream ss(""); ss << "+0x" << std::hex << (addr - sym.ip); sym.name += ss.str(); - symbol = sym.name; g_symbol_parser.putin_symbol_cache(id.pid, addr, sym.name); - } else { + } + else + { std::stringstream ss(""); ss << "0x" << std::hex << addr; - symbol = ss.str(); - g_symbol_parser.putin_symbol_cache(id.pid, addr, symbol); + sym.name = ss.str(); + g_symbol_parser.putin_symbol_cache(id.pid, addr, sym.name); } - traces[id.usid].push_back(symbol); + traces[id.usid].push_back(sym.name); } } - if(id.ksid > 0 && traces.find(id.ksid) == traces.end()) { + if (id.ksid > 0 && traces.find(id.ksid) == traces.end()) + { bpf_map_lookup_elem(trace_fd, &id.ksid, trace); - for(p = trace + MAX_STACKS - 1; !*p; p--); - for (; p >= trace; p--) { + for (p = trace + MAX_STACKS - 1; !*p; p--) + ; + for (; p >= trace; p--) + { uint64_t &addr = *p; symbol sym; sym.reset(addr); - if (g_symbol_parser.find_kernel_symbol(sym)); - else { + if (g_symbol_parser.find_kernel_symbol(sym)) + ; + else + { std::stringstream ss(""); ss << "0x" << std::hex << addr; sym.name = ss.str(); @@ -449,22 +317,26 @@ class StackCollector traces[id.ksid].push_back(sym.name); } } - } delete D; } - oss << "traces:\n"; { + oss << "traces:\n"; + { oss << "sid\ttrace\n"; - for(auto i : traces) { + for (auto i : traces) + { oss << i.first << "\t"; - for(auto s : i.second) { - oss << s << ','; + for (auto s : i.second) + { + oss << s << ';'; } oss << "\b \n"; } } - oss << "groups:\n"; { - if(tgid_fd < 0) { + oss << "groups:\n"; + { + if (tgid_fd < 0) + { return oss.str(); } auto keys = new uint32_t[MAX_ENTRIES]; @@ -472,18 +344,22 @@ class StackCollector uint32_t count = MAX_ENTRIES; uint32_t next_key; int err = bpf_map_lookup_batch(tgid_fd, NULL, &next_key, keys, vals, &count, NULL); - if(err == EFAULT) { + if (err == EFAULT) + { return oss.str(); } oss << "pid\ttgid\n"; - for(uint32_t i = 0; i < count; i++) { + for (uint32_t i = 0; i < count; i++) + { oss << keys[i] << '\t' << vals[i] << '\n'; } delete[] keys; delete[] vals; } - oss << "commands:\n"; { - if(comm_fd < 0) { + oss << "commands:\n"; + { + if (comm_fd < 0) + { return oss.str(); } auto keys = new uint32_t[MAX_ENTRIES]; @@ -491,11 +367,13 @@ class StackCollector uint32_t count = MAX_ENTRIES; uint32_t next_key; int err = bpf_map_lookup_batch(comm_fd, NULL, &next_key, keys, vals, &count, NULL); - if(err == EFAULT) { + if (err == EFAULT) + { return oss.str(); } oss << "pid\tcommand\n"; - for(uint32_t i = 0; i < count; i++) { + for (uint32_t i = 0; i < count; i++) + { oss << keys[i] << '\t' << vals[i] << '\n'; } delete[] keys; @@ -504,7 +382,6 @@ class StackCollector oss << "OK\n"; return oss.str(); } - }; class OnCPUStackCollector : public StackCollector @@ -529,7 +406,8 @@ class OnCPUStackCollector : public StackCollector CHECK_ERR_EXIT(num_cpus <= 0, "Fail to get the number of processors"); }; - std::string data_str(uint64_t f) override { return std::to_string(f) + "Count:" + std::to_string(freq) + "HZ:5s"; }; + double data_value(void *data) override { return 1. * *(uint32_t *)data * 1000 / freq; } + std::string data_str(void) override { return "ThisTimeOnCpu/ms"; }; int load(void) override { @@ -610,7 +488,7 @@ class OffCPUStackCollector : public StackCollector declareEBPF(off_cpu_count_bpf); protected: - std::string data_str(uint64_t f) override { return std::to_string(f) + "ms:5s"; }; + std::string data_str(void) override { return "OffCpuThisTime/ms"; }; defaultLoad; defaultAttach; defaultDetach; @@ -626,7 +504,7 @@ class MemoryStackCollector : public StackCollector declareEBPF(mem_count_bpf); protected: - std::string data_str(uint64_t f) override { return std::to_string(f) + "LeakByte"; }; + std::string data_str(void) override { return "LeakMomery/Byte"; }; public: char *object = (char *)"libc.so.6"; @@ -692,28 +570,19 @@ class IOStackCollector : public StackCollector declareEBPF(io_count_bpf); protected: - std::string data_str(uint64_t f) override + std::string data_str(void) override { - const std::string IOScale[] = {"counts", "size(B)", "aver(B/1)"}; - return IOScale[DataType] + ":" + std::to_string(f); + static const std::string IOScale[] = {"IOCountThisTime/1", "IOSizeThisTime/Byte", "AverageIOSizeThisTime/Byte"}; + return IOScale[DataType]; }; - uint64_t data_value() override + double data_value(void *data) override { - io_tuple *p = (io_tuple *)data_buf; + io_tuple *p = (io_tuple *)data; switch (DataType) { - - std::string data_str(uint64_t f) override { - const std::string IOScale[] = {"Count", "Byte", "Byte:Count"}; - return std::to_string(f) + IOScale[DataType] + ":5s"; - }; - - uint64_t data_value(void *data) override { - io_tuple *p = (io_tuple *)data; - switch (DataType) { case AVE: - return p->size / p->count; + return 1. * p->size / p->count; case SIZE: return p->size; case COUNT: @@ -728,17 +597,6 @@ class IOStackCollector : public StackCollector IOStackCollector() { - delete (uint64_t *)data_buf; - data_buf = new io_tuple{0}; - name = "io"; - }; - - ~IOStackCollector() override - { - delete (io_tuple *)data_buf; - }; - - IOStackCollector() { count_size = sizeof(io_tuple); name = "io"; }; @@ -755,19 +613,13 @@ class ReadaheadStackCollector : public StackCollector declareEBPF(pre_count_bpf); protected: - std::string data_str(uint64_t f) override + std::string data_str(void) override { - return "rest_pages:" + std::to_string(f); + return "TotalUnusedReadaheadPages/Page"; }; - uint64_t data_value() override + double data_value(void *data) override { - ra_tuple *p = (ra_tuple *)data_buf; - std::string data_str(uint64_t f) override { - return std::to_string(f) + "UnusedPage"; - }; - - uint64_t data_value(void *data) override { ra_tuple *p = (ra_tuple *)data; return p->expect - p->truth; }; @@ -780,18 +632,10 @@ class ReadaheadStackCollector : public StackCollector ReadaheadStackCollector() { - delete (uint64_t *)data_buf; - data_buf = new ra_tuple{0}; - ReadaheadStackCollector() { name = "readahead"; count_size = sizeof(ra_tuple); showDelta = false; }; - - ~ReadaheadStackCollector() override - { - delete (ra_tuple *)data_buf; - } }; namespace MainConfig @@ -811,9 +655,6 @@ void endCollect(void) { if (MainConfig::run_time > 0) { - Item->format(); - for(auto Item : StackCollectorList) { - if(MainConfig::run_time > 0) { std::cout << std::string(*Item) << std::endl; } Item->detach(); @@ -833,8 +674,7 @@ int main(int argc, char *argv[]) ((clipp::option("-c", "--command") & clipp::value("to be sampled command to run, default none", MainConfig::command)) % "set command for monitoring the whole life")), (clipp::option("-d", "--delay") & clipp::value("delay time(seconds) to output, default 5", MainConfig::delay)) % "set the interval to output", clipp::option("-l", "--realtime-list").set(MainConfig::d_mode, LIST_OUTPUT) % "output in console, default false", - clipp::option("-t", "--timeout") & clipp::value("run time, default nearly infinite", MainConfig::run_time) % "set the total simpling time", - clipp::option("-s", "--server") & clipp::value("server address, default 127.0.0.1:12345", MainConfig::server_address) % "set the server address"); + clipp::option("-t", "--timeout") & clipp::value("run time, default nearly infinite", MainConfig::run_time) % "set the total simpling time"); auto SubOption = (clipp::option("-U", "--user-stack-only").call([] { StackCollectorList.back()->kstack = false; }) % @@ -900,83 +740,6 @@ int main(int argc, char *argv[]) if (!clipp::parse(argc, argv, cli)) { -int main(int argc, char *argv[]) { - auto MainOption = ( - ( - ((clipp::option("-p", "--pid") & clipp::value("pid of sampled process, default -1 for all", MainConfig::target_pid)) % "set pid of process to monitor") | - ((clipp::option("-c", "--command") & clipp::value("to be sampled command to run, default none", MainConfig::command)) % "set command for monitoring the whole life") - ), - (clipp::option("-d", "--delay") & clipp::value("delay time(seconds) to output, default 5", MainConfig::delay)) % "set the interval to output", - clipp::option("-l", "--realtime-list").set(MainConfig::d_mode, LIST_OUTPUT) % "output in console, default false", - clipp::option("-t", "--timeout") & clipp::value("run time, default nearly infinite", MainConfig::run_time) % "set the total simpling time" - ); - - auto SubOption = ( - clipp::option("-U", "--user-stack-only").call([]{ - StackCollectorList.back()->kstack = false; - }) % "only sample user stacks", - clipp::option("-K", "--kernel-stack-only").call([]{ - StackCollectorList.back()->ustack = false; - }) % "only sample kernel stacks", - (clipp::option("-m", "--max-value") & clipp::value("max threshold of sampled value", optbuff).call([]{ - StackCollectorList.back()->max = optbuff; - })) % "set the max threshold of sampled value", - (clipp::option("-n", "--min-value") & clipp::value("min threshold of sampled value", optbuff).call([]{ - StackCollectorList.back()->min = optbuff; - })) % "set the min threshold of sampled value" - ); - - auto OnCpuOption = clipp::option("on-cpu").call([]{ - StackCollectorList.push_back(new OnCPUStackCollector()); - }) % "sample the call stacks of on-cpu processes" & ( - clipp::option("-F", "--frequency") & clipp::value("sampling frequency", optbuff).call([]{ - static_cast(StackCollectorList.back())->freq = optbuff; - }) % "sampling at a set frequency", - SubOption - ); - - auto OffCpuOption = clipp::option("off-cpu").call([]{ - StackCollectorList.push_back(new OffCPUStackCollector()); - }) % "sample the call stacks of off-cpu processes" & SubOption; - - auto MemoryOption = clipp::option("mem").call([]{ - StackCollectorList.push_back(new MemoryStackCollector()); - }) % "sample the memory usage of call stacks" & SubOption; - - auto IOOption = clipp::option("io").call([]{ - StackCollectorList.push_back(new IOStackCollector()); - }) % "sample the IO data volume of call stacks" & ( - (clipp::option("--mod") & ( - clipp::option("count").call([]{ - static_cast(StackCollectorList.back())->DataType = COUNT; - }) % "Counting the number of I/O operations" | - clipp::option("ave").call([]{ - static_cast(StackCollectorList.back())->DataType = AVE; - }) % "Counting the ave of I/O operations" | - clipp::option("size").call([]{ - static_cast(StackCollectorList.back())->DataType = SIZE; - }) % "Counting the size of I/O operations" - )) % "set the statistic mod", - SubOption - ); - - auto ReadaheadOption = clipp::option("ra").call([]{ - StackCollectorList.push_back(new ReadaheadStackCollector()); - }) % "sample the readahead hit rate of call stacks" & SubOption; - - auto cli = ( - MainOption, - clipp::option("-v", "--version").call([] { - std::cout << "verion 2.0\n\n"; - }) % "show version", - OnCpuOption, - OffCpuOption, - MemoryOption, - IOOption, - ReadaheadOption - ) % "statistic call trace relate with some metrics"; - - if (!clipp::parse(argc, argv, cli)) { std::cout << clipp::make_man_page(cli, argv[0]) << '\n'; return 0; } @@ -1037,90 +800,18 @@ int main(int argc, char *argv[]) { write(child_exec_event_fd, &eventbuff, sizeof(eventbuff)); } - printf("display mode: %d\n", MainConfig::d_mode); - - // 创建 socket - bool ToRemote = true; - int clientSocket = socket(AF_INET, SOCK_STREAM, 0); - if (clientSocket == -1) - { - std::cerr << "Error creating socket" << std::endl; - // return -1; - ToRemote = false; - } - else - { - // 服务器地址信息 - sockaddr_in serverAddress; - serverAddress.sin_family = AF_INET; - auto ColonPos = MainConfig::server_address.find(':'); - if (ColonPos < 0) - { - std::cerr << "server address err" << std::endl; - return 0; - } - auto IPAddr = MainConfig::server_address.substr(0, ColonPos); - auto PortAddr = MainConfig::server_address.substr(ColonPos + 1); - serverAddress.sin_port = htons(std::stoi(PortAddr)); - inet_pton(AF_INET, IPAddr.c_str(), &serverAddress.sin_addr); - // 连接到服务器 - if (connect(clientSocket, (struct sockaddr *)&serverAddress, sizeof(serverAddress)) == -1) - { - std::cerr << "Error connecting to server" << std::endl; - close(clientSocket); - // return -1; - ToRemote = false; - } - } + // printf("display mode: %d\n", MainConfig::d_mode); for (; MainConfig::run_time > 0 && (MainConfig::target_pid < 0 || !kill(MainConfig::target_pid, 0)); MainConfig::run_time -= MainConfig::delay) { - sleep(MainConfig::delay); // 模拟实时性 - time_t timep; - ::time(&timep); - printf("%s", ctime(&timep)); - + sleep(MainConfig::delay); for (auto Item : StackCollectorList) { - Item->detach(); - // if(MainConfig::d_mode == display_t::LIST_OUTPUT) { - // Item->print_list(); - // } - auto StreamData = Item->format(); - if (!StreamData) - { - continue; - } - auto dataToSend = StreamData->str(); - if (ToRemote) - { - // 发送数据到服务器 - struct diy_header AHeader = { - .len = dataToSend.size()}; - strcpy(AHeader.name, Item->name.c_str()); - send(clientSocket, &AHeader, sizeof(AHeader), 0); - send(clientSocket, dataToSend.c_str(), AHeader.len, 0); - } - else - { - Item->print_list(); - std::ofstream fout; - fout.open(Item->name + "_stack_data.txt", std::ios::out | std::ios::app); - fout << dataToSend; - } - delete StreamData; - Item->check_clear_count(); - - // printf("display mode: %d\n", MainConfig::d_mode); - - for(; MainConfig::run_time > 0 && (MainConfig::target_pid < 0 || !kill(MainConfig::target_pid, 0)); MainConfig::run_time -= MainConfig::delay) { - sleep(MainConfig::delay); - for(auto Item : StackCollectorList) { Item->detach(); std::cout << std::string(*Item) << std::endl; Item->attach(); } } - + atexit(endCollect); } \ No newline at end of file From 7a38410ee90e3c433cde1a8a30a07ead26a88758 Mon Sep 17 00:00:00 2001 From: gaoyixiang1 <1739037263@qq.com> Date: Sat, 27 Jan 2024 09:50:15 +0800 Subject: [PATCH 3/5] add stack_count and improve doc --- .../Stack_Analyser/libbpf/Makefile | 2 +- .../Stack_Analyser/libbpf/include/sa_common.h | 6 +++ .../Stack_Analyser/libbpf/include/sa_user.h | 9 +++- .../Stack_Analyser/libbpf/stack_analyzer.cc | 48 +++++++++++++++++-- ...77\347\224\250\346\226\271\346\263\225.md" | 6 +-- 5 files changed, 61 insertions(+), 10 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/libbpf/Makefile b/eBPF_Supermarket/Stack_Analyser/libbpf/Makefile index 6553fff2f..5561e438f 100644 --- a/eBPF_Supermarket/Stack_Analyser/libbpf/Makefile +++ b/eBPF_Supermarket/Stack_Analyser/libbpf/Makefile @@ -38,7 +38,7 @@ INCLUDES := -I$(OUTPUT) -I./libbpf-bootstrap/libbpf/include/uapi -I$(dir $(VMLIN CFLAGS := -g -Wall ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS) -APPS = on_cpu_count off_cpu_count mem_count io_count pre_count +APPS = on_cpu_count off_cpu_count mem_count io_count pre_count stack_count SYMBOL = elf symbol TARGETS = stack_analyzer diff --git a/eBPF_Supermarket/Stack_Analyser/libbpf/include/sa_common.h b/eBPF_Supermarket/Stack_Analyser/libbpf/include/sa_common.h index a142fa261..b15ac747c 100644 --- a/eBPF_Supermarket/Stack_Analyser/libbpf/include/sa_common.h +++ b/eBPF_Supermarket/Stack_Analyser/libbpf/include/sa_common.h @@ -46,4 +46,10 @@ typedef struct { __u64 size; } io_tuple; +typedef struct { + __u32 tgid; + comm name; + __u64 count; +} stack_tuple; + #endif \ No newline at end of file diff --git a/eBPF_Supermarket/Stack_Analyser/libbpf/include/sa_user.h b/eBPF_Supermarket/Stack_Analyser/libbpf/include/sa_user.h index 4e0ffdd27..7d51562da 100644 --- a/eBPF_Supermarket/Stack_Analyser/libbpf/include/sa_user.h +++ b/eBPF_Supermarket/Stack_Analyser/libbpf/include/sa_user.h @@ -41,6 +41,7 @@ typedef enum { MOD_MEM, // 内存模式 MOD_IO, // io模式 MOD_RA, // 预读取分析模式 + MOD_STACK, //调用栈次数统计模式 MOD_NUM // 该枚举类值的总数 } StackCollectMode; @@ -49,7 +50,8 @@ char StackCollectModeName[MOD_NUM][16] = { "off_cpu", "memory", "io", - "readahead" + "readahead", + "stackcount", }; typedef enum { @@ -63,6 +65,11 @@ typedef enum { AVE } io_mod; +typedef enum { + COUNTS, + CPU +} stack_mod; + /// @brief 获取epbf程序中指定表的文件描述符 /// @param name 表的名字 #define OPEN_MAP(name) bpf_map__fd(skel->maps.name) diff --git a/eBPF_Supermarket/Stack_Analyser/libbpf/stack_analyzer.cc b/eBPF_Supermarket/Stack_Analyser/libbpf/stack_analyzer.cc index b6dc50f2b..aa28e612f 100644 --- a/eBPF_Supermarket/Stack_Analyser/libbpf/stack_analyzer.cc +++ b/eBPF_Supermarket/Stack_Analyser/libbpf/stack_analyzer.cc @@ -39,6 +39,7 @@ extern "C" #include "bpf/mem_count.skel.h" #include "bpf/io_count.skel.h" #include "bpf/pre_count.skel.h" +#include "bpf/stack_count.skel.h" } std::string demangleCppSym(std::string symbol) @@ -96,6 +97,7 @@ class StackCollector double val; CountItem(int32_t p, int32_t k, int32_t u, double v) { + pid = p; ksid = k; usid = u; @@ -173,7 +175,7 @@ class StackCollector public: std::string name; // 标识类名 - + int pid = -1; // 用于设置ebpf程序跟踪的pid int cpu = -1; // 用于设置ebpf程序跟踪的cpu int err = 0; // 用于保存错误代码 @@ -184,7 +186,6 @@ class StackCollector uint64_t max = __UINT64_MAX__; // 设置采集指标最大值，最小值 bool clear = false; // 清除已输出的指标积累量 - int self_pid; StackCollector() @@ -384,6 +385,7 @@ class StackCollector } }; + class OnCPUStackCollector : public StackCollector { private: @@ -417,7 +419,7 @@ class OnCPUStackCollector : public StackCollector fscanf(fp, "%p", &load_a); pclose(fp); StackProgLoadOpen( - skel->bss->load_a = load_a) return 0; + skel->bss->load_a = load_a ) return 0; }; int attach(void) override @@ -638,6 +640,40 @@ class ReadaheadStackCollector : public StackCollector }; }; +class StackCountStackCollector : public StackCollector +{ +private: + declareEBPF(stack_count_bpf); + + +protected: + std::string data_str(void) override + { + return "Calling Counts"; + }; + double data_value(void *data) override + { + stack_tuple *p = (stack_tuple *)data; + return p->count; + }; + +public: + stack_mod DataType = stack_mod::COUNTS; + + StackCountStackCollector() + { + count_size = sizeof(stack_tuple); + name = "stackcount"; + }; + + defaultLoad; + defaultAttach; + defaultDetach; + defaultUnload; + +}; + + namespace MainConfig { int run_time = __INT_MAX__; // 运行时间 @@ -726,6 +762,9 @@ int main(int argc, char *argv[]) { StackCollectorList.push_back(new ReadaheadStackCollector()); }) % "sample the readahead hit rate of call stacks" & SubOption; + auto StackCountOption = clipp::option("stackcount").call([] + { StackCollectorList.push_back(new StackCountStackCollector()); }) % + "sample the counts of calling stacks" & SubOption; auto cli = (MainOption, clipp::option("-v", "--version").call([] @@ -735,7 +774,8 @@ int main(int argc, char *argv[]) OffCpuOption, MemoryOption, IOOption, - ReadaheadOption) % + ReadaheadOption, + StackCountOption) % "statistic call trace relate with some metrics"; if (!clipp::parse(argc, argv, cli)) diff --git "a/eBPF_Supermarket/Stack_Analyser/libbpf/\346\241\206\346\236\266\344\275\277\347\224\250\346\226\271\346\263\225.md" "b/eBPF_Supermarket/Stack_Analyser/libbpf/\346\241\206\346\236\266\344\275\277\347\224\250\346\226\271\346\263\225.md" index 96af9c47b..4c68a9bb6 100644 --- "a/eBPF_Supermarket/Stack_Analyser/libbpf/\346\241\206\346\236\266\344\275\277\347\224\250\346\226\271\346\263\225.md" +++ "b/eBPF_Supermarket/Stack_Analyser/libbpf/\346\241\206\346\236\266\344\275\277\347\224\250\346\226\271\346\263\225.md" @@ -8,7 +8,7 @@ # 用户侧 -1. 在 `include/stack_analyzer.h` 中的 `MOD` 枚举类型定义中新增一个功能模块的标识 +1. 在 `include/stack_analyzer.h` 中的 `MOD` 枚举类型定义中新增一个功能模块的标识，将该名称添加到`MOD_NUM`前面即可 2. 在 `stack_analyzer.cc` 中创建一个 `bpf_loader` 的子类，在其中重写以下函数： @@ -24,9 +24,7 @@ 6. 自定义eBPF程序清除函数 `void remove(void)`，将eBPF程序清除 -3. 在 main 函数中添加新增子命令和对应参数解析语句，将子命令解析为新增子功能对应的标识符并设置给 `env::mod` - -4. 在 `bpf_loader arr[]` 中添加 包装子类构造函数的匿名函数，添加顺序需和其在 MOD 枚举类型中对应的 标识 的顺序一致 +3. 在 main 函数中添加新增子命令和对应参数解析语句 # 编译侧 From 34680f24b021b052897215d88889058eef1fef1a Mon Sep 17 00:00:00 2001 From: gaoyixiang1 <45355878+gaoyixiang1@users.noreply.github.com> Date: Sat, 27 Jan 2024 10:28:36 +0800 Subject: [PATCH 4/5] add stack_count.bpf.c --- eBPF_Supermarket/Stack_Analyser/libbpf/bpf/stack_count.bpf.c | 1 + 1 file changed, 1 insertion(+) create mode 100644 eBPF_Supermarket/Stack_Analyser/libbpf/bpf/stack_count.bpf.c diff --git a/eBPF_Supermarket/Stack_Analyser/libbpf/bpf/stack_count.bpf.c b/eBPF_Supermarket/Stack_Analyser/libbpf/bpf/stack_count.bpf.c new file mode 100644 index 000000000..b4785957b --- /dev/null +++ b/eBPF_Supermarket/Stack_Analyser/libbpf/bpf/stack_count.bpf.c @@ -0,0 +1 @@ +s From 33188fd84f187eeda9e776ae8334c4c3f00becea Mon Sep 17 00:00:00 2001 From: gaoyixiang1 <45355878+gaoyixiang1@users.noreply.github.com> Date: Sat, 27 Jan 2024 10:31:16 +0800 Subject: [PATCH 5/5] Update stack_count.bpf.c --- .../libbpf/bpf/stack_count.bpf.c | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/eBPF_Supermarket/Stack_Analyser/libbpf/bpf/stack_count.bpf.c b/eBPF_Supermarket/Stack_Analyser/libbpf/bpf/stack_count.bpf.c index b4785957b..719c9cb24 100644 --- a/eBPF_Supermarket/Stack_Analyser/libbpf/bpf/stack_count.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/libbpf/bpf/stack_count.bpf.c @@ -1 +1,72 @@ -s +#include "vmlinux.h" +#include +#include +#include + + +#include "sa_ebpf.h" +#include "task.h" + +DeclareCommonMaps(stack_tuple); +DeclareCommonVar(); + +//传进来的参数 +int apid = 0; +// int acpu = 0; + +const char LICENSE[] SEC("license") = "GPL"; + +static int handle(struct trace_event_raw_sys_enter *ctx) +{ + struct task_struct* curr = (struct task_struct*)bpf_get_current_task(); //利用bpf_get_current_task()获得当前的进程tsk + ignoreKthread(curr); + // u32 cpu_id = bpf_get_smp_processor_id(); + // if(cpu_id != acpu){ + // return 0; + // } + stack_tuple key = {}; + u32 pid = get_task_ns_pid(curr); //利用帮助函数获得当前进程的pid + if ((apid >= 0 && pid != apid) || !pid || pid == self_pid) + return 0; + + u32 tgid = get_task_ns_tgid(curr); //利用帮助函数获取进程的tgid + bpf_map_update_elem(&pid_tgid, &pid, &tgid, BPF_ANY); //将pid_tgid表中的pid选项更新为tgid,若没有该表项，则创建 + comm *p = bpf_map_lookup_elem(&pid_comm, &pid); //p指向pid_comm哈希表中的pid表项对应的value + if (!p) //如果p不为空，获取当前进程名保存至name中，如果pid_comm当中不存在pid name项，则更新 + { + comm name; + bpf_get_current_comm(&name, COMM_LEN); + bpf_map_update_elem(&pid_comm, &pid, &name, BPF_NOEXIST); + p = &name; + } + key.name = *p; + u32 *t = bpf_map_lookup_elem(&pid_tgid, &pid); + if(!t){ + key.tgid = 0xffffffff; + }else{ + key.tgid = *t; + } + + psid apsid = { + .pid = pid, + .usid = u ? USER_STACK : -1, + .ksid = k ? KERNEL_STACK : -1, + }; + stack_tuple *d = bpf_map_lookup_elem(&psid_count, &apsid); //d指向psid_count表当中的apsid表项的值 + + if(!d) { + stack_tuple nd = {.count = 1, .name = key.name,.tgid = key.tgid}; + bpf_map_update_elem(&psid_count, &apsid, &nd, BPF_NOEXIST); + } else { + d->count++; + } + return 0; + +} + +#define io_sec_tp(name) \ + SEC("tp/syscalls/sys_enter_" #name) \ + int prog_t_##name(struct trace_event_raw_sys_enter *ctx) { return handle(ctx); } + +io_sec_tp(write); +io_sec_tp(read);