From 07b24882c2e45f199166640de505af36645d900c Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 2 Feb 2024 13:10:20 +0800 Subject: [PATCH 01/47] add kvmexit watcher --- .../kvm_watcher/kvm_exit_bcc/kvmexit.py | 378 ++++++++++++++++++ .../kvm_exit_bcc/kvmexit_example.txt | 250 ++++++++++++ 2 files changed, 628 insertions(+) create mode 100644 eBPF_Supermarket/kvm_watcher/kvm_exit_bcc/kvmexit.py create mode 100644 eBPF_Supermarket/kvm_watcher/kvm_exit_bcc/kvmexit_example.txt diff --git a/eBPF_Supermarket/kvm_watcher/kvm_exit_bcc/kvmexit.py b/eBPF_Supermarket/kvm_watcher/kvm_exit_bcc/kvmexit.py new file mode 100644 index 000000000..2f82530f0 --- /dev/null +++ b/eBPF_Supermarket/kvm_watcher/kvm_exit_bcc/kvmexit.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python +# +# kvmexit.py +# +# Display the exit_reason and its statistics of each vm exit +# for all vcpus of all virtual machines. For example: +# $./kvmexit.py +# PID TID KVM_EXIT_REASON COUNT +# 1273551 1273568 EXIT_REASON_MSR_WRITE 6 +# 1274253 1274261 EXIT_REASON_EXTERNAL_INTERRUPT 1 +# 1274253 1274261 EXIT_REASON_HLT 12 +# ... +# +# Besides, we also allow users to specify one pid, tid(s), or one +# pid and its vcpu. See kvmexit_example.txt for more examples. +# +# @PID: each vitual machine's pid in the user space. +# @TID: the user space's thread of each vcpu of that virtual machine. +# @KVM_EXIT_REASON: the reason why the vm exits. +# @COUNT: the counts of the @KVM_EXIT_REASONS. +# +# REQUIRES: Linux 6.2 (BPF_PROG_TYPE_TRACEPOINT support) +# +# Copyright (c) 2024 YYS. All rights reserved. +# Original code © 2024 ByteDance Inc. All rights reserved. +# Author(s): +# YYS +# 以下代码段是根据Fei Li的实现进行的修改 +# 原始代码链接：https://github.com/iovisor/bcc/blob/master/tools/kvmexit.py + + +from __future__ import print_function +from time import sleep +from bcc import BPF +import argparse +import multiprocessing +import os +import subprocess + +# +# Process Arguments +# +def valid_args_list(args): + args_list = args.split(",") + for arg in args_list: + try: + int(arg) + except: + raise argparse.ArgumentTypeError("must be valid integer") + return args_list + +# arguments +examples = """examples: + ./kvmexit # Display kvm_exit_reason and its statistics in real-time until Ctrl-C + ./kvmexit 5 # Display in real-time after sleeping 5s + ./kvmexit -p 3195281 # Collpase all tids for pid 3195281 with exit reasons sorted in descending order + ./kvmexit -p 3195281 20 # Collpase all tids for pid 3195281 with exit reasons sorted in descending order, and display after sleeping 20s + ./kvmexit -p 3195281 -v 0 # Display only vcpu0 for pid 3195281, descending sort by default + ./kvmexit -p 3195281 -a # Display all tids for pid 3195281 + ./kvmexit -t 395490 # Display only for tid 395490 with exit reasons sorted in descending order + ./kvmexit -t 395490 20 # Display only for tid 395490 with exit reasons sorted in descending order after sleeping 20s + ./kvmexit -T '395490,395491' # Display for a union like {395490, 395491} +""" +parser = argparse.ArgumentParser( + description="Display kvm_exit_reason and its statistics at a timed interval", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=examples) +parser.add_argument("duration", nargs="?", default=99999999, type=int, help="show delta for next several seconds") +parser.add_argument("-p", "--pid", type=int, help="trace this PID only") +exgroup = parser.add_mutually_exclusive_group() +exgroup.add_argument("-t", "--tid", type=int, help="trace this TID only") +exgroup.add_argument("-T", "--tids", type=valid_args_list, help="trace a comma separated series of tids with no space in between") +exgroup.add_argument("-v", "--vcpu", type=int, help="trace this vcpu only") +exgroup.add_argument("-a", "--alltids", action="store_true", help="trace all tids for this pid") +args = parser.parse_args() +duration = int(args.duration) + +# +# Setup BPF +# + +# load BPF program +bpf_text = """ +#include + +#define REASON_NUM 76 +#define TGID_NUM 1024 + +struct exit_count { + u64 exit_ct[REASON_NUM]; +}; +BPF_PERCPU_ARRAY(init_value, struct exit_count, 1); +BPF_TABLE("percpu_hash", u64, struct exit_count, pcpu_kvm_stat, TGID_NUM); + +struct cache_info { + u64 cache_pid_tgid; + struct exit_count cache_exit_ct; +}; +BPF_PERCPU_ARRAY(pcpu_cache, struct cache_info, 1); + +TRACEPOINT_PROBE(kvm, kvm_exit) { + int cache_miss = 0; + int zero = 0; + u32 er = args->exit_reason; + if (er >= REASON_NUM) { + return 0; + } + + u64 cur_pid_tgid = bpf_get_current_pid_tgid(); + u32 tgid = cur_pid_tgid >> 32; + u32 pid = cur_pid_tgid; + + if (THREAD_FILTER) + return 0; + + struct exit_count *tmp_info = NULL, *initial = NULL; + struct cache_info *cache_p; + cache_p = pcpu_cache.lookup(&zero); + if (cache_p == NULL) { + return 0; + } + + if (cache_p->cache_pid_tgid == cur_pid_tgid) { + //a. If the cur_pid_tgid hit this physical cpu consecutively, save it to pcpu_cache + tmp_info = &cache_p->cache_exit_ct; + } else { + //b. If another pid_tgid matches this pcpu for the last hit, OR it is the first time to hit this physical cpu. + cache_miss = 1; + + // b.a Try to load the last cache struct if exists. + tmp_info = pcpu_kvm_stat.lookup(&cur_pid_tgid); + + // b.b If it is the first time for the cur_pid_tgid to hit this pcpu, employ a + // per_cpu array to initialize pcpu_kvm_stat's exit_count with each exit reason's count is zero + if (tmp_info == NULL) { + initial = init_value.lookup(&zero); + if (initial == NULL) { + return 0; + } + + pcpu_kvm_stat.update(&cur_pid_tgid, initial); + tmp_info = pcpu_kvm_stat.lookup(&cur_pid_tgid); + // To pass the verifier + if (tmp_info == NULL) { + return 0; + } + } + } + + if (er < REASON_NUM) { + tmp_info->exit_ct[er]++; + if (cache_miss == 1) { + if (cache_p->cache_pid_tgid != 0) { + // b.*.a Let's save the last hit cache_info into kvm_stat. + pcpu_kvm_stat.update(&cache_p->cache_pid_tgid, &cache_p->cache_exit_ct); + } + // b.* As the cur_pid_tgid meets current pcpu_cache_array for the first time, save it. + cache_p->cache_pid_tgid = cur_pid_tgid; + bpf_probe_read(&cache_p->cache_exit_ct, sizeof(*tmp_info), tmp_info); + } + return 0; + } + + return 0; +} +""" + +# format output +exit_reasons = ( + "EXCEPTION_NMI", + "EXTERNAL_INTERRUPT", + "TRIPLE_FAULT", + "INIT_SIGNAL", + "SIPI_SIGNAL ", + "N/A", + "N/A", + "INTERRUPT_WINDOW", + "NMI_WINDOW", + "TASK_SWITCH", + "CPUID", + "N/A", + "HLT", + "INVD", + "INVLPG", + "RDPMC", + "RDTSC", + "N/A", + "VMCALL", + "VMCLEAR", + "VMLAUNCH", + "VMPTRLD", + "VMPTRST", + "VMREAD", + "VMRESUME", + "VMWRITE", + "VMOFF", + "VMON", + "CR_ACCESS", + "DR_ACCESS", + "IO_INSTRUCTION", + "MSR_READ", + "MSR_WRITE", + "INVALID_STATE", + "MSR_LOAD_FAIL", + "N/A", + "MWAIT_INSTRUCTION", + "MONITOR_TRAP_FLAG", + "N/A", + "MONITOR_INSTRUCTION", + "PAUSE_INSTRUCTION", + "MCE_DURING_VMENTRY", + "N/A", + "TPR_BELOW_THRESHOLD", + "APIC_ACCESS", + "EOI_INDUCED", + "GDTR_IDTR", + "LDTR_TR", + "EPT_VIOLATION", + "EPT_MISCONFIG", + "INVEPT", + "RDTSCP", + "PREEMPTION_TIMER", + "INVVPID", + "WBINVD", + "XSETBV", + "APIC_WRITE", + "RDRAND", + "INVPCID", + "VMFUNC", + "ENCLS", + "RDSEED", + "PML_FULL", + "XSAVES", + "XRSTORS", + "N/A", + "N/A", + "UMWAIT", + "TPAUSE", + "N/A", + "N/A", + "N/A", + "N/A", + "N/A", + "BUS_LOCK", + "NOTIFY " +) + +# +# Do some checks +# +try: + # Currently, only adapte on intel architecture + cmd = "cat /proc/cpuinfo | grep vendor_id | head -n 1" + arch_info = subprocess.check_output(cmd, shell=True).strip() + if b"Intel" in arch_info: + pass + else: + raise Exception("Currently we only support Intel architecture, please do expansion if needs more.") + + # Check if kvm module is loaded + if os.access("/dev/kvm", os.R_OK | os.W_OK): + pass + else: + raise Exception("Please insmod kvm module to use kvmexit tool.") +except Exception as e: + raise Exception("Failed to do precondition check, due to: %s." % e) + +def find_tid(tgt_dir, tgt_vcpu): + for tid in os.listdir(tgt_dir): + path = tgt_dir + "/" + tid + "/comm" + fp = open(path, "r") + comm = fp.read() + if (comm.find(tgt_vcpu) != -1): + return tid + return -1 + +# set process/thread filter +thread_context = "" +header_format = "" +need_collapse = not args.alltids +if args.tid is not None: + thread_context = "TID %s" % args.tid + thread_filter = 'pid != %s' % args.tid +elif args.tids is not None: + thread_context = "TIDS %s" % args.tids + thread_filter = "pid != " + " && pid != ".join(args.tids) + header_format = "TIDS " +elif args.pid is not None: + thread_context = "PID %s" % args.pid + thread_filter = 'tgid != %s' % args.pid + if args.vcpu is not None: + thread_context = "PID %s VCPU %s" % (args.pid, args.vcpu) + # transfer vcpu to tid + tgt_dir = '/proc/' + str(args.pid) + '/task' + tgt_vcpu = "CPU " + str(args.vcpu) + args.tid = find_tid(tgt_dir, tgt_vcpu) + if args.tid == -1: + raise Exception("There's no v%s for PID %d." % (tgt_vcpu, args.pid)) + thread_filter = 'pid != %s' % args.tid + elif args.alltids: + thread_context = "PID %s and its all threads" % args.pid + header_format = "TID " +else: + thread_context = "all threads" + thread_filter = '0' + header_format = "PID TID " +bpf_text = bpf_text.replace('THREAD_FILTER', thread_filter) +b = BPF(text=bpf_text) + + +# header +print("Display kvm exit reasons and statistics for %s" % thread_context, end="") +if duration < 99999999: + print(" after sleeping %d secs." % duration) +else: + print("... Hit Ctrl-C to end.") + +try: + sleep(duration) +except KeyboardInterrupt: + print() + + +# Currently, sort multiple tids in descending order is not supported. +if (args.pid or args.tid): + ct_reason = [] + if args.pid: + tgid_exit = [0 for i in range(len(exit_reasons))] + +# output +print("%s%-35s %s" % (header_format, "KVM_EXIT_REASON", "COUNT")) + +pcpu_kvm_stat = b["pcpu_kvm_stat"] +pcpu_cache = b["pcpu_cache"] +for k, v in pcpu_kvm_stat.items(): + tgid = k.value >> 32 + pid = k.value & 0xffffffff + for i in range(0, len(exit_reasons)): + sum1 = 0 + for inner_cpu in range(0, multiprocessing.cpu_count()): + cachePIDTGID = pcpu_cache[0][inner_cpu].cache_pid_tgid + # Take priority to check if it is in cache + if cachePIDTGID == k.value: + sum1 += pcpu_cache[0][inner_cpu].cache_exit_ct.exit_ct[i] + # If not in cache, find from kvm_stat + else: + sum1 += v[inner_cpu].exit_ct[i] + if sum1 == 0: + continue + + if (args.pid and args.pid == tgid and need_collapse): + tgid_exit[i] += sum1 + elif (args.tid and args.tid == pid): + ct_reason.append((sum1, i)) + elif not need_collapse or args.tids: + print("%-8u %-35s %-8u" % (pid, exit_reasons[i], sum1)) + else: + print("%-8u %-8u %-35s %-8u" % (tgid, pid, exit_reasons[i], sum1)) + + # Display only for the target tid in descending sort + if (args.tid and args.tid == pid): + ct_reason.sort(reverse=True) + for i in range(0, len(ct_reason)): + if ct_reason[i][0] == 0: + continue + print("%-35s %-8u" % (exit_reasons[ct_reason[i][1]], ct_reason[i][0])) + break + + +# Aggregate all tids' counts for this args.pid in descending sort +if args.pid and need_collapse: + for i in range(0, len(exit_reasons)): + ct_reason.append((tgid_exit[i], i)) + ct_reason.sort(reverse=True) + for i in range(0, len(ct_reason)): + if ct_reason[i][0] == 0: + continue + print("%-35s %-8u" % (exit_reasons[ct_reason[i][1]], ct_reason[i][0])) \ No newline at end of file diff --git a/eBPF_Supermarket/kvm_watcher/kvm_exit_bcc/kvmexit_example.txt b/eBPF_Supermarket/kvm_watcher/kvm_exit_bcc/kvmexit_example.txt new file mode 100644 index 000000000..3ee773bbe --- /dev/null +++ b/eBPF_Supermarket/kvm_watcher/kvm_exit_bcc/kvmexit_example.txt @@ -0,0 +1,250 @@ +Demonstrations of kvm exit reasons, the Linux eBPF/bcc version. + + +Considering virtual machines' frequent exits can cause performance problems, +this tool aims to locate the frequent exited reasons and then find solutions +to reduce or even avoid the exit, by displaying the detail exit reasons and +the counts of each vm exit for all vms running on one physical machine. + + +Features of this tool +===================== + +- Although there is a patch: [KVM: x86: add full vm-exit reason debug entries] + (https://patchwork.kernel.org/project/kvm/patch/1555939499-30854-1-git-send-email-pizhenwei@bytedance.com/) + trying to fill more vm-exit reason debug entries, just as the comments said, + the code allocates lots of memory that may never be consumed, misses some + arch-specific kvm causes, and can not do kernel aggregation. Instead bcc, as + a user space tool, can implement all these functions more easily and flexibly. +- The bcc python logic could provide nice kernel aggregation and custom output, + like collpasing all tids for one pid (e.i. one vm's qemu process id) with exit + reasons sorted in descending order. For more information, see the following + #USAGE message. +- The bpf in-kernel percpu_array and percpu_cache further improves performance. + For more information, see the following #Help to understand. + + +Limited +======= + +In view of the hardware-assisted virtualization technology of +different architectures, currently we only adapt on vmx in intel. +And the amd feature is on the road.. + + +Example output: +=============== + +# ./kvmexit.py +Display kvm exit reasons and statistics for all threads... Hit Ctrl-C to end. +PID TID KVM_EXIT_REASON COUNT +^C1273551 1273568 EXIT_REASON_HLT 12 +1273551 1273568 EXIT_REASON_MSR_WRITE 6 +1274253 1274261 EXIT_REASON_EXTERNAL_INTERRUPT 1 +1274253 1274261 EXIT_REASON_HLT 12 +1274253 1274261 EXIT_REASON_MSR_WRITE 4 + +# ./kvmexit.py 6 +Display kvm exit reasons and statistics for all threads after sleeping 6 secs. +PID TID KVM_EXIT_REASON COUNT +1273903 1273922 EXIT_REASON_EXTERNAL_INTERRUPT 175 +1273903 1273922 EXIT_REASON_CPUID 10 +1273903 1273922 EXIT_REASON_HLT 6043 +1273903 1273922 EXIT_REASON_IO_INSTRUCTION 24 +1273903 1273922 EXIT_REASON_MSR_WRITE 15025 +1273903 1273922 EXIT_REASON_PAUSE_INSTRUCTION 11 +1273903 1273922 EXIT_REASON_EOI_INDUCED 12 +1273903 1273922 EXIT_REASON_EPT_VIOLATION 6 +1273903 1273922 EXIT_REASON_EPT_MISCONFIG 380 +1273903 1273922 EXIT_REASON_PREEMPTION_TIMER 194 +1273551 1273568 EXIT_REASON_EXTERNAL_INTERRUPT 18 +1273551 1273568 EXIT_REASON_HLT 989 +1273551 1273568 EXIT_REASON_IO_INSTRUCTION 10 +1273551 1273568 EXIT_REASON_MSR_WRITE 2205 +1273551 1273568 EXIT_REASON_PAUSE_INSTRUCTION 1 +1273551 1273568 EXIT_REASON_EOI_INDUCED 5 +1273551 1273568 EXIT_REASON_EPT_MISCONFIG 61 +1273551 1273568 EXIT_REASON_PREEMPTION_TIMER 14 + +# ./kvmexit.py -p 1273795 5 +Display kvm exit reasons and statistics for PID 1273795 after sleeping 5 secs. +KVM_EXIT_REASON COUNT +MSR_WRITE 13467 +HLT 5060 +PREEMPTION_TIMER 345 +EPT_MISCONFIG 264 +EXTERNAL_INTERRUPT 169 +EPT_VIOLATION 18 +PAUSE_INSTRUCTION 6 +IO_INSTRUCTION 4 +EOI_INDUCED 2 + +# ./kvmexit.py -p 1273795 5 -a +Display kvm exit reasons and statistics for PID 1273795 and its all threads after sleeping 5 secs. +TID KVM_EXIT_REASON COUNT +1273819 EXTERNAL_INTERRUPT 64 +1273819 HLT 2802 +1273819 IO_INSTRUCTION 4 +1273819 MSR_WRITE 7196 +1273819 PAUSE_INSTRUCTION 2 +1273819 EOI_INDUCED 2 +1273819 EPT_VIOLATION 6 +1273819 EPT_MISCONFIG 162 +1273819 PREEMPTION_TIMER 194 +1273820 EXTERNAL_INTERRUPT 78 +1273820 HLT 2054 +1273820 MSR_WRITE 5199 +1273820 EPT_VIOLATION 2 +1273820 EPT_MISCONFIG 77 +1273820 PREEMPTION_TIMER 102 + +# ./kvmexit.py -p 1273795 -v 0 +Display kvm exit reasons and statistics for PID 1273795 VCPU 0... Hit Ctrl-C to end. +KVM_EXIT_REASON COUNT +^CMSR_WRITE 2076 +HLT 795 +PREEMPTION_TIMER 86 +EXTERNAL_INTERRUPT 20 +EPT_MISCONFIG 10 +PAUSE_INSTRUCTION 2 +IO_INSTRUCTION 2 +EPT_VIOLATION 1 +EOI_INDUCED 1 + +# ./kvmexit.py -p 1273795 -v 0 4 +Display kvm exit reasons and statistics for PID 1273795 VCPU 0 after sleeping 4 secs. +KVM_EXIT_REASON COUNT +MSR_WRITE 4726 +HLT 1827 +PREEMPTION_TIMER 78 +EPT_MISCONFIG 67 +EXTERNAL_INTERRUPT 28 +IO_INSTRUCTION 4 +EOI_INDUCED 2 +PAUSE_INSTRUCTION 2 + +# ./kvmexit.py -p 1273795 -v 4 4 +Traceback (most recent call last): + File "tools/kvmexit.py", line 306, in + raise Exception("There's no v%s for PID %d." % (tgt_vcpu, args.pid)) + Exception: There's no vCPU 4 for PID 1273795. + +# ./kvmexit.py -t 1273819 10 +Display kvm exit reasons and statistics for TID 1273819 after sleeping 10 secs. +KVM_EXIT_REASON COUNT +MSR_WRITE 13318 +HLT 5274 +EPT_MISCONFIG 263 +PREEMPTION_TIMER 171 +EXTERNAL_INTERRUPT 109 +IO_INSTRUCTION 8 +PAUSE_INSTRUCTION 5 +EOI_INDUCED 4 +EPT_VIOLATION 2 + +# ./kvmexit.py -T '1273820,1273819' +Display kvm exit reasons and statistics for TIDS ['1273820', '1273819']... Hit Ctrl-C to end. +TIDS KVM_EXIT_REASON COUNT +^C1273819 EXTERNAL_INTERRUPT 300 +1273819 HLT 13718 +1273819 IO_INSTRUCTION 26 +1273819 MSR_WRITE 37457 +1273819 PAUSE_INSTRUCTION 13 +1273819 EOI_INDUCED 13 +1273819 EPT_VIOLATION 53 +1273819 EPT_MISCONFIG 654 +1273819 PREEMPTION_TIMER 958 +1273820 EXTERNAL_INTERRUPT 212 +1273820 HLT 9002 +1273820 MSR_WRITE 25495 +1273820 PAUSE_INSTRUCTION 2 +1273820 EPT_VIOLATION 64 +1273820 EPT_MISCONFIG 396 +1273820 PREEMPTION_TIMER 268 + + +Help to understand +================== + +We use a PERCPU_ARRAY: pcpuArrayA and a percpu_hash: hashA to collaboratively +store each kvm exit reason and its count. The reason is there exists a rule when +one vcpu exits and re-enters, it tends to continue to run on the same physical +cpu (pcpu as follows) as the last cycle, which is also called 'cache hit'. Thus +we turn to use a PERCPU_ARRAY to record the 'cache hit' situation to speed +things up; and for other cases, then use a percpu_hash. + +BTW, we originally use a common hash to do this, with a u64(exit_reason) +key and a struct exit_info {tgid_pid, exit_reason} value. But due to +the big lock in bpf_hash, each updating is quite performance consuming. + +Now imagine here is a pid_tgidA (vcpu A) exits and is going to run on +pcpuArrayA, the BPF code flow is as follows: + + pid_tgidA keeps running on the same pcpu + // \\ + // \\ + // Y N \\ + // \\ + a. cache_hit b. cache_miss +(cacheA's pid_tgid matches pid_tgidA) || + | || + | || + "increase percpu exit_ct and return" || + [*Note*] || + pid_tgidA ever been exited on pcpuArrayA? + // \\ + // \\ + // \\ + // Y N \\ + // \\ + b.a load_last_hashA b.b initialize_hashA_with_zero + \ / + \ / + \ / + "increase percpu exit_ct" + || + || + is another pid_tgid been running on pcpuArrayA? + // \\ + // Y N \\ + // \\ + b.*.a save_theLastHit_hashB do_nothing + \\ // + \\ // + \\ // + b.* save_to_pcpuArrayA + + +[*Note*] we do not update the table in above "a.", in case the vcpu hit the same +pcpu again when exits next time, instead we only update until this pcpu is not +hitted by the same tgidpid(vcpu) again, which is in "b.*.a" and "b.*". + + +USAGE message: +============== + +# ./kvmexit.py -h +usage: kvmexit.py [-h] [-p PID [-v VCPU | -a] ] [-t TID | -T 'TID1,TID2'] [duration] + +Display kvm_exit_reason and its statistics at a timed interval + +optional arguments: + -h, --help show this help message and exit + -p PID, --pid PID display process with this PID only, collpase all tids with exit reasons sorted in descending order + -v VCPU, --v VCPU display this VCPU only for this PID + -a, --alltids display all TIDS for this PID + -t TID, --tid TID display thread with this TID only with exit reasons sorted in descending order + -T 'TID1,TID2', --tids 'TID1,TID2' + display threads for a union like {395490, 395491} + duration duration of display, after sleeping several seconds + +examples: + ./kvmexit # Display kvm_exit_reason and its statistics in real-time until Ctrl-C + ./kvmexit 5 # Display in real-time after sleeping 5s + ./kvmexit -p 3195281 # Collpase all tids for pid 3195281 with exit reasons sorted in descending order + ./kvmexit -p 3195281 20 # Collpase all tids for pid 3195281 with exit reasons sorted in descending order, and display after sleeping 20s + ./kvmexit -p 3195281 -v 0 # Display only vcpu0 for pid 3195281, descending sort by default + ./kvmexit -p 3195281 -a # Display all tids for pid 3195281 + ./kvmexit -t 395490 # Display only for tid 395490 with exit reasons sorted in descending order + ./kvmexit -t 395490 20 # Display only for tid 395490 with exit reasons sorted in descending order after sleeping 20s + ./kvmexit -T '395490,395491' # Display for a union like {395490, 395491} \ No newline at end of file From c7e7088770cc949cbc6c6f139160b2a6ecb8ae4d Mon Sep 17 00:00:00 2001 From: Y_y_s <78297703+Monkey857@users.noreply.github.com> Date: Fri, 2 Feb 2024 18:01:36 +0800 Subject: [PATCH 02/47] Update kvmexit.py --- eBPF_Supermarket/kvm_watcher/kvm_exit_bcc/kvmexit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/kvm_exit_bcc/kvmexit.py b/eBPF_Supermarket/kvm_watcher/kvm_exit_bcc/kvmexit.py index 2f82530f0..dd157488f 100644 --- a/eBPF_Supermarket/kvm_watcher/kvm_exit_bcc/kvmexit.py +++ b/eBPF_Supermarket/kvm_watcher/kvm_exit_bcc/kvmexit.py @@ -19,7 +19,7 @@ # @KVM_EXIT_REASON: the reason why the vm exits. # @COUNT: the counts of the @KVM_EXIT_REASONS. # -# REQUIRES: Linux 6.2 (BPF_PROG_TYPE_TRACEPOINT support) +# REQUIRES: Linux 4.7+ (BPF_PROG_TYPE_TRACEPOINT support) # # Copyright (c) 2024 YYS. All rights reserved. # Original code © 2024 ByteDance Inc. All rights reserved. @@ -375,4 +375,4 @@ def find_tid(tgt_dir, tgt_vcpu): for i in range(0, len(ct_reason)): if ct_reason[i][0] == 0: continue - print("%-35s %-8u" % (exit_reasons[ct_reason[i][1]], ct_reason[i][0])) \ No newline at end of file + print("%-35s %-8u" % (exit_reasons[ct_reason[i][1]], ct_reason[i][0])) From dbb0fc656ea959c7a26e701973c920ed22ce4b8a Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 1 Mar 2024 12:46:49 +0800 Subject: [PATCH 03/47] VMexit --- .../kvm_watcher/include/kvm_exits.h | 85 ++++++++++--------- .../kvm_watcher/include/kvm_watcher.h | 15 +++- .../kvm_watcher/src/kvm_watcher.bpf.c | 2 +- .../kvm_watcher/src/kvm_watcher.c | 75 +++++++++++----- 4 files changed, 116 insertions(+), 61 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h b/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h index d41e5cc3f..a7fe15b73 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h @@ -24,21 +24,25 @@ #include #include #include -// 定义哈希结构，存储时间信息 + +#define EXIT_REASON_HLT 12 + struct { __uint(type, BPF_MAP_TYPE_HASH); __uint(max_entries, 8192); - __type(key, pid_t); - __type(value, struct reason_info); -} times SEC(".maps"); + __type(key, struct exit_key); //exit_key:reason pid pad[2] + __type(value, struct exit_value); //exit_value : max_time total_time min_time count pad +} exit_map SEC(".maps"); struct { __uint(type, BPF_MAP_TYPE_HASH); __uint(max_entries, 8192); - __type(key, u32); - __type(value, u32); -} counts SEC(".maps"); -// 记录退出的信息 + __type(key, pid_t); + __type(value, struct reason_info); //reason_info:time、reason、count +} times SEC(".maps"); + + + struct exit { u64 pad; unsigned int exit_reason; @@ -51,38 +55,29 @@ struct exit { unsigned int vcpu_id; }; -int total = 0; -// 记录vm_exit的原因以及时间 static int trace_kvm_exit(struct exit *ctx, pid_t vm_pid) { CHECK_PID(vm_pid); + u32 reason; + reason = (u32)ctx->exit_reason; + //如果是节能停止退出，就不采集数据 + if(reason==EXIT_REASON_HLT){ + return 0; + } u64 id, ts; id = bpf_get_current_pid_tgid(); pid_t tid = (u32)id; ts = bpf_ktime_get_ns(); - u32 reason; - reason = (u32)ctx->exit_reason; struct reason_info reas = {}; reas.reason = reason; reas.time = ts; - u32 *count; - count = bpf_map_lookup_elem(&counts, &reason); - if (count) { - (*count)++; - reas.count = *count; - bpf_map_update_elem(&counts, &reason, count, BPF_ANY); - } else { - u32 new_count = 1; - reas.count = new_count; - bpf_map_update_elem(&counts, &reason, &new_count, BPF_ANY); - } bpf_map_update_elem(×, &tid, &reas, BPF_ANY); return 0; } -// 通过kvm_exit所记录的信息，来计算出整个处理的时间 -static int trace_kvm_entry(void *rb, struct common_event *e) { + +static int trace_kvm_entry() { struct reason_info *reas; pid_t pid, tid; - u64 id, ts, *start_ts, duration_ns = 0; + u64 id, ts, *start_ts, duration_ns; id = bpf_get_current_pid_tgid(); pid = id >> 32; tid = (u32)id; @@ -90,22 +85,32 @@ static int trace_kvm_entry(void *rb, struct common_event *e) { if (!reas) { return 0; } - u32 reason; - int count = 0; duration_ns = bpf_ktime_get_ns() - reas->time; bpf_map_delete_elem(×, &tid); - reason = reas->reason; - count = reas->count; - RESERVE_RINGBUF_ENTRY(rb, e); - e->exit_data.reason_number = reason; - e->process.pid = pid; - e->process.tid = tid; - e->exit_data.duration_ns = duration_ns; - bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm)); - e->exit_data.total = ++total; - e->exit_data.count = count; - e->time = reas->time; - bpf_ringbuf_submit(e, 0); + struct exit_key exit_key; + __builtin_memset(&exit_key, 0, sizeof(struct exit_key)); + exit_key.pid=pid; + exit_key.reason=reas->reason; + struct exit_value *exit_value; + exit_value = bpf_map_lookup_elem(&exit_map, &exit_key); + if (exit_value) { + exit_value->count ++; + exit_value->total_time +=duration_ns; + if(exit_value->max_time < duration_ns){ + exit_value->max_time = duration_ns; + } + if(exit_value->min_time > duration_ns){ + exit_value->min_time=duration_ns; + } + } else { + struct exit_value new_exit_value = { + .count=1, + .max_time=duration_ns, + .total_time=duration_ns, + .min_time=duration_ns + }; + bpf_map_update_elem(&exit_map, &exit_key, &new_exit_value, BPF_ANY); + } return 0; } #endif /* __KVM_EXITS_H */ diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h index 55d55343b..c388f0f2e 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h @@ -29,7 +29,7 @@ #define NS_TO_MS_WITH_DECIMAL(ns) ((double)(ns) / NS_TO_MS_FACTOR) #define MICROSECONDS_IN_SECOND 1000000 -#define OUTPUT_INTERVAL_SECONDS 0.5 +#define OUTPUT_INTERVAL_SECONDS 2 #define OUTPUT_INTERVAL(us) usleep((__u32)(us * MICROSECONDS_IN_SECOND)) @@ -96,7 +96,20 @@ struct ExitReason { struct reason_info { __u64 time; __u64 reason; +}; + +struct exit_key { + __u64 reason; + __u32 pid; + __u32 pad; +}; + +struct exit_value { + __u64 max_time; + __u64 total_time; + __u64 min_time; __u32 count; + __u32 pad; }; struct dirty_page_info { diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c index 0564e4704..b5c9e5462 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c @@ -60,7 +60,7 @@ int tp_exit(struct exit *ctx) { // 记录vm_entry和vm_exit的时间差 SEC("tp/kvm/kvm_entry") int tp_entry(struct exit *ctx) { - return trace_kvm_entry(&rb, e); + return trace_kvm_entry(); } SEC("kprobe/mark_page_dirty_in_slot") diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 127b5c69c..700aadc41 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -490,22 +490,6 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { break; } case EXIT: { - char info_buffer[256]; - // 使用 e->exit_data 访问 EXIT 特有成员 - printf("%-18.6f %-2d/%-18s %-18s %-6u/%-8u %-8d %-13.4f \n", - timestamp_ms, e->exit_data.reason_number, - getExitReasonName(e->exit_data.reason_number), - e->process.comm, e->process.pid, e->process.tid, - e->exit_data.count, - NS_TO_US_WITH_DECIMAL(e->exit_data.duration_ns)); - - if (env.ShowStats) { - snprintf(info_buffer, sizeof(info_buffer), "%-18s %-8u %-8d", - e->process.comm, e->process.pid, e->exit_data.count); - addExitInfo(&exitInfoBuffer, e->exit_data.reason_number, - info_buffer, e->exit_data.duration_ns, - e->exit_data.count); - } break; } case HALT_POLL: { @@ -661,8 +645,8 @@ static int print_event_head(struct env *env) { "VAILD?"); break; case EXIT: - printf("%-18s %-21s %-18s %-15s %-8s %-13s \n", "TIME(ms)", - "EXIT_REASON", "COMM", "PID/TID", "COUNT", "DURATION(us)"); + // printf("%-18s %-21s %-18s %-15s %-8s %-13s \n", "TIME(ms)", + // "EXIT_REASON", "COMM", "PID/TID", "COUNT", "DURATION(us)"); break; case HALT_POLL: printf("%-18s %-15s %-15s %-10s %-7s %-11s %-10s\n", "TIME(ms)", @@ -734,6 +718,57 @@ static void set_disable_load(struct kvm_watcher_bpf *skel) { env.execute_irq_inject ? true : false); } +int print_exit_map(struct kvm_watcher_bpf *skel) { + int fd = bpf_map__fd(skel->maps.exit_map); + int err; + struct exit_key lookup_key = {}; + struct exit_key next_key = {}; + struct exit_value exit_value; + struct tm *tm; + char ts[32]; + time_t t; + time(&t); + tm = localtime(&t); + strftime(ts, sizeof(ts), "%H:%M:%S", tm); + int first_run = 1; + // Iterate over the map + while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { + if (first_run) { + first_run = 0; + printf("\nTIME:%s\n", ts); + printf("%-12s %-12s %-12s %-12s %-12s %-12s\n", "pid", "total_time", + "max_time","min_time", "counts", "reason"); + printf( + "------------ ------------ ------------ ------------ ------------ " + "------------\n"); + } + // Print the current entry + err = bpf_map_lookup_elem(fd, &next_key, &exit_value); + if (err < 0) { + fprintf(stderr, "failed to lookup exit_value: %d\n", err); + return -1; + } + printf("%-12d %-12.4f %-12.4f %-12.4f %-12u %-12s\n", next_key.pid, + NS_TO_MS_WITH_DECIMAL(exit_value.total_time), + NS_TO_MS_WITH_DECIMAL(exit_value.max_time), + NS_TO_MS_WITH_DECIMAL(exit_value.min_time),exit_value.count, + getExitReasonName(next_key.reason)); + + // Move to the next key + lookup_key = next_key; + } + memset(&lookup_key, 0, sizeof(struct exit_key)); + while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { + err = bpf_map_delete_elem(fd, &next_key); + if (err < 0) { + fprintf(stderr, "failed to cleanup counters: %d\n", err); + return -1; + } + lookup_key = next_key; + } + return 0; +} + int main(int argc, char **argv) { // 定义一个环形缓冲区 struct ring_buffer *rb = NULL; @@ -806,8 +841,10 @@ int main(int argc, char **argv) { goto cleanup; } while (!exiting) { - // OUTPUT_INTERVAL(OUTPUT_INTERVAL_SECONDS); // 输出间隔 + err = ring_buffer__poll(rb, RING_BUFFER_TIMEOUT_MS /* timeout, ms */); + sleep(3); + err = print_exit_map(skel); /* Ctrl-C will cause -EINTR */ if (err == -EINTR) { err = 0; From 9190fa17b6e8c93a0d9e070da1394ddac212a52a Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 1 Mar 2024 13:24:30 +0800 Subject: [PATCH 04/47] =?UTF-8?q?=E4=BF=AE=E6=94=B9action?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eBPF_Supermarket/kvm_watcher/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eBPF_Supermarket/kvm_watcher/Makefile b/eBPF_Supermarket/kvm_watcher/Makefile index b330f8503..7aebb9cf2 100644 --- a/eBPF_Supermarket/kvm_watcher/Makefile +++ b/eBPF_Supermarket/kvm_watcher/Makefile @@ -14,7 +14,8 @@ define common_rules1 # 安装依赖 sudo apt install clang libelf1 libelf-dev zlib1g-dev libbpf-dev linux-tools-$$(uname -r) linux-cloud-tools-$$(uname -r) # 加载KVM模块 - sudo modprobe kvm && sudo modprobe kvm-intel + sudo modprobe kvm + sudo modprobe kvm-intel # 生成vmlinux.h文件 bpftool btf dump file /sys/kernel/btf/kvm format c > ./include/vmlinux.h endef From 47a3077f081b3f1a94397903acee146a073b7caf Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 1 Mar 2024 13:31:18 +0800 Subject: [PATCH 05/47] =?UTF-8?q?=E4=BF=AE=E6=94=B9action?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eBPF_Supermarket/kvm_watcher/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eBPF_Supermarket/kvm_watcher/Makefile b/eBPF_Supermarket/kvm_watcher/Makefile index 7aebb9cf2..887783cd4 100644 --- a/eBPF_Supermarket/kvm_watcher/Makefile +++ b/eBPF_Supermarket/kvm_watcher/Makefile @@ -15,7 +15,7 @@ define common_rules1 sudo apt install clang libelf1 libelf-dev zlib1g-dev libbpf-dev linux-tools-$$(uname -r) linux-cloud-tools-$$(uname -r) # 加载KVM模块 sudo modprobe kvm - sudo modprobe kvm-intel + #sudo modprobe kvm-intel # 生成vmlinux.h文件 bpftool btf dump file /sys/kernel/btf/kvm format c > ./include/vmlinux.h endef From 389a1921e2647a6df62145022583c5f43648eb84 Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 1 Mar 2024 13:36:39 +0800 Subject: [PATCH 06/47] =?UTF-8?q?=E4=BF=AE=E6=94=B9makefile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eBPF_Supermarket/kvm_watcher/Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/Makefile b/eBPF_Supermarket/kvm_watcher/Makefile index 887783cd4..a6de86633 100644 --- a/eBPF_Supermarket/kvm_watcher/Makefile +++ b/eBPF_Supermarket/kvm_watcher/Makefile @@ -7,15 +7,14 @@ ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \ | sed 's/riscv64/riscv/' \ | sed 's/loongarch64/loongarch/') APP = src/kvm_watcher -OPTIONS = -f -w -n -d -c '-e -s' +OPTIONS = -f -w -n -d -c -e # 共同规则1 define common_rules1 # 安装依赖 sudo apt install clang libelf1 libelf-dev zlib1g-dev libbpf-dev linux-tools-$$(uname -r) linux-cloud-tools-$$(uname -r) # 加载KVM模块 - sudo modprobe kvm - #sudo modprobe kvm-intel + sudo modprobe kvm && modprobe kvm-intel # 生成vmlinux.h文件 bpftool btf dump file /sys/kernel/btf/kvm format c > ./include/vmlinux.h endef From 8d1830634aa97305c7142bc911275b079d6264cd Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 1 Mar 2024 13:46:05 +0800 Subject: [PATCH 07/47] update makefile --- eBPF_Supermarket/kvm_watcher/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/Makefile b/eBPF_Supermarket/kvm_watcher/Makefile index a6de86633..fe0ad9eb5 100644 --- a/eBPF_Supermarket/kvm_watcher/Makefile +++ b/eBPF_Supermarket/kvm_watcher/Makefile @@ -7,14 +7,15 @@ ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \ | sed 's/riscv64/riscv/' \ | sed 's/loongarch64/loongarch/') APP = src/kvm_watcher -OPTIONS = -f -w -n -d -c -e +OPTIONS = -f -w -n -d -e # 共同规则1 define common_rules1 # 安装依赖 sudo apt install clang libelf1 libelf-dev zlib1g-dev libbpf-dev linux-tools-$$(uname -r) linux-cloud-tools-$$(uname -r) # 加载KVM模块 - sudo modprobe kvm && modprobe kvm-intel + sudo modprobe kvm + #sudo modprobe kvm-intel # 生成vmlinux.h文件 bpftool btf dump file /sys/kernel/btf/kvm format c > ./include/vmlinux.h endef From f4f9ce230c0dcf944432ceba4838fce4086ac917 Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 1 Mar 2024 14:15:41 +0800 Subject: [PATCH 08/47] update yml --- .github/workflows/kvm_watcher.yml | 2 +- eBPF_Supermarket/kvm_watcher/Makefile | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/kvm_watcher.yml b/.github/workflows/kvm_watcher.yml index 49b22a6c0..0a3f7525a 100644 --- a/.github/workflows/kvm_watcher.yml +++ b/.github/workflows/kvm_watcher.yml @@ -23,5 +23,5 @@ jobs: - name: Test program execution run: | cd eBPF_Supermarket/kvm_watcher/ - make test + make diff --git a/eBPF_Supermarket/kvm_watcher/Makefile b/eBPF_Supermarket/kvm_watcher/Makefile index fe0ad9eb5..5264cb9a3 100644 --- a/eBPF_Supermarket/kvm_watcher/Makefile +++ b/eBPF_Supermarket/kvm_watcher/Makefile @@ -13,9 +13,6 @@ OPTIONS = -f -w -n -d -e define common_rules1 # 安装依赖 sudo apt install clang libelf1 libelf-dev zlib1g-dev libbpf-dev linux-tools-$$(uname -r) linux-cloud-tools-$$(uname -r) - # 加载KVM模块 - sudo modprobe kvm - #sudo modprobe kvm-intel # 生成vmlinux.h文件 bpftool btf dump file /sys/kernel/btf/kvm format c > ./include/vmlinux.h endef @@ -45,11 +42,13 @@ ifeq ($(MAKECMDGOALS),test) ifeq ($(shell grep -Eoc '(vmx|svm)' /proc/cpuinfo),0) $(error "The CPU in your device does not support virtualization!") endif + # 加载KVM模块 + sudo modprobe kvm && sudo modprobe kvm-intel + $(common_rules1) + $(common_rules2) ifeq ($(wildcard ./cirros-0.5.2-x86_64-disk.img),) wget https://gitee.com/nan-shuaibo/cirros/releases/download/0.5.2/cirros-0.5.2-x86_64-disk.img endif - $(common_rules1) - $(common_rules2) # 安装 qemu $(INSTALL_QEMU) # 启动虚拟机 From 22489668a2e588e01b1f4b8aff43215549a2e7e9 Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 1 Mar 2024 14:21:13 +0800 Subject: [PATCH 09/47] =?UTF-8?q?=E8=B0=83=E6=95=B4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kvm_watcher/include/kvm_exits.h | 37 +++++++++---------- .../kvm_watcher/src/kvm_watcher.c | 13 ++++--- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h b/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h index a7fe15b73..4171c89e8 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h @@ -25,24 +25,23 @@ #include #include -#define EXIT_REASON_HLT 12 +#define EXIT_REASON_HLT 12 struct { __uint(type, BPF_MAP_TYPE_HASH); __uint(max_entries, 8192); - __type(key, struct exit_key); //exit_key:reason pid pad[2] - __type(value, struct exit_value); //exit_value : max_time total_time min_time count pad + __type(key, struct exit_key); // exit_key:reason pid pad[2] + __type(value, struct exit_value); // exit_value : max_time total_time + // min_time count pad } exit_map SEC(".maps"); struct { __uint(type, BPF_MAP_TYPE_HASH); __uint(max_entries, 8192); __type(key, pid_t); - __type(value, struct reason_info); //reason_info:time、reason、count + __type(value, struct reason_info); // reason_info:time、reason、count } times SEC(".maps"); - - struct exit { u64 pad; unsigned int exit_reason; @@ -60,7 +59,7 @@ static int trace_kvm_exit(struct exit *ctx, pid_t vm_pid) { u32 reason; reason = (u32)ctx->exit_reason; //如果是节能停止退出，就不采集数据 - if(reason==EXIT_REASON_HLT){ + if (reason == EXIT_REASON_HLT) { return 0; } u64 id, ts; @@ -89,26 +88,24 @@ static int trace_kvm_entry() { bpf_map_delete_elem(×, &tid); struct exit_key exit_key; __builtin_memset(&exit_key, 0, sizeof(struct exit_key)); - exit_key.pid=pid; - exit_key.reason=reas->reason; + exit_key.pid = pid; + exit_key.reason = reas->reason; struct exit_value *exit_value; exit_value = bpf_map_lookup_elem(&exit_map, &exit_key); if (exit_value) { - exit_value->count ++; - exit_value->total_time +=duration_ns; - if(exit_value->max_time < duration_ns){ + exit_value->count++; + exit_value->total_time += duration_ns; + if (exit_value->max_time < duration_ns) { exit_value->max_time = duration_ns; } - if(exit_value->min_time > duration_ns){ - exit_value->min_time=duration_ns; + if (exit_value->min_time > duration_ns) { + exit_value->min_time = duration_ns; } } else { - struct exit_value new_exit_value = { - .count=1, - .max_time=duration_ns, - .total_time=duration_ns, - .min_time=duration_ns - }; + struct exit_value new_exit_value = {.count = 1, + .max_time = duration_ns, + .total_time = duration_ns, + .min_time = duration_ns}; bpf_map_update_elem(&exit_map, &exit_key, &new_exit_value, BPF_ANY); } return 0; diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 700aadc41..3c11e0b3a 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -646,7 +646,8 @@ static int print_event_head(struct env *env) { break; case EXIT: // printf("%-18s %-21s %-18s %-15s %-8s %-13s \n", "TIME(ms)", - // "EXIT_REASON", "COMM", "PID/TID", "COUNT", "DURATION(us)"); + // "EXIT_REASON", "COMM", "PID/TID", "COUNT", + // "DURATION(us)"); break; case HALT_POLL: printf("%-18s %-15s %-15s %-10s %-7s %-11s %-10s\n", "TIME(ms)", @@ -737,9 +738,10 @@ int print_exit_map(struct kvm_watcher_bpf *skel) { first_run = 0; printf("\nTIME:%s\n", ts); printf("%-12s %-12s %-12s %-12s %-12s %-12s\n", "pid", "total_time", - "max_time","min_time", "counts", "reason"); + "max_time", "min_time", "counts", "reason"); printf( - "------------ ------------ ------------ ------------ ------------ " + "------------ ------------ ------------ ------------ " + "------------ " "------------\n"); } // Print the current entry @@ -751,7 +753,7 @@ int print_exit_map(struct kvm_watcher_bpf *skel) { printf("%-12d %-12.4f %-12.4f %-12.4f %-12u %-12s\n", next_key.pid, NS_TO_MS_WITH_DECIMAL(exit_value.total_time), NS_TO_MS_WITH_DECIMAL(exit_value.max_time), - NS_TO_MS_WITH_DECIMAL(exit_value.min_time),exit_value.count, + NS_TO_MS_WITH_DECIMAL(exit_value.min_time), exit_value.count, getExitReasonName(next_key.reason)); // Move to the next key @@ -841,7 +843,6 @@ int main(int argc, char **argv) { goto cleanup; } while (!exiting) { - err = ring_buffer__poll(rb, RING_BUFFER_TIMEOUT_MS /* timeout, ms */); sleep(3); err = print_exit_map(skel); @@ -863,7 +864,7 @@ int main(int argc, char **argv) { if (err < 0) { printf("Save count dirty page map to file fail: %d\n", err); goto cleanup; - }else{ + } else { printf("\nSave count dirty page map to file success!\n"); goto cleanup; } From 1e4a8804c44cde069165af4d8da6e05050406b86 Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 8 Mar 2024 11:23:59 +0800 Subject: [PATCH 10/47] vm exit --- .../kvm_watcher/include/kvm_exits.h | 1 + .../kvm_watcher/include/kvm_watcher.h | 1 + .../kvm_watcher/src/kvm_watcher.c | 96 +++++++++++++++---- 3 files changed, 79 insertions(+), 19 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h b/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h index 4171c89e8..9fb034a07 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h @@ -89,6 +89,7 @@ static int trace_kvm_entry() { struct exit_key exit_key; __builtin_memset(&exit_key, 0, sizeof(struct exit_key)); exit_key.pid = pid; + exit_key.tid = tid; exit_key.reason = reas->reason; struct exit_value *exit_value; exit_value = bpf_map_lookup_elem(&exit_map, &exit_key); diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h index 3a8a8df61..25a00063c 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h @@ -96,6 +96,7 @@ struct reason_info { struct exit_key { __u64 reason; __u32 pid; + __u32 tid; __u32 pad; }; diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index b1261b056..1c22db470 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -785,13 +785,56 @@ int print_hc_map(struct kvm_watcher_bpf *skel) { } return 0; } +// In order to sort vm_exit maps +int sort_by_key(struct kvm_watcher_bpf *skel, struct exit_key *keys, + struct exit_value *values) { + int fd = bpf_map__fd(skel->maps.exit_map); + int err = 0; + struct exit_key lookup_key = {}; + struct exit_key next_key = {}; + struct exit_value exit_value; + int first = 1; + int i = 0, j; + int count = 0; + while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { + count++; + if (first) { + first = 0; + bpf_map_lookup_elem(fd, &next_key, &exit_value); + keys[0] = next_key; + values[0] = exit_value; + i++; + continue; + } + err = bpf_map_lookup_elem(fd, &next_key, &exit_value); + if (err < 0) { + fprintf(stderr, "failed to lookup exit_value: %d\n", err); + return -1; + } + //insert sort + j = i - 1; + struct exit_key temp_key = next_key; + struct exit_value temp_value = exit_value; + while (j >= 0 && + (keys[j].pid > temp_key.pid || (keys[j].tid > temp_key.tid))) { + keys[j + 1] = keys[j]; + values[j + 1] = values[j]; + j--; + } + i++; + keys[j + 1] = next_key; + values[j + 1] = temp_value; + // Move to the next key + lookup_key = next_key; + } + return count; +} int print_exit_map(struct kvm_watcher_bpf *skel) { int fd = bpf_map__fd(skel->maps.exit_map); int err; struct exit_key lookup_key = {}; struct exit_key next_key = {}; - struct exit_value exit_value; struct tm *tm; char ts[32]; time_t t; @@ -799,33 +842,46 @@ int print_exit_map(struct kvm_watcher_bpf *skel) { tm = localtime(&t); strftime(ts, sizeof(ts), "%H:%M:%S", tm); int first_run = 1; - // Iterate over the map - while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { + struct exit_key keys[8192]; + struct exit_value values[8192]; + int count = sort_by_key(skel, keys, values); + // Iterate over the array + __u32 pid = 0; + __u32 tid = 0; + for (int i = 0; i < count; i++) { if (first_run) { first_run = 0; printf("\nTIME:%s\n", ts); - printf("%-12s %-12s %-12s %-12s %-12s %-12s\n", "pid", "total_time", - "max_time", "min_time", "counts", "reason"); + printf("%-12s %-12s %-12s %-12s %-12s %-12s %-12s\n", "pid", "tid", + "total_time", "max_time", "min_time", "counts", "reason"); printf( "------------ ------------ ------------ ------------ " "------------ " "------------\n"); } // Print the current entry - err = bpf_map_lookup_elem(fd, &next_key, &exit_value); - if (err < 0) { - fprintf(stderr, "failed to lookup exit_value: %d\n", err); - return -1; + if (tid == 0 || tid != keys[i].tid) { + tid = keys[i].tid; + if (pid == 0 || pid != keys[i].pid) { + pid = keys[i].pid; + printf("%-12d", pid); + } else { + printf("%-12s", ""); + } + printf("%-12d %-12.4f %-12.4f %-12.4f %-12u %-12s\n", keys[i].tid, + NS_TO_MS_WITH_DECIMAL(values[i].total_time), + NS_TO_MS_WITH_DECIMAL(values[i].max_time), + NS_TO_MS_WITH_DECIMAL(values[i].min_time), values[i].count, + getExitReasonName(keys[i].reason)); + } else if (tid == keys[i].tid) { + printf("%24s %-12.4f %-12.4f %-12.4f %-12u %-12s\n", "", + NS_TO_MS_WITH_DECIMAL(values[i].total_time), + NS_TO_MS_WITH_DECIMAL(values[i].max_time), + NS_TO_MS_WITH_DECIMAL(values[i].min_time), values[i].count, + getExitReasonName(keys[i].reason)); } - printf("%-12d %-12.4f %-12.4f %-12.4f %-12u %-12s\n", next_key.pid, - NS_TO_MS_WITH_DECIMAL(exit_value.total_time), - NS_TO_MS_WITH_DECIMAL(exit_value.max_time), - NS_TO_MS_WITH_DECIMAL(exit_value.min_time), exit_value.count, - getExitReasonName(next_key.reason)); - - // Move to the next key - lookup_key = next_key; } + // clear the maps memset(&lookup_key, 0, sizeof(struct exit_key)); while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { err = bpf_map_delete_elem(fd, &next_key); @@ -838,7 +894,9 @@ int print_exit_map(struct kvm_watcher_bpf *skel) { return 0; } -void print_map_and_check_error(int (*print_func)(struct kvm_watcher_bpf *), struct kvm_watcher_bpf *skel, const char *map_name, int err) { +void print_map_and_check_error(int (*print_func)(struct kvm_watcher_bpf *), + struct kvm_watcher_bpf *skel, + const char *map_name, int err) { OUTPUT_INTERVAL(OUTPUT_INTERVAL_SECONDS); print_func(skel); if (err < 0) { @@ -919,7 +977,7 @@ int main(int argc, char **argv) { } while (!exiting) { err = ring_buffer__poll(rb, RING_BUFFER_TIMEOUT_MS /* timeout, ms */); - + if (env.execute_hypercall) { print_map_and_check_error(print_hc_map, skel, "hypercall", err); } From 15484302f21aae063cdae8c938edd57014cfb567 Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 8 Mar 2024 17:41:57 +0800 Subject: [PATCH 11/47] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h | 1 - eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h index 25a00063c..a1a81e037 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h @@ -97,7 +97,6 @@ struct exit_key { __u64 reason; __u32 pid; __u32 tid; - __u32 pad; }; struct exit_value { diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 1c22db470..96889285a 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -812,7 +812,7 @@ int sort_by_key(struct kvm_watcher_bpf *skel, struct exit_key *keys, fprintf(stderr, "failed to lookup exit_value: %d\n", err); return -1; } - //insert sort + // insert sort j = i - 1; struct exit_key temp_key = next_key; struct exit_value temp_value = exit_value; @@ -857,6 +857,7 @@ int print_exit_map(struct kvm_watcher_bpf *skel) { printf( "------------ ------------ ------------ ------------ " "------------ " + "------------ " "------------\n"); } // Print the current entry @@ -864,9 +865,9 @@ int print_exit_map(struct kvm_watcher_bpf *skel) { tid = keys[i].tid; if (pid == 0 || pid != keys[i].pid) { pid = keys[i].pid; - printf("%-12d", pid); + printf("%-13d", pid); } else { - printf("%-12s", ""); + printf("%-13s", ""); } printf("%-12d %-12.4f %-12.4f %-12.4f %-12u %-12s\n", keys[i].tid, NS_TO_MS_WITH_DECIMAL(values[i].total_time), @@ -874,7 +875,7 @@ int print_exit_map(struct kvm_watcher_bpf *skel) { NS_TO_MS_WITH_DECIMAL(values[i].min_time), values[i].count, getExitReasonName(keys[i].reason)); } else if (tid == keys[i].tid) { - printf("%24s %-12.4f %-12.4f %-12.4f %-12u %-12s\n", "", + printf("%25s %-12.4f %-12.4f %-12.4f %-12u %-12s\n", "", NS_TO_MS_WITH_DECIMAL(values[i].total_time), NS_TO_MS_WITH_DECIMAL(values[i].max_time), NS_TO_MS_WITH_DECIMAL(values[i].min_time), values[i].count, From a1ca34d120c24c2afcabdd8fa826041dadbbad97 Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 22 Mar 2024 14:07:03 +0800 Subject: [PATCH 12/47] modify sort_bug --- .../kvm_watcher/src/kvm_watcher.c | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 2746149fe..2259a45c4 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -257,6 +257,7 @@ int save_count_dirtypagemap_to_file(struct bpf_map *map) { static struct env { bool execute_vcpu_wakeup; bool execute_exit; + bool execute_vcpu_load; bool execute_halt_poll_ns; bool execute_mark_page_dirty; bool execute_page_fault; @@ -270,6 +271,7 @@ static struct env { enum EventType event_type; } env = { .execute_vcpu_wakeup = false, + .execute_vcpu_load = false, .execute_exit = false, .execute_halt_poll_ns = false, .execute_mark_page_dirty = false, @@ -291,6 +293,7 @@ int option_selected = 0; // 功能标志变量,确保激活子功能 // 具体解释命令行参数 static const struct argp_option opts[] = { {"vcpu_wakeup", 'w', NULL, 0, "Monitoring the wakeup of vcpu."}, + {"vcpu_load", 'o', NULL, 0, "Monitoring the load of vcpu."}, {"vm_exit", 'e', NULL, 0, "Monitoring the event of vm exit."}, {"halt_poll_ns", 'n', NULL, 0, "Monitoring the variation in vCPU halt-polling time."}, @@ -325,6 +328,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) { case 'e': SET_OPTION_AND_CHECK_USAGE(option_selected, env.execute_exit); break; + case 'o': + SET_OPTION_AND_CHECK_USAGE(option_selected, env.execute_vcpu_load); + break; case 'n': SET_OPTION_AND_CHECK_USAGE(option_selected, env.execute_halt_poll_ns); @@ -424,6 +430,8 @@ static int determineEventType(struct env *env) { env->event_type = HYPERCALL; } else if (env->execute_ioctl) { env->event_type = IOCTL; + } else if (env->execute_vcpu_load) { + env->event_type = VCPU_LOAD; } else { env->event_type = NONE_TYPE; // 或者根据需要设置一个默认的事件类型 } @@ -449,6 +457,12 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { case EXIT: { break; } + case VCPU_LOAD: { + printf("%-20.6f %-15s %-6d/%-8d %-10d\n", timestamp_ms, + e->process.comm, e->process.pid, e->process.tid, + e->vcpu_load_data.vcpu_id); + break; + } case HALT_POLL: { // 使用 e->halt_poll_data 访问 HALT_POLL 特有成员 printf("%-18.6f %-15s %-6d/%-8d %-10s %-7d %-7d --> %d \n", @@ -695,6 +709,8 @@ static int print_event_head(struct env *env) { static void set_disable_load(struct kvm_watcher_bpf *skel) { bpf_program__set_autoload(skel->progs.tp_vcpu_wakeup, env.execute_vcpu_wakeup ? true : false); + bpf_program__set_autoload(skel->progs.kp_vmx_vcpu_load, + env.execute_vcpu_load ? true : false); bpf_program__set_autoload(skel->progs.fentry_kvm_vcpu_halt, env.execute_vcpu_wakeup ? true : false); bpf_program__set_autoload(skel->progs.tp_exit, @@ -821,10 +837,10 @@ int sort_by_key(struct kvm_watcher_bpf *skel, struct exit_key *keys, keys[0] = next_key; values[0] = exit_value; i++; + lookup_key = next_key; continue; } err = bpf_map_lookup_elem(fd, &next_key, &exit_value); - if (err < 0) { fprintf(stderr, "failed to lookup exit_value: %d\n", err); return -1; @@ -890,7 +906,7 @@ int print_exit_map(struct kvm_watcher_bpf *skel) { NS_TO_MS_WITH_DECIMAL(values[i].total_time), NS_TO_MS_WITH_DECIMAL(values[i].max_time), NS_TO_MS_WITH_DECIMAL(values[i].min_time), values[i].count, - getName(keys[i].reason, EXIT)); + getName(keys[i].reason, EXIT)); } } // clear the maps @@ -905,7 +921,6 @@ int print_exit_map(struct kvm_watcher_bpf *skel) { } return 0; } - void print_map_and_check_error(int (*print_func)(struct kvm_watcher_bpf *), struct kvm_watcher_bpf *skel, const char *map_name, int err) { @@ -915,7 +930,6 @@ void print_map_and_check_error(int (*print_func)(struct kvm_watcher_bpf *), printf("Error printing %s map: %d\n", map_name, err); } } - int main(int argc, char **argv) { // 定义一个环形缓冲区 struct ring_buffer *rb = NULL; From 0c02211a10cf1b811b211572a9970a98e959435e Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 22 Mar 2024 14:22:36 +0800 Subject: [PATCH 13/47] modify sort_bug --- .../kvm_watcher/src/kvm_watcher.c | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 2259a45c4..5ece0ec34 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -257,7 +257,6 @@ int save_count_dirtypagemap_to_file(struct bpf_map *map) { static struct env { bool execute_vcpu_wakeup; bool execute_exit; - bool execute_vcpu_load; bool execute_halt_poll_ns; bool execute_mark_page_dirty; bool execute_page_fault; @@ -271,7 +270,6 @@ static struct env { enum EventType event_type; } env = { .execute_vcpu_wakeup = false, - .execute_vcpu_load = false, .execute_exit = false, .execute_halt_poll_ns = false, .execute_mark_page_dirty = false, @@ -293,7 +291,6 @@ int option_selected = 0; // 功能标志变量,确保激活子功能 // 具体解释命令行参数 static const struct argp_option opts[] = { {"vcpu_wakeup", 'w', NULL, 0, "Monitoring the wakeup of vcpu."}, - {"vcpu_load", 'o', NULL, 0, "Monitoring the load of vcpu."}, {"vm_exit", 'e', NULL, 0, "Monitoring the event of vm exit."}, {"halt_poll_ns", 'n', NULL, 0, "Monitoring the variation in vCPU halt-polling time."}, @@ -328,9 +325,6 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) { case 'e': SET_OPTION_AND_CHECK_USAGE(option_selected, env.execute_exit); break; - case 'o': - SET_OPTION_AND_CHECK_USAGE(option_selected, env.execute_vcpu_load); - break; case 'n': SET_OPTION_AND_CHECK_USAGE(option_selected, env.execute_halt_poll_ns); @@ -430,8 +424,6 @@ static int determineEventType(struct env *env) { env->event_type = HYPERCALL; } else if (env->execute_ioctl) { env->event_type = IOCTL; - } else if (env->execute_vcpu_load) { - env->event_type = VCPU_LOAD; } else { env->event_type = NONE_TYPE; // 或者根据需要设置一个默认的事件类型 } @@ -457,12 +449,6 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { case EXIT: { break; } - case VCPU_LOAD: { - printf("%-20.6f %-15s %-6d/%-8d %-10d\n", timestamp_ms, - e->process.comm, e->process.pid, e->process.tid, - e->vcpu_load_data.vcpu_id); - break; - } case HALT_POLL: { // 使用 e->halt_poll_data 访问 HALT_POLL 特有成员 printf("%-18.6f %-15s %-6d/%-8d %-10s %-7d %-7d --> %d \n", @@ -709,8 +695,6 @@ static int print_event_head(struct env *env) { static void set_disable_load(struct kvm_watcher_bpf *skel) { bpf_program__set_autoload(skel->progs.tp_vcpu_wakeup, env.execute_vcpu_wakeup ? true : false); - bpf_program__set_autoload(skel->progs.kp_vmx_vcpu_load, - env.execute_vcpu_load ? true : false); bpf_program__set_autoload(skel->progs.fentry_kvm_vcpu_halt, env.execute_vcpu_wakeup ? true : false); bpf_program__set_autoload(skel->progs.tp_exit, @@ -841,6 +825,7 @@ int sort_by_key(struct kvm_watcher_bpf *skel, struct exit_key *keys, continue; } err = bpf_map_lookup_elem(fd, &next_key, &exit_value); + if (err < 0) { fprintf(stderr, "failed to lookup exit_value: %d\n", err); return -1; @@ -921,6 +906,7 @@ int print_exit_map(struct kvm_watcher_bpf *skel) { } return 0; } + void print_map_and_check_error(int (*print_func)(struct kvm_watcher_bpf *), struct kvm_watcher_bpf *skel, const char *map_name, int err) { @@ -930,6 +916,7 @@ void print_map_and_check_error(int (*print_func)(struct kvm_watcher_bpf *), printf("Error printing %s map: %d\n", map_name, err); } } + int main(int argc, char **argv) { // 定义一个环形缓冲区 struct ring_buffer *rb = NULL; From 013e14695299ee5bcd3d694e91f49996486397fe Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 22 Mar 2024 14:38:23 +0800 Subject: [PATCH 14/47] add fun --- .../kvm_watcher/include/kvm_vcpu.h | 23 ++++++++++++++++++- .../kvm_watcher/include/kvm_watcher.h | 7 +++++- .../kvm_watcher/src/kvm_watcher.bpf.c | 6 ++++- .../kvm_watcher/src/kvm_watcher.c | 19 ++++++++++++--- 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h b/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h index 7bba6477d..4be02a2c4 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h @@ -53,6 +53,7 @@ struct { __type(key, u32); __type(value, u32); } vcpu_tid SEC(".maps"); + // 记录vcpu_halt的id信息 static int trace_kvm_vcpu_halt(struct kvm_vcpu *vcpu, pid_t vm_pid) { CHECK_PID(vm_pid); @@ -102,7 +103,27 @@ static int trace_kvm_halt_poll_ns(struct halt_poll_ns *ctx, void *rb, bpf_ringbuf_submit(e, 0); return 0; } - +//记录VCPU调度的信息 +static int trace_vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu, void *rb, + struct common_event *e) { + RESERVE_RINGBUF_ENTRY(rb, e); + //获取pid&tid + pid_t pid, tid; + u64 id; + id = bpf_get_current_pid_tgid(); + pid = id >> 32; + tid = (u32)id; + // //获取时间 + u64 ts = bpf_ktime_get_ns(); + e->process.pid = pid; + e->process.tid = tid; + e->time = ts; + bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm)); + bpf_probe_read_kernel(&e->vcpu_load_data.vcpu_id, + sizeof(e->vcpu_load_data.vcpu_id), &vcpu->vcpu_id); + bpf_ringbuf_submit(e, 0); + return 1; +} static int trace_mark_page_dirty_in_slot(struct kvm *kvm, const struct kvm_memory_slot *memslot, gfn_t gfn, void *rb, diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h index 97c77244b..d94ea6bfa 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h @@ -89,7 +89,6 @@ struct reason_info { __u64 time; __u64 reason; }; - struct exit_key { __u64 reason; __u32 pid; @@ -137,6 +136,7 @@ struct process { enum EventType { NONE_TYPE, VCPU_WAKEUP, + VCPU_LOAD, EXIT, HALT_POLL, MARK_PAGE_DIRTY, @@ -161,6 +161,11 @@ struct common_event { // VCPU_WAKEUP 特有成员 } vcpu_wakeup_data; + struct { + __u32 vcpu_id; + // VCPU_LOAD 特有成员 + } vcpu_load_data; + struct { __u32 reason_number; __u64 duration_ns; diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c index 406e5d273..e7d480cc3 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c @@ -64,7 +64,11 @@ SEC("tp/kvm/kvm_entry") int tp_entry(struct exit *ctx) { return trace_kvm_entry(); } - +//记录VCPU调度的信息 +SEC("kprobe/vmx_vcpu_load") +int BPF_KPROBE(kp_vmx_vcpu_load, struct kvm_vcpu *vcpu, int cpu) { + return trace_vmx_vcpu_load(vcpu, cpu, &rb, e); +} SEC("kprobe/mark_page_dirty_in_slot") int BPF_KPROBE(kp_mark_page_dirty_in_slot, struct kvm *kvm, const struct kvm_memory_slot *memslot, gfn_t gfn) { diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 5ece0ec34..2259a45c4 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -257,6 +257,7 @@ int save_count_dirtypagemap_to_file(struct bpf_map *map) { static struct env { bool execute_vcpu_wakeup; bool execute_exit; + bool execute_vcpu_load; bool execute_halt_poll_ns; bool execute_mark_page_dirty; bool execute_page_fault; @@ -270,6 +271,7 @@ static struct env { enum EventType event_type; } env = { .execute_vcpu_wakeup = false, + .execute_vcpu_load = false, .execute_exit = false, .execute_halt_poll_ns = false, .execute_mark_page_dirty = false, @@ -291,6 +293,7 @@ int option_selected = 0; // 功能标志变量,确保激活子功能 // 具体解释命令行参数 static const struct argp_option opts[] = { {"vcpu_wakeup", 'w', NULL, 0, "Monitoring the wakeup of vcpu."}, + {"vcpu_load", 'o', NULL, 0, "Monitoring the load of vcpu."}, {"vm_exit", 'e', NULL, 0, "Monitoring the event of vm exit."}, {"halt_poll_ns", 'n', NULL, 0, "Monitoring the variation in vCPU halt-polling time."}, @@ -325,6 +328,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) { case 'e': SET_OPTION_AND_CHECK_USAGE(option_selected, env.execute_exit); break; + case 'o': + SET_OPTION_AND_CHECK_USAGE(option_selected, env.execute_vcpu_load); + break; case 'n': SET_OPTION_AND_CHECK_USAGE(option_selected, env.execute_halt_poll_ns); @@ -424,6 +430,8 @@ static int determineEventType(struct env *env) { env->event_type = HYPERCALL; } else if (env->execute_ioctl) { env->event_type = IOCTL; + } else if (env->execute_vcpu_load) { + env->event_type = VCPU_LOAD; } else { env->event_type = NONE_TYPE; // 或者根据需要设置一个默认的事件类型 } @@ -449,6 +457,12 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { case EXIT: { break; } + case VCPU_LOAD: { + printf("%-20.6f %-15s %-6d/%-8d %-10d\n", timestamp_ms, + e->process.comm, e->process.pid, e->process.tid, + e->vcpu_load_data.vcpu_id); + break; + } case HALT_POLL: { // 使用 e->halt_poll_data 访问 HALT_POLL 特有成员 printf("%-18.6f %-15s %-6d/%-8d %-10s %-7d %-7d --> %d \n", @@ -695,6 +709,8 @@ static int print_event_head(struct env *env) { static void set_disable_load(struct kvm_watcher_bpf *skel) { bpf_program__set_autoload(skel->progs.tp_vcpu_wakeup, env.execute_vcpu_wakeup ? true : false); + bpf_program__set_autoload(skel->progs.kp_vmx_vcpu_load, + env.execute_vcpu_load ? true : false); bpf_program__set_autoload(skel->progs.fentry_kvm_vcpu_halt, env.execute_vcpu_wakeup ? true : false); bpf_program__set_autoload(skel->progs.tp_exit, @@ -825,7 +841,6 @@ int sort_by_key(struct kvm_watcher_bpf *skel, struct exit_key *keys, continue; } err = bpf_map_lookup_elem(fd, &next_key, &exit_value); - if (err < 0) { fprintf(stderr, "failed to lookup exit_value: %d\n", err); return -1; @@ -906,7 +921,6 @@ int print_exit_map(struct kvm_watcher_bpf *skel) { } return 0; } - void print_map_and_check_error(int (*print_func)(struct kvm_watcher_bpf *), struct kvm_watcher_bpf *skel, const char *map_name, int err) { @@ -916,7 +930,6 @@ void print_map_and_check_error(int (*print_func)(struct kvm_watcher_bpf *), printf("Error printing %s map: %d\n", map_name, err); } } - int main(int argc, char **argv) { // 定义一个环形缓冲区 struct ring_buffer *rb = NULL; From 26537d1c149a29f63900b06b0345fe2c4cd4ee0a Mon Sep 17 00:00:00 2001 From: YYS Date: Tue, 2 Apr 2024 14:47:32 +0800 Subject: [PATCH 15/47] vcpu_load --- .../kvm_watcher/include/kvm_vcpu.h | 94 +++++++++++++++---- .../kvm_watcher/include/kvm_watcher.h | 23 ++++- .../kvm_watcher/src/kvm_watcher.bpf.c | 9 +- .../kvm_watcher/src/kvm_watcher.c | 58 +++++++++++- 4 files changed, 154 insertions(+), 30 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h b/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h index f38a939b2..a06fc4a88 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h @@ -54,6 +54,20 @@ struct { __type(value, u32); } vcpu_tid SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 8192); + __type(key, struct load_key); + __type(value, struct load_value); +} load_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 8192); + __type(key, struct load_key); + __type(value, struct time_value); +} load_time SEC(".maps"); + // 记录vcpu_halt的id信息 static int trace_kvm_vcpu_halt(struct kvm_vcpu *vcpu) { u32 tid = bpf_get_current_pid_tgid(); @@ -102,26 +116,66 @@ static int trace_kvm_halt_poll_ns(struct halt_poll_ns *ctx, void *rb, bpf_ringbuf_submit(e, 0); return 0; } -//记录VCPU调度的信息 -static int trace_vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu, void *rb, - struct common_event *e) { - RESERVE_RINGBUF_ENTRY(rb, e); - //获取pid&tid - pid_t pid, tid; - u64 id; - id = bpf_get_current_pid_tgid(); - pid = id >> 32; - tid = (u32)id; - // //获取时间 - u64 ts = bpf_ktime_get_ns(); - e->process.pid = pid; - e->process.tid = tid; - e->time = ts; - bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm)); - bpf_probe_read_kernel(&e->vcpu_load_data.vcpu_id, - sizeof(e->vcpu_load_data.vcpu_id), &vcpu->vcpu_id); - bpf_ringbuf_submit(e, 0); - return 1; +//记录VCPU调度的信息--进调度 +static int trace_vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { + u32 pid = bpf_get_current_pid_tgid() >> 32; + u32 tid = bpf_get_current_pid_tgid(); + u64 time = bpf_ktime_get_ns(); + u32 vcpu_id; + if (!vcpu) { + return 0; + } + bpf_probe_read_kernel(&vcpu_id, sizeof(u32), &vcpu->vcpu_id); + struct time_value time_value; + __builtin_memset(&time_value, 0, sizeof(struct time_value)); + time_value.time = time; + time_value.vcpu_id = vcpu_id; + time_value.pcpu_id = cpu; + struct load_key curr_load_key; + __builtin_memset(&curr_load_key, 0, sizeof(struct load_key)); + curr_load_key.pid = pid; + curr_load_key.tid = tid; + bpf_map_update_elem(&load_time, &curr_load_key, &time_value, BPF_ANY); + return 0; +} +//记录VCPU调度的信息--出调度 +static int trace_vmx_vcpu_put() { + u32 pid = bpf_get_current_pid_tgid() >> 32; + u32 tid = bpf_get_current_pid_tgid(); + struct load_key load_key; + __builtin_memset(&load_key, 0, sizeof(struct load_key)); + load_key.pid = pid; + load_key.tid = tid; + struct time_value *t_value; + t_value = bpf_map_lookup_elem(&load_time, &load_key); + if (!t_value) { + return 0; + } + u64 duration = bpf_ktime_get_ns() - t_value->time; + bpf_map_delete_elem(&load_time, &load_key); + struct load_value *load_value; + load_value = bpf_map_lookup_elem(&load_map, &load_key); + if (load_value) { + load_value->count++; + load_value->total_time += duration; + if (load_value->max_time < duration) { + load_value->max_time = duration; + } + if (load_value->min_time > duration) { + load_value->min_time = duration; + } + load_value->pcpu_id = t_value->pcpu_id; + load_value->vcpu_id = t_value->vcpu_id; + } else { + struct load_value new_load_value = {.count = 1, + .max_time = duration, + .total_time = duration, + .min_time = duration, + .vcpu_id = t_value->vcpu_id, + .pcpu_id = t_value->pcpu_id}; + bpf_map_update_elem(&load_map, &load_key, &new_load_value, BPF_ANY); + } + return 0; } static int trace_mark_page_dirty_in_slot(struct kvm *kvm, const struct kvm_memory_slot *memslot, diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h index 4cc3db9c4..b9be73d32 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h @@ -93,6 +93,24 @@ struct exit_key { __u32 tid; }; +struct load_key { + __u32 pid; + __u32 tid; +}; +struct load_value { + __u64 max_time; + __u64 total_time; + __u64 min_time; + __u32 count; + __u32 vcpu_id; + __u32 pcpu_id; +}; +struct time_value { + __u32 pad; + __u64 time; + __u32 vcpu_id; + __u32 pcpu_id; +}; struct exit_value { __u64 max_time; __u64 total_time; @@ -159,11 +177,6 @@ struct common_event { // VCPU_WAKEUP 特有成员 } vcpu_wakeup_data; - struct { - __u32 vcpu_id; - // VCPU_LOAD 特有成员 - } vcpu_load_data; - struct { __u32 reason_number; __u64 duration_ns; diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c index a7867c8eb..09b72f223 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c @@ -67,10 +67,15 @@ SEC("tp/kvm/kvm_entry") int tp_entry(struct exit *ctx) { return trace_kvm_entry(); } -//记录VCPU调度的信息 +//记录VCPU调度的信息--进入 SEC("kprobe/vmx_vcpu_load") int BPF_KPROBE(kp_vmx_vcpu_load, struct kvm_vcpu *vcpu, int cpu) { - return trace_vmx_vcpu_load(vcpu, cpu, &rb, e); + return trace_vmx_vcpu_load(vcpu, cpu); +} +//记录VCPU调度的信息--退出 +SEC("kprobe/vmx_vcpu_put") +int BPF_KPROBE(kp_vmx_vcpu_put, struct kvm_vcpu *vcpu) { + return trace_vmx_vcpu_put(); } SEC("kprobe/mark_page_dirty_in_slot") int BPF_KPROBE(kp_mark_page_dirty_in_slot, struct kvm *kvm, diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 1a751baf8..5bf0afb14 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -466,9 +466,6 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { break; } case VCPU_LOAD: { - printf("%-20.6f %-15s %-6d/%-8d %-10d\n", timestamp_ms, - e->process.comm, e->process.pid, e->process.tid, - e->vcpu_load_data.vcpu_id); break; } case HALT_POLL: { @@ -667,6 +664,9 @@ static int print_event_head(struct env *env) { case EXIT: printf("Waiting vm_exit ... \n"); break; + case VCPU_LOAD: + printf("Waiting vm_vcpu_load ... \n"); + break; case HALT_POLL: printf("%-18s %-15s %-15s %-10s %-7s %-11s %-10s\n", "TIME(ms)", "COMM", "PID/TID", "TYPE", "VCPU_ID", "OLD(ns)", "NEW(ns)"); @@ -723,6 +723,8 @@ static void set_disable_load(struct kvm_watcher_bpf *skel) { env.execute_vcpu_wakeup ? true : false); bpf_program__set_autoload(skel->progs.kp_vmx_vcpu_load, env.execute_vcpu_load ? true : false); + bpf_program__set_autoload(skel->progs.kp_vmx_vcpu_put, + env.execute_vcpu_load ? true : false); bpf_program__set_autoload(skel->progs.fentry_kvm_vcpu_halt, env.execute_vcpu_wakeup ? true : false); bpf_program__set_autoload(skel->progs.tp_exit, @@ -875,6 +877,52 @@ int sort_by_key(struct kvm_watcher_bpf *skel, struct exit_key *keys, } return count; } +int print_vcpu_load_map(struct kvm_watcher_bpf *skel) { + int fd = bpf_map__fd(skel->maps.load_map); + int err; + struct load_key lookup_key = {}; + struct load_key next_key = {}; + struct load_value load_value = {}; + int first = 1; + while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { + if (first) { + first = 0; + printf("\nTIME:%s\n", getCurrentTimeFormatted()); + printf("%-12s %-12s %-12s %-12s %-12s %-12s %-12s %-12s\n", "pid", + "tid", "total_time", "max_time", "min_time", "counts", + "vcpuid", "pcpuid"); + printf( + "------------ ------------ ------------ ------------ " + "------------ " + "------------ " + "------------ " + "------------\n"); + } + err = bpf_map_lookup_elem(fd, &next_key, &load_value); + if (err < 0) { + fprintf(stderr, "failed to lookup vcpu_load_value: %d\n", err); + return -1; + } + printf("%-12d %-12d %-12.4f %-12.4f %-12.4f %-12u %-12d %-12d\n", + next_key.pid, next_key.tid, + NS_TO_MS_WITH_DECIMAL(load_value.total_time), + NS_TO_MS_WITH_DECIMAL(load_value.max_time), + NS_TO_MS_WITH_DECIMAL(load_value.min_time), load_value.count, + load_value.vcpu_id, load_value.pcpu_id); + lookup_key = next_key; + } + // clear the maps + memset(&lookup_key, 0, sizeof(struct load_key)); + while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { + err = bpf_map_delete_elem(fd, &next_key); + if (err < 0) { + fprintf(stderr, "failed to cleanup counters: %d\n", err); + return -1; + } + lookup_key = next_key; + } + return 0; +} int print_exit_map(struct kvm_watcher_bpf *skel) { int fd = bpf_map__fd(skel->maps.exit_map); int err; @@ -1016,6 +1064,10 @@ int main(int argc, char **argv) { if (env.execute_exit) { print_map_and_check_error(print_exit_map, skel, "exit", err); } + if (env.execute_vcpu_load) { + print_map_and_check_error(print_vcpu_load_map, skel, "vcpu_load", + err); + } /* Ctrl-C will cause -EINTR */ if (err == -EINTR) { err = 0; From f9f2ce71ff7e3727c36f161828065431ba6427fa Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Tue, 9 Apr 2024 10:43:00 +0000 Subject: [PATCH 16/47] fix symbolization Signed-off-by: LiuLingze --- .../src/bpf_wapper/eBPFStackCollector.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp index f69a509f1..6d2ce144a 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp @@ -132,7 +132,7 @@ StackCollector::operator std::string() sym.name = demangleCppSym(sym.name); } std::stringstream ss(""); - ss << "+0x" << std::hex << (addr - sym.start); + ss << "+0x" << std::hex << (sym.ip - sym.start); sym.name += ss.str(); g_symbol_parser.putin_symbol_cache(id.pid, addr, sym.name); } @@ -160,7 +160,11 @@ StackCollector::operator std::string() symbol sym; sym.reset(addr); if (g_symbol_parser.find_kernel_symbol(sym)) - ; + { + std::stringstream ss(""); + ss << "+0x" << std::hex << (sym.ip - sym.start); + sym.name += ss.str(); + } else { std::stringstream ss(""); @@ -224,4 +228,4 @@ StackCollector::operator std::string() oss << _BLUE "OK" _RE "\n"; return oss.str(); -} \ No newline at end of file +} From ac1604ab0197d5d8456c6b7bab04f7ad49b0ac04 Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Wed, 10 Apr 2024 09:38:52 +0000 Subject: [PATCH 17/47] update Makefile Signed-off-by: LiuLingze --- eBPF_Supermarket/Stack_Analyser/Makefile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/Makefile b/eBPF_Supermarket/Stack_Analyser/Makefile index 132bb1c85..37cbf813e 100644 --- a/eBPF_Supermarket/Stack_Analyser/Makefile +++ b/eBPF_Supermarket/Stack_Analyser/Makefile @@ -115,18 +115,16 @@ $(BPF_SKEL)/%.skel.h: $(OUTPUT)/%.bpf.o | $(OUTPUT) $(BPFTOOL) $(BPF_SKEL) $(call msg,GEN-SKEL,$@) $(Q)$(BPFTOOL) gen skeleton $< > $@ -$(patsubst %,include/bpf_wapper/%.h,$(BPF)): include/bpf_wapper/%.h: $(BPF_SKEL)/%.skel.h - -$(patsubst %,$(OUTPUT)/%.o,$(BPF)): $(OUTPUT)/%.o: src/bpf_wapper/%.cpp include/bpf_wapper/%.h $(OUTPUT)/eBPFStackCollector.o +$(patsubst %,$(OUTPUT)/%.o,$(BPF)): $(OUTPUT)/%.o: src/bpf_wapper/%.cpp include/bpf_wapper/%.h $(BPF_SKEL)/%.skel.h $(OUTPUT)/eBPFStackCollector.o $(call msg,CXX,$@) $(Q)$(CXX) $(CFLAGS) $(INCLUDES) -c $< -o $@ # Build depending library -$(patsubst %,$(OUTPUT)/%.o,$(BIN)): $(OUTPUT)/%.o: src/%.cpp $(patsubst %,include/bpf_wapper/%.h,$(BPF)) +$(patsubst %,$(OUTPUT)/%.o,$(BIN)): $(OUTPUT)/%.o: src/%.cpp $(patsubst %,$(BPF_SKEL)/%.skel.h,$(BPF)) $(call msg,CXX,$@) $(Q)$(CXX) $(CFLAGS) $(INCLUDES) -c $< -o $@ -$(OUTPUT)/eBPFStackCollector.o: src/bpf_wapper/eBPFStackCollector.cpp | $(LIBBPF_OBJ) +$(OUTPUT)/eBPFStackCollector.o: src/bpf_wapper/eBPFStackCollector.cpp include/bpf_wapper/eBPFStackCollector.h | $(LIBBPF_OBJ) $(call msg,CXX,$@) $(Q)$(CXX) $(CFLAGS) $(INCLUDES) -c $< -o $@ From 865e237f0f3e53f5d69a0caddb409fccfa3fc33e Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Wed, 10 Apr 2024 09:39:55 +0000 Subject: [PATCH 18/47] add filter Signed-off-by: LiuLingze --- eBPF_Supermarket/Stack_Analyser/bpf/io.bpf.c | 14 +++++++---- .../Stack_Analyser/bpf/llc_stat.bpf.c | 14 ++++++++--- .../Stack_Analyser/bpf/memleak.bpf.c | 22 ++++++++++-------- .../Stack_Analyser/bpf/off_cpu.bpf.c | 23 +++++++++++++++---- .../Stack_Analyser/bpf/on_cpu.bpf.c | 14 +++++++---- .../Stack_Analyser/bpf/probe.bpf.c | 15 ++++++++---- .../Stack_Analyser/bpf/readahead.bpf.c | 15 ++++++++---- 7 files changed, 82 insertions(+), 35 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/io.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/io.bpf.c index 65a9a60a7..4f490e794 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/io.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/io.bpf.c @@ -27,7 +27,6 @@ COMMON_MAPS(io_tuple); COMMON_VALS; -const volatile int target_pid = 0; const char LICENSE[] SEC("license") = "GPL"; @@ -35,12 +34,19 @@ static int do_stack(struct trace_event_raw_sys_enter *ctx) { CHECK_ACTIVE; struct task_struct *curr = (struct task_struct *)bpf_get_current_task(); // 利用bpf_get_current_task()获得当前的进程tsk - RET_IF_KERN(curr); + + if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) + return 0; u32 pid = BPF_CORE_READ(curr, pid); // 利用帮助函数获得当前进程的pid - if ((target_pid >= 0 && pid != target_pid) || !pid || pid == self_pid) + if ((!pid) || (pid == self_pid) || (target_pid > 0 && pid != target_pid)) + return 0; + if (target_tgid > 0 && BPF_CORE_READ(curr, tgid) != target_tgid) + return 0; + SET_KNODE(curr, knode); + if (target_cgroupid > 0 && BPF_CORE_READ(knode, id) != target_cgroupid) return 0; - SAVE_TASK_INFO(pid, curr); + SAVE_TASK_INFO(pid, curr, knode); // record time delta psid apsid = GET_COUNT_KEY(pid, ctx); diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/llc_stat.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/llc_stat.bpf.c index 89c444690..70121bc6f 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/llc_stat.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/llc_stat.bpf.c @@ -32,11 +32,19 @@ static __always_inline int trace_event(__u64 sample_period, bool miss, struct bp { CHECK_ACTIVE; struct task_struct *curr = (struct task_struct *)bpf_get_current_task(); - RET_IF_KERN(curr); + + if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) + return 0; u32 pid = BPF_CORE_READ(curr, pid); // 利用帮助函数获得当前进程的pid - if (!pid || pid == self_pid) + if ((!pid) || (pid == self_pid) || (target_pid > 0 && pid != target_pid)) + return 0; + if (target_tgid > 0 && BPF_CORE_READ(curr, tgid) != target_tgid) return 0; - SAVE_TASK_INFO(pid, curr); + SET_KNODE(curr, knode); + if (target_cgroupid > 0 && BPF_CORE_READ(knode, id) != target_cgroupid) + return 0; + + SAVE_TASK_INFO(pid, curr, knode); psid apsid = GET_COUNT_KEY(pid, ctx); llc_stat *infop = bpf_map_lookup_elem(&psid_count_map, &apsid); if (!infop) diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/memleak.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/memleak.bpf.c index 239a7edba..70a80e2e4 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/memleak.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/memleak.bpf.c @@ -41,21 +41,25 @@ const char LICENSE[] SEC("license") = "GPL"; static int gen_alloc_enter(size_t size) { CHECK_ACTIVE; - if (!size) + if (!size || (sample_rate > 1 && bpf_ktime_get_ns() % sample_rate != 0)) + return 0; + struct task_struct *curr = (struct task_struct *)bpf_get_current_task(); + + if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) return 0; - if (sample_rate > 1) { - if (bpf_ktime_get_ns() % sample_rate != 0) + u32 pid = BPF_CORE_READ(curr, pid); // 利用帮助函数获得当前进程的pid + if ((!pid) || (pid == self_pid) || (target_pid > 0 && pid != target_pid)) return 0; } - struct task_struct *curr = (struct task_struct *)bpf_get_current_task(); - RET_IF_KERN(curr); - // update group - // group share memory u32 tgid = BPF_CORE_READ(curr, tgid); - if (tgid == self_pid) + if (target_tgid > 0 && tgid != target_tgid) + return 0; + SET_KNODE(curr, knode); + if (target_cgroupid > 0 && BPF_CORE_READ(knode, id) != target_cgroupid) return 0; - SAVE_TASK_INFO(tgid, curr); + + SAVE_TASK_INFO(tgid, curr, knode); if (trace_all) bpf_printk("alloc entered, size = %lu\n", size); // record size diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/off_cpu.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/off_cpu.bpf.c index 10445e9db..b06a44572 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/off_cpu.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/off_cpu.bpf.c @@ -27,7 +27,6 @@ COMMON_MAPS(u32); COMMON_VALS; -const volatile int target_pid = 0; BPF_HASH(pid_offTs_map, u32, u64); // 记录进程运行的起始时间 const char LICENSE[] SEC("license") = "GPL"; @@ -36,9 +35,20 @@ SEC("kprobe/finish_task_switch") // 动态挂载点finish_task_switch.isra.0 int BPF_KPROBE(do_stack, struct task_struct *curr) { CHECK_ACTIVE; - u32 pid = BPF_CORE_READ(curr, pid); // 利用帮助函数获取换出进程tsk的pid - RET_IF_KERN(curr); - if ((target_pid >= 0 && pid == target_pid) || (target_pid < 0 && pid && pid != self_pid)) + bool record = true; + if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) + record = false; + u32 pid = BPF_CORE_READ(curr, pid); // 利用帮助函数获得当前进程的pid + if ((!pid) || (pid == self_pid) || (target_pid > 0 && pid != target_pid)) + record = false; + if (target_tgid > 0 && BPF_CORE_READ(curr, tgid) != target_tgid) + record = false; + { + SET_KNODE(curr, knode); + if (target_cgroupid > 0 && BPF_CORE_READ(knode, id) != target_cgroupid) + record = false; + } + if (record) { // record curr block time u64 ts = bpf_ktime_get_ns(); // ts=当前的时间戳（ns） @@ -57,7 +67,10 @@ int BPF_KPROBE(do_stack, struct task_struct *curr) return 0; // record data - SAVE_TASK_INFO(pid, next); + { + SET_KNODE(next, knode); + SAVE_TASK_INFO(pid, next, knode); + } psid apsid = GET_COUNT_KEY(pid, ctx); // record time delta diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/on_cpu.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/on_cpu.bpf.c index 7830850ad..642965b45 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/on_cpu.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/on_cpu.bpf.c @@ -34,11 +34,17 @@ int do_stack(void *ctx) { CHECK_ACTIVE; struct task_struct *curr = (void *)bpf_get_current_task(); // curr指向当前进程的tsk - RET_IF_KERN(curr); // 忽略内核线程 - u32 pid = BPF_CORE_READ(curr, pid); // pid保存当前进程的pid，是cgroup pid 对应的level 0 pid - if (!pid || pid == self_pid) + + if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) + return 0; + u32 pid = BPF_CORE_READ(curr, pid); // 利用帮助函数获得当前进程的pid + if ((!pid) || (pid == self_pid) || (target_pid > 0 && pid != target_pid)) return 0; - SAVE_TASK_INFO(pid, curr); + SET_KNODE(curr, knode); + if (target_cgroupid > 0 && BPF_CORE_READ(knode, id) != target_cgroupid) + return 0; + + SAVE_TASK_INFO(pid, curr, knode); psid apsid = GET_COUNT_KEY(pid, ctx); // add cosunt diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c index a5e29f6d0..c7c08e7cf 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c @@ -26,7 +26,6 @@ COMMON_MAPS(u32); COMMON_VALS; -const volatile int target_pid = 0; const char LICENSE[] SEC("license") = "GPL"; @@ -34,13 +33,19 @@ static int handle_func(void *ctx) { CHECK_ACTIVE; struct task_struct *curr = (struct task_struct *)bpf_get_current_task(); // 利用bpf_get_current_task()获得当前的进程tsk - RET_IF_KERN(curr); - u32 pid = get_task_ns_pid(curr); // 利用帮助函数获得当前进程的pid - if ((target_pid >= 0 && pid != target_pid) || !pid || pid == self_pid) + if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) + return 0; + u32 pid = BPF_CORE_READ(curr, pid); // 利用帮助函数获得当前进程的pid + if ((!pid) || (pid == self_pid) || (target_pid > 0 && pid != target_pid)) + return 0; + if (target_tgid > 0 && BPF_CORE_READ(curr, tgid) != target_tgid) + return 0; + SET_KNODE(curr, knode); + if (target_cgroupid > 0 && BPF_CORE_READ(knode, id) != target_cgroupid) return 0; - SAVE_TASK_INFO(pid, curr); + SAVE_TASK_INFO(pid, curr, knode); psid a_psid = GET_COUNT_KEY(pid, ctx); u32 *cnt = bpf_map_lookup_elem(&psid_count_map, &a_psid); diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/readahead.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/readahead.bpf.c index 59c390076..f5b34a869 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/readahead.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/readahead.bpf.c @@ -31,7 +31,6 @@ COMMON_MAPS(ra_tuple); COMMON_VALS; -int target_pid = 0; BPF_HASH(in_ra_map, u32, psid); BPF_HASH(page_psid_map, struct page *, psid); @@ -40,13 +39,19 @@ int BPF_PROG(page_cache_ra_unbounded) { CHECK_ACTIVE; struct task_struct *curr = (struct task_struct *)bpf_get_current_task(); - RET_IF_KERN(curr); - u32 pid = get_task_ns_pid(curr); // 获取当前进程tgid，用户空间的pid即是tgid - if ((target_pid >= 0 && pid != target_pid) || !pid || pid == self_pid) + if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) + return 0; + u32 pid = BPF_CORE_READ(curr, pid); // 利用帮助函数获得当前进程的pid + if ((!pid) || (pid == self_pid) || (target_pid > 0 && pid != target_pid)) + return 0; + if (target_tgid > 0 && BPF_CORE_READ(curr, tgid) != target_tgid) + return 0; + SET_KNODE(curr, knode); + if (target_cgroupid > 0 && BPF_CORE_READ(knode, id) != target_cgroupid) return 0; - SAVE_TASK_INFO(pid, curr); + SAVE_TASK_INFO(pid, curr, knode); psid apsid = GET_COUNT_KEY(pid, ctx); ra_tuple *d = bpf_map_lookup_elem(&psid_count_map, &apsid); // d指向psid_count表中的apsid对应的类型为tuple的值 From c8eed17e1caf934b2a0868f3d8c12159a03cbdc3 Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Wed, 10 Apr 2024 09:41:23 +0000 Subject: [PATCH 19/47] add filter Signed-off-by: LiuLingze --- .../include/bpf_wapper/eBPFStackCollector.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/eBPFStackCollector.h b/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/eBPFStackCollector.h index 08bf2cb04..ee91cb274 100644 --- a/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/eBPFStackCollector.h +++ b/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/eBPFStackCollector.h @@ -58,8 +58,11 @@ class StackCollector public: Scale *scales; - int pid = -1; // 用于设置ebpf程序跟踪的pid - int err = 0; // 用于保存错误代码 + short top = 10; + uint64_t cgroup = 0; + uint32_t tgid = 0; + uint32_t pid = 0; // 用于设置ebpf程序跟踪的pid + int err = 0; // 用于保存错误代码 bool ustack = false; // 是否跟踪用户栈 bool kstack = false; // 是否跟踪内核栈 @@ -114,6 +117,9 @@ class StackCollector skel->rodata->trace_user = ustack; \ skel->rodata->trace_kernel = kstack; \ skel->rodata->self_pid = self_pid; \ + skel->rodata->target_pid = pid; \ + skel->rodata->target_tgid = tgid; \ + skel->rodata->target_cgroupid = cgroup; \ err = skel->load(skel); \ CHECK_ERR(err, "Fail to load BPF skeleton"); \ obj = skel->obj; \ From 3b0c60e9738ec6591b905c0d883a6a212096ed8c Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Wed, 10 Apr 2024 09:43:58 +0000 Subject: [PATCH 20/47] split cgroup from thread info Signed-off-by: LiuLingze --- .../Stack_Analyser/include/sa_common.h | 1 - .../Stack_Analyser/include/sa_ebpf.h | 31 ++-- eBPF_Supermarket/Stack_Analyser/src/main.cpp | 148 +++++++++++++----- 3 files changed, 129 insertions(+), 51 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/include/sa_common.h b/eBPF_Supermarket/Stack_Analyser/include/sa_common.h index 0a181ca70..b29bc51b1 100644 --- a/eBPF_Supermarket/Stack_Analyser/include/sa_common.h +++ b/eBPF_Supermarket/Stack_Analyser/include/sa_common.h @@ -35,7 +35,6 @@ typedef struct { typedef struct { __u32 pid; __u32 tgid; - char cid[CONTAINER_ID_LEN]; char comm[COMM_LEN]; } task_info; diff --git a/eBPF_Supermarket/Stack_Analyser/include/sa_ebpf.h b/eBPF_Supermarket/Stack_Analyser/include/sa_ebpf.h index 433068353..f26314031 100644 --- a/eBPF_Supermarket/Stack_Analyser/include/sa_ebpf.h +++ b/eBPF_Supermarket/Stack_Analyser/include/sa_ebpf.h @@ -22,13 +22,6 @@ #include "sa_common.h" #define PF_KTHREAD 0x00200000 -#define RET_IF_KERN(task) \ - do \ - { \ - int flags = BPF_CORE_READ(task, flags); \ - if (flags & PF_KTHREAD) \ - return 0; \ - } while (false) /// @brief 创建一个指定名字的ebpf调用栈表 /// @param 新栈表的名字 @@ -65,25 +58,39 @@ #define COMMON_MAPS(count_type) \ BPF_HASH(psid_count_map, psid, count_type); \ BPF_STACK_TRACE(sid_trace_map); \ + BPF_HASH(tgid_cgroup_map, __u32, \ + char[CONTAINER_ID_LEN]); \ BPF_HASH(pid_info_map, u32, task_info); #define COMMON_VALS \ const volatile bool trace_user = false; \ const volatile bool trace_kernel = false; \ - const volatile int self_pid = 0; \ + const volatile __u64 target_cgroupid = 0; \ + const volatile __u32 target_tgid = 0; \ + const volatile __u32 target_pid = 0; \ + const volatile __u32 self_pid = 0; \ bool __active = false; -#define CHECK_ACTIVE if(!__active) return 0; +#define CHECK_ACTIVE \ + if (!__active) \ + return 0; -#define SAVE_TASK_INFO(_pid, _task) \ +#define SET_KNODE(_task, _knode) \ + struct kernfs_node *_knode = BPF_CORE_READ(_task, cgroups, dfl_cgrp, kn); + +#define SAVE_TASK_INFO(_pid, _task, _knode) \ if (!bpf_map_lookup_elem(&pid_info_map, &_pid)) \ { \ - task_info info; \ + task_info info = {0}; \ info.pid = get_task_ns_pid(_task); \ bpf_get_current_comm(info.comm, COMM_LEN); \ info.tgid = get_task_ns_tgid(_task); \ - fill_container_id(_task, info.cid); \ bpf_map_update_elem(&pid_info_map, &_pid, &info, BPF_NOEXIST); \ + \ + char cgroup_name[CONTAINER_ID_LEN] = {0}; \ + fill_container_id(_knode, cgroup_name); \ + bpf_map_update_elem(&tgid_cgroup_map, &(info.tgid), \ + &cgroup_name, BPF_NOEXIST); \ } #define GET_COUNT_KEY(_pid, _ctx) \ diff --git a/eBPF_Supermarket/Stack_Analyser/src/main.cpp b/eBPF_Supermarket/Stack_Analyser/src/main.cpp index 06f8e5d86..72a466bd4 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/main.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/main.cpp @@ -38,18 +38,82 @@ bool timeout = false; uint64_t IntTmp; std::string StrTmp; clipp::man_page *man_page; +std::vector StackCollectorList; namespace MainConfig { uint64_t run_time = -1; // 运行时间 unsigned delay = 5; // 设置输出间隔 std::string command = ""; - int32_t target_pid = -1; + uint32_t target_pid = 0; + uint32_t target_tgid = 0; + uint64_t target_cgroup = 0; std::string trigger = ""; // 触发器 std::string trig_event = ""; // 触发事件 + short top = 10; + bool trace_user = false; + bool trace_kernel = false; } -std::vector StackCollectorList; +namespace helper +{ +#include +#include + struct cgid_file_handle + { + // struct file_handle handle; + unsigned int handle_bytes; + int handle_type; + uint64_t cgid; + }; + uint64_t get_cgroupid(const char *pathname) + { + struct statfs fs; + int err; + struct cgid_file_handle *h; + int mount_id; + uint64_t ret; + + err = statfs(pathname, &fs); + if (err != 0) + { + fprintf(stderr, "statfs on %s failed: %s\n", pathname, strerror(errno)); + exit(1); + } + + if ((fs.f_type != (typeof(fs.f_type))CGROUP2_SUPER_MAGIC)) + { + fprintf(stderr, "File %s is not on a cgroup2 mount.\n", pathname); + exit(1); + } + + h = (cgid_file_handle *)malloc(sizeof(struct cgid_file_handle)); + if (!h) + { + fprintf(stderr, "Cannot allocate memory.\n"); + exit(1); + } + + h->handle_bytes = 8; + err = name_to_handle_at(AT_FDCWD, pathname, (struct file_handle *)h, &mount_id, 0); + if (err != 0) + { + fprintf(stderr, "name_to_handle_at failed: %s\n", strerror(errno)); + exit(1); + } + + if (h->handle_bytes != 8) + { + fprintf(stderr, "Unexpected handle size: %d. \n", h->handle_bytes); + exit(1); + } + + ret = h->cgid; + free(h); + + return ret; + } +} void end_handle(void) { @@ -75,15 +139,6 @@ int main(int argc, char *argv[]) man_page = new clipp::man_page(); clipp::group cli; { - auto TraceOption = (clipp::option("-u") - .call([] - { StackCollectorList.back()->ustack = true; }) % - "Sample user stacks", - clipp::option("-k") - .call([] - { StackCollectorList.back()->kstack = true; }) % - "Sample kernel stacks\n"); - auto OnCpuOption = (clipp::option("on_cpu") .call([] { StackCollectorList.push_back(new OnCPUStackCollector()); }) % @@ -93,14 +148,12 @@ int main(int argc, char *argv[]) .call([] { static_cast(StackCollectorList.back()) ->setScale(IntTmp); })) % - "Set sampling frequency; default is 49", - TraceOption); + "Set sampling frequency; default is 49"); auto OffCpuOption = clipp::option("off_cpu") - .call([] - { StackCollectorList.push_back(new OffCPUStackCollector()); }) % - COLLECTOR_INFO("off-cpu") & - (TraceOption); + .call([] + { StackCollectorList.push_back(new OffCPUStackCollector()); }) % + COLLECTOR_INFO("off-cpu"); auto MemleakOption = (clipp::option("memleak") .call([] @@ -116,31 +169,27 @@ int main(int argc, char *argv[]) .call([] { static_cast(StackCollectorList.back()) ->wa_missing_free = true; }) % - "Free when missing in kernel to alleviate misjudgments", - TraceOption); + "Free when missing in kernel to alleviate misjudgments"); auto IOOption = clipp::option("io") - .call([] - { StackCollectorList.push_back(new IOStackCollector()); }) % - COLLECTOR_INFO("io") & - (TraceOption); + .call([] + { StackCollectorList.push_back(new IOStackCollector()); }) % + COLLECTOR_INFO("io"); auto ReadaheadOption = clipp::option("readahead") - .call([] - { StackCollectorList.push_back(new ReadaheadStackCollector()); }) % - COLLECTOR_INFO("readahead") & - (TraceOption); + .call([] + { StackCollectorList.push_back(new ReadaheadStackCollector()); }) % + COLLECTOR_INFO("readahead"); auto ProbeOption = clipp::option("probe") .call([] { StackCollectorList.push_back(new ProbeStackCollector()); }) % COLLECTOR_INFO("probe") & (clipp::value("probe", StrTmp) - .call([] - { static_cast(StackCollectorList.back()) - ->setScale(StrTmp); }) % - "Set the probe string" & - TraceOption); + .call([] + { static_cast(StackCollectorList.back()) + ->setScale(StrTmp); }) % + "Set the probe string"); auto LlcStatOption = clipp::option("llc_stat").call([] { StackCollectorList.push_back(new LlcStatStackCollector()); }) % @@ -150,17 +199,27 @@ int main(int argc, char *argv[]) .call([] { static_cast(StackCollectorList.back()) ->setScale(IntTmp); })) % - "Set sampling period; default is 100", - TraceOption); + "Set sampling period; default is 100"); auto MainOption = _GREEN "Some overall options" _RE % (( - ((clipp::option("-p") & - clipp::value("pid", MainConfig::target_pid)) % + ((clipp::option("-G") & + clipp::value("cgroup path", StrTmp) + .call([] + { MainConfig::target_cgroup = helper::get_cgroupid(StrTmp.c_str()); printf("Trace cgroup %ld\n", MainConfig::target_cgroup); })) % + "Set the cgroup of the process to be tracked; default is -1, which keeps track of all cgroups") | + ((clipp::option("-P") & + clipp::value("pid", MainConfig::target_tgid)) % "Set the pid of the process to be tracked; default is -1, which keeps track of all processes") | - ((clipp::option("-c") & + ((clipp::option("-T") & + clipp::value("tid", MainConfig::target_pid)) % + "Set the tid of the thread to be tracked; default is -1, which keeps track of all threads") | + ((clipp::option("-C") & clipp::value("command", MainConfig::command)) % "Set the command to be run and sampled; defaults is none")), + (clipp::option("-O") & + clipp::value("top", MainConfig::top)) % + "Set the top number; default is 10", (clipp::option("-d") & clipp::value("interval", MainConfig::delay)) % "Set the output delay time (seconds); default is 5", @@ -169,6 +228,14 @@ int main(int argc, char *argv[]) .call([] { stop_time = time(NULL) + MainConfig::run_time; })) % "Set the total sampling time; default is __INT_MAX__", + (clipp::option("-u") + .call([] + { MainConfig::trace_user = true; }) % + "Sample user stacks", + clipp::option("-k") + .call([] + { MainConfig::trace_kernel = true; }) % + "Sample kernel stacks"), (clipp::option("-T") & ((clipp::required("cpu").set(MainConfig::trigger) | clipp::required("memory").set(MainConfig::trigger) | @@ -228,7 +295,7 @@ int main(int argc, char *argv[]) MainConfig::target_pid = fork(); switch (MainConfig::target_pid) { - case -1: + case (uint32_t)-1: { CHECK_ERR(true, "Command create failed."); } @@ -254,6 +321,11 @@ int main(int argc, char *argv[]) fprintf(stderr, _RED "Attach collecotor%d %s.\n" _RE, (int)(Item - StackCollectorList.begin()) + 1, (*Item)->getName()); (*Item)->pid = MainConfig::target_pid; + (*Item)->tgid = MainConfig::target_tgid; + (*Item)->cgroup = MainConfig::target_cgroup; + (*Item)->top = MainConfig::top; + (*Item)->kstack = MainConfig::trace_kernel; + (*Item)->ustack = MainConfig::trace_user; if ((*Item)->load() || (*Item)->attach()) goto err; Item++; From 8713cf24e7ac51a92a9b787273e6467aa0de1deb Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Wed, 10 Apr 2024 09:45:54 +0000 Subject: [PATCH 21/47] opt stringing Signed-off-by: LiuLingze --- .../src/bpf_wapper/eBPFStackCollector.cpp | 56 ++++++++++++------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp index 6d2ce144a..a30835e77 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp @@ -93,6 +93,11 @@ StackCollector::operator std::string() auto D = sortedCountList(); if (!D) return oss.str(); + if ((*D).size() > top) + { + auto end = (*D).end(); + (*D).assign(end - top, end); + } oss << _GREEN "pid\tusid\tksid"; for (int i = 0; i < scale_num; i++) oss << '\t' << scales[i].Type << "/" << scales[i].Period << scales[i].Unit; @@ -112,11 +117,11 @@ StackCollector::operator std::string() auto trace_fd = bpf_object__find_map_fd_by_name(obj, "sid_trace_map"); if (id.usid > 0 && traces.find(id.usid) == traces.end()) { - std::vector sym_trace; bpf_map_lookup_elem(trace_fd, &id.usid, trace); for (p = trace + MAX_STACKS - 1; !*p; p--) ; - for (; p >= trace; p--) + std::vector sym_trace(p - trace + 1); + for (int i = 0; p >= trace; p--) { uint64_t &addr = *p; symbol sym; @@ -127,10 +132,8 @@ StackCollector::operator std::string() else if (g_symbol_parser.get_symbol_info(id.pid, sym, file) && g_symbol_parser.find_elf_symbol(sym, file, id.pid, id.pid)) { if (sym.name[0] == '_' && sym.name[1] == 'Z') - // 代表是C++符号，则调用demangle解析 - { + // 代表是C++符号，则调用demangle解析 sym.name = demangleCppSym(sym.name); - } std::stringstream ss(""); ss << "+0x" << std::hex << (sym.ip - sym.start); sym.name += ss.str(); @@ -144,36 +147,35 @@ StackCollector::operator std::string() g_symbol_parser.putin_symbol_cache(id.pid, addr, sym.name); } clearSpace(sym.name); - sym_trace.push_back(sym.name); + sym_trace[i++] = sym.name; } traces[id.usid] = sym_trace; } if (id.ksid > 0 && traces.find(id.ksid) == traces.end()) { - std::vector sym_trace; bpf_map_lookup_elem(trace_fd, &id.ksid, trace); for (p = trace + MAX_STACKS - 1; !*p; p--) ; - for (; p >= trace; p--) + std::vector sym_trace(p - trace + 1); + for (int i = 0; p >= trace; p--) { uint64_t &addr = *p; symbol sym; sym.reset(addr); + std::stringstream ss(""); if (g_symbol_parser.find_kernel_symbol(sym)) - { - std::stringstream ss(""); + { ss << "+0x" << std::hex << (sym.ip - sym.start); sym.name += ss.str(); - } + } else { - std::stringstream ss(""); ss << "0x" << std::hex << addr; sym.name = ss.str(); g_symbol_parser.putin_symbol_cache(pid, addr, sym.name); } clearSpace(sym.name); - sym_trace.push_back(sym.name); + sym_trace[i++] = sym.name; } traces[id.ksid] = sym_trace; } @@ -195,16 +197,29 @@ StackCollector::operator std::string() oss << _BLUE "info:" _RE "\n"; { - auto info_fd = bpf_object__find_map_fd_by_name(obj, "pid_info_map"); - if (info_fd < 0) + std::map tgid_cgroup_map; + auto cgroups = new char[MAX_ENTRIES][CONTAINER_ID_LEN]; + uint32_t count = MAX_ENTRIES; { - return oss.str(); + auto tgids = new uint32_t[MAX_ENTRIES]; + auto cgroup_fd = bpf_object__find_map_fd_by_name(obj, "tgid_cgroup_map"); + if (cgroup_fd < 0) + return oss.str(); + uint32_t next_key; + err = bpf_map_lookup_batch(cgroup_fd, NULL, &next_key, tgids, cgroups, &count, NULL); + for (auto i = 0; i < count; i++) + tgid_cgroup_map[tgids[i]] = cgroups[i]; + delete[] tgids; } + auto keys = new uint32_t[MAX_ENTRIES]; auto vals = new task_info[MAX_ENTRIES]; - uint32_t count = MAX_ENTRIES; - uint32_t next_key; + count = MAX_ENTRIES; { + auto info_fd = bpf_object__find_map_fd_by_name(obj, "pid_info_map"); + if (info_fd < 0) + return oss.str(); + uint32_t next_key; int err; if (showDelta) err = bpf_map_lookup_and_delete_batch(info_fd, NULL, &next_key, @@ -215,15 +230,16 @@ StackCollector::operator std::string() if (err == EFAULT) return oss.str(); } - oss << _GREEN "pid\tNSpid\tcomm\ttgid\tcgroup" _RE "\n"; + oss << _GREEN "pid\tNSpid\tcomm\ttgid\tcgroup\t" _RE "\n"; for (uint32_t i = 0; i < count; i++) oss << keys[i] << '\t' << vals[i].pid << '\t' << vals[i].comm << '\t' << vals[i].tgid << '\t' - << vals[i].cid << '\n'; + << tgid_cgroup_map[vals[i].tgid] << '\n'; delete[] keys; delete[] vals; + delete[] cgroups; } oss << _BLUE "OK" _RE "\n"; From 5f347e617dcc5baf558b32db0bdf3360129cde07 Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Wed, 10 Apr 2024 09:46:35 +0000 Subject: [PATCH 22/47] fix filter err Signed-off-by: LiuLingze --- eBPF_Supermarket/Stack_Analyser/include/task.h | 5 ++--- .../Stack_Analyser/src/bpf_wapper/llc_stat.cpp | 7 ++++--- .../Stack_Analyser/src/bpf_wapper/memleak.cpp | 9 ++++----- .../Stack_Analyser/src/bpf_wapper/on_cpu.cpp | 6 +++--- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/include/task.h b/eBPF_Supermarket/Stack_Analyser/include/task.h index 79f532543..db2e9b388 100644 --- a/eBPF_Supermarket/Stack_Analyser/include/task.h +++ b/eBPF_Supermarket/Stack_Analyser/include/task.h @@ -71,13 +71,12 @@ statfunc u32 get_task_ns_ppid(struct task_struct *task) return get_task_pid_vnr(real_parent); } -static void fill_container_id(struct task_struct *task, char *container_id) +static void fill_container_id(struct kernfs_node *knode, char *container_id) { - struct kernfs_node *knode = BPF_CORE_READ(task, cgroups, subsys[0], cgroup, kn); if (BPF_CORE_READ(knode, parent) != NULL) { char *aus; - bpf_probe_read(&aus, sizeof(void *), &knode->name); + bpf_probe_read(&aus, sizeof(void *), &(knode->name)); bpf_probe_read_str(container_id, CONTAINER_ID_LEN, aus); } } diff --git a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/llc_stat.cpp b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/llc_stat.cpp index a6bdc3986..a95bd9362 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/llc_stat.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/llc_stat.cpp @@ -78,7 +78,7 @@ int LlcStatStackCollector::attach(void) } /* Set up performance monitoring on a CPU/Core */ attr.config = PERF_COUNT_HW_CACHE_MISSES; - int pefd = syscall(SYS_perf_event_open, &attr, pid, cpu, -1, 0); + int pefd = syscall(SYS_perf_event_open, &attr, pid ? pid : -1, cpu, -1, 0); CHECK_ERR(pefd < 0, "Fail to set up performance monitor on a CPU/Core"); mpefds[cpu] = pefd; /* Attach a BPF program on a CPU */ @@ -86,7 +86,7 @@ int LlcStatStackCollector::attach(void) CHECK_ERR(!mlinks[cpu], "Fail to attach bpf program"); attr.config = PERF_COUNT_HW_CACHE_REFERENCES; - pefd = syscall(SYS_perf_event_open, &attr, pid, cpu, -1, 0); + pefd = syscall(SYS_perf_event_open, &attr, pid ? pid : -1, cpu, -1, 0); CHECK_ERR(pefd < 0, "Fail to set up performance monitor on a CPU/Core"); rpefds[cpu] = pefd; /* Attach a BPF program on a CPU */ @@ -141,6 +141,7 @@ LlcStatStackCollector::LlcStatStackCollector() }; }; -void LlcStatStackCollector::setScale(uint64_t p) { +void LlcStatStackCollector::setScale(uint64_t p) +{ scales[0].Period = scales[1].Period = p; } \ No newline at end of file diff --git a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/memleak.cpp b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/memleak.cpp index e0edb6cda..21c72c588 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/memleak.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/memleak.cpp @@ -119,9 +119,7 @@ int MemleakStackCollector::load(void) disable_kernel_node_tracepoints(skel); if (!percpu) disable_kernel_percpu_tracepoints(skel); - } else { - disable_kernel_tracepoints(skel); - }; + } else disable_kernel_tracepoints(skel); skel->rodata->sample_rate = sample_rate; skel->rodata->wa_missing_free = wa_missing_free; skel->rodata->page_size = sysconf(_SC_PAGE_SIZE);); @@ -152,6 +150,7 @@ void MemleakStackCollector::activate(bool tf) ACTIVE_SET(tf); } -const char *MemleakStackCollector::getName(void) { +const char *MemleakStackCollector::getName(void) +{ return "MemleakStackCollector"; -} \ No newline at end of file +} diff --git a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/on_cpu.cpp b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/on_cpu.cpp index 8e4e1cf4a..5fa58a537 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/on_cpu.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/on_cpu.cpp @@ -62,7 +62,6 @@ uint64_t *OnCPUStackCollector::count_values(void *data) int OnCPUStackCollector::load(void) { EBPF_LOAD_OPEN_INIT(); - return 0; }; @@ -99,7 +98,7 @@ int OnCPUStackCollector::attach(void) continue; } /* Set up performance monitoring on a CPU/Core */ - int pefd = perf_event_open(&attr, pid, cpu, -1, 0); + int pefd = perf_event_open(&attr, pid ? pid : -1, cpu, -1, 0); CHECK_ERR(pefd < 0, "Fail to set up performance monitor on a CPU/Core"); pefds[cpu] = pefd; /* Attach a BPF program on a CPU */ @@ -144,6 +143,7 @@ void OnCPUStackCollector::activate(bool tf) ACTIVE_SET(tf); } -const char *OnCPUStackCollector::getName(void) { +const char *OnCPUStackCollector::getName(void) +{ return "OnCPUStackCollector"; } \ No newline at end of file From 42700064423ac2ccd20d7071c56ab2e59b3b1f45 Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Thu, 11 Apr 2024 12:42:09 +0000 Subject: [PATCH 23/47] opt stringing Signed-off-by: LiuLingze --- .../src/bpf_wapper/eBPFStackCollector.cpp | 59 ++++++------------- 1 file changed, 18 insertions(+), 41 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp index a30835e77..7fa24cc1c 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp @@ -87,6 +87,7 @@ StackCollector::operator std::string() std::ostringstream oss; oss << _RED "time:" << getLocalDateTime() << _RE "\n"; std::map> traces; + std::map infos; oss << _BLUE "counts:" _RE "\n"; { @@ -96,7 +97,10 @@ StackCollector::operator std::string() if ((*D).size() > top) { auto end = (*D).end(); - (*D).assign(end - top, end); + auto begin = end - top; + for (auto i = (*D).begin(); i < begin; i++) + delete i->v; + (*D).assign(begin, end); } oss << _GREEN "pid\tusid\tksid"; for (int i = 0; i < scale_num; i++) @@ -179,6 +183,10 @@ StackCollector::operator std::string() } traces[id.ksid] = sym_trace; } + auto info_fd = bpf_object__find_map_fd_by_name(obj, "pid_info_map"); + task_info info; + bpf_map_lookup_elem(info_fd, &id.pid, &info); + infos[id.pid] = info; } delete D; } @@ -197,49 +205,18 @@ StackCollector::operator std::string() oss << _BLUE "info:" _RE "\n"; { - std::map tgid_cgroup_map; - auto cgroups = new char[MAX_ENTRIES][CONTAINER_ID_LEN]; - uint32_t count = MAX_ENTRIES; + oss << _GREEN "pid\tNSpid\tcomm\ttgid\tcgroup\t" _RE "\n"; + for (auto i : infos) { - auto tgids = new uint32_t[MAX_ENTRIES]; auto cgroup_fd = bpf_object__find_map_fd_by_name(obj, "tgid_cgroup_map"); - if (cgroup_fd < 0) - return oss.str(); - uint32_t next_key; - err = bpf_map_lookup_batch(cgroup_fd, NULL, &next_key, tgids, cgroups, &count, NULL); - for (auto i = 0; i < count; i++) - tgid_cgroup_map[tgids[i]] = cgroups[i]; - delete[] tgids; - } - - auto keys = new uint32_t[MAX_ENTRIES]; - auto vals = new task_info[MAX_ENTRIES]; - count = MAX_ENTRIES; - { - auto info_fd = bpf_object__find_map_fd_by_name(obj, "pid_info_map"); - if (info_fd < 0) - return oss.str(); - uint32_t next_key; - int err; - if (showDelta) - err = bpf_map_lookup_and_delete_batch(info_fd, NULL, &next_key, - keys, vals, &count, NULL); - else - err = bpf_map_lookup_batch(info_fd, NULL, &next_key, - keys, vals, &count, NULL); - if (err == EFAULT) - return oss.str(); + char group[CONTAINER_ID_LEN]; + bpf_map_lookup_elem(cgroup_fd, &(i.second.tgid), &group); + oss << i.first << '\t' + << i.second.pid << '\t' + << i.second.comm << '\t' + << i.second.tgid << '\t' + << group << '\n'; } - oss << _GREEN "pid\tNSpid\tcomm\ttgid\tcgroup\t" _RE "\n"; - for (uint32_t i = 0; i < count; i++) - oss << keys[i] << '\t' - << vals[i].pid << '\t' - << vals[i].comm << '\t' - << vals[i].tgid << '\t' - << tgid_cgroup_map[vals[i].tgid] << '\n'; - delete[] keys; - delete[] vals; - delete[] cgroups; } oss << _BLUE "OK" _RE "\n"; From 7025a34f3602f43f820b070136a762e9d5431a7f Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Thu, 11 Apr 2024 14:06:53 +0000 Subject: [PATCH 24/47] add global freq option Signed-off-by: LiuLingze --- eBPF_Supermarket/Stack_Analyser/bpf/io.bpf.c | 1 + .../Stack_Analyser/bpf/llc_stat.bpf.c | 1 + .../Stack_Analyser/bpf/memleak.bpf.c | 4 +--- .../Stack_Analyser/bpf/off_cpu.bpf.c | 1 + .../Stack_Analyser/bpf/probe.bpf.c | 1 + .../Stack_Analyser/bpf/readahead.bpf.c | 1 + .../include/bpf_wapper/eBPFStackCollector.h | 4 +++- .../include/bpf_wapper/on_cpu.h | 1 - .../Stack_Analyser/include/sa_ebpf.h | 21 +++++++++++++++++-- .../Stack_Analyser/src/bpf_wapper/memleak.cpp | 1 - eBPF_Supermarket/Stack_Analyser/src/main.cpp | 15 +++++++------ 11 files changed, 35 insertions(+), 16 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/io.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/io.bpf.c index 4f490e794..328c9c448 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/io.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/io.bpf.c @@ -33,6 +33,7 @@ const char LICENSE[] SEC("license") = "GPL"; static int do_stack(struct trace_event_raw_sys_enter *ctx) { CHECK_ACTIVE; + CHECK_FREQ; struct task_struct *curr = (struct task_struct *)bpf_get_current_task(); // 利用bpf_get_current_task()获得当前的进程tsk if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/llc_stat.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/llc_stat.bpf.c index 70121bc6f..d14f8922f 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/llc_stat.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/llc_stat.bpf.c @@ -31,6 +31,7 @@ COMMON_VALS; static __always_inline int trace_event(__u64 sample_period, bool miss, struct bpf_perf_event_data *ctx) { CHECK_ACTIVE; + CHECK_FREQ; struct task_struct *curr = (struct task_struct *)bpf_get_current_task(); if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/memleak.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/memleak.bpf.c index 70a80e2e4..edf090a41 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/memleak.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/memleak.bpf.c @@ -27,7 +27,6 @@ COMMON_MAPS(union combined_alloc_info); COMMON_VALS; -const volatile __u64 sample_rate = 1; const volatile bool wa_missing_free = false; const volatile size_t page_size = 4096; const volatile bool trace_all = false; @@ -41,8 +40,7 @@ const char LICENSE[] SEC("license") = "GPL"; static int gen_alloc_enter(size_t size) { CHECK_ACTIVE; - if (!size || (sample_rate > 1 && bpf_ktime_get_ns() % sample_rate != 0)) - return 0; + CHECK_FREQ; struct task_struct *curr = (struct task_struct *)bpf_get_current_task(); if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/off_cpu.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/off_cpu.bpf.c index b06a44572..1387d0712 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/off_cpu.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/off_cpu.bpf.c @@ -35,6 +35,7 @@ SEC("kprobe/finish_task_switch") // 动态挂载点finish_task_switch.isra.0 int BPF_KPROBE(do_stack, struct task_struct *curr) { CHECK_ACTIVE; + CHECK_FREQ; bool record = true; if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) record = false; diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c index c7c08e7cf..f608f4fc9 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c @@ -32,6 +32,7 @@ const char LICENSE[] SEC("license") = "GPL"; static int handle_func(void *ctx) { CHECK_ACTIVE; + CHECK_FREQ; struct task_struct *curr = (struct task_struct *)bpf_get_current_task(); // 利用bpf_get_current_task()获得当前的进程tsk if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/readahead.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/readahead.bpf.c index f5b34a869..c9fc3c3c3 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/readahead.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/readahead.bpf.c @@ -38,6 +38,7 @@ SEC("fentry/page_cache_ra_unbounded") // fentry在内核函数page_cache_ra_unbo int BPF_PROG(page_cache_ra_unbounded) { CHECK_ACTIVE; + CHECK_FREQ; struct task_struct *curr = (struct task_struct *)bpf_get_current_task(); if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) diff --git a/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/eBPFStackCollector.h b/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/eBPFStackCollector.h index ee91cb274..01d989367 100644 --- a/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/eBPFStackCollector.h +++ b/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/eBPFStackCollector.h @@ -58,7 +58,8 @@ class StackCollector public: Scale *scales; - short top = 10; + uint32_t top = 10; + uint32_t freq = 49; uint64_t cgroup = 0; uint32_t tgid = 0; uint32_t pid = 0; // 用于设置ebpf程序跟踪的pid @@ -120,6 +121,7 @@ class StackCollector skel->rodata->target_pid = pid; \ skel->rodata->target_tgid = tgid; \ skel->rodata->target_cgroupid = cgroup; \ + skel->rodata->freq = freq; \ err = skel->load(skel); \ CHECK_ERR(err, "Fail to load BPF skeleton"); \ obj = skel->obj; \ diff --git a/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/on_cpu.h b/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/on_cpu.h index cd79dfccc..34f66be85 100644 --- a/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/on_cpu.h +++ b/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/on_cpu.h @@ -31,7 +31,6 @@ class OnCPUStackCollector : public StackCollector int *pefds = NULL; int num_cpus = 0; struct bpf_link **links = NULL; - unsigned long long freq = 49; protected: virtual uint64_t *count_values(void *); diff --git a/eBPF_Supermarket/Stack_Analyser/include/sa_ebpf.h b/eBPF_Supermarket/Stack_Analyser/include/sa_ebpf.h index f26314031..e3f409eed 100644 --- a/eBPF_Supermarket/Stack_Analyser/include/sa_ebpf.h +++ b/eBPF_Supermarket/Stack_Analyser/include/sa_ebpf.h @@ -69,12 +69,29 @@ const volatile __u32 target_tgid = 0; \ const volatile __u32 target_pid = 0; \ const volatile __u32 self_pid = 0; \ - bool __active = false; + const volatile __u32 freq = 0; \ + bool __active = false; \ + __u32 __last_n = 0; \ + __u32 __next_n = 0; \ + bool __recorded = false; #define CHECK_ACTIVE \ if (!__active) \ return 0; +#define CHECK_FREQ \ + if (freq) \ + { \ + __next_n = ((bpf_ktime_get_ns() & ((1ul << 30) - 1)) * freq) >> 30; \ + if ((__last_n == __next_n) && __recorded) \ + return 0; \ + else \ + { \ + __recorded == true; \ + __last_n = __next_n; \ + } \ + } + #define SET_KNODE(_task, _knode) \ struct kernfs_node *_knode = BPF_CORE_READ(_task, cgroups, dfl_cgrp, kn); @@ -84,7 +101,7 @@ task_info info = {0}; \ info.pid = get_task_ns_pid(_task); \ bpf_get_current_comm(info.comm, COMM_LEN); \ - info.tgid = get_task_ns_tgid(_task); \ + info.tgid = BPF_CORE_READ(_task, tgid); \ bpf_map_update_elem(&pid_info_map, &_pid, &info, BPF_NOEXIST); \ \ char cgroup_name[CONTAINER_ID_LEN] = {0}; \ diff --git a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/memleak.cpp b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/memleak.cpp index 21c72c588..b7603036f 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/memleak.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/memleak.cpp @@ -120,7 +120,6 @@ int MemleakStackCollector::load(void) if (!percpu) disable_kernel_percpu_tracepoints(skel); } else disable_kernel_tracepoints(skel); - skel->rodata->sample_rate = sample_rate; skel->rodata->wa_missing_free = wa_missing_free; skel->rodata->page_size = sysconf(_SC_PAGE_SIZE);); return 0; diff --git a/eBPF_Supermarket/Stack_Analyser/src/main.cpp b/eBPF_Supermarket/Stack_Analyser/src/main.cpp index 72a466bd4..343e3ce75 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/main.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/main.cpp @@ -50,7 +50,8 @@ namespace MainConfig uint64_t target_cgroup = 0; std::string trigger = ""; // 触发器 std::string trig_event = ""; // 触发事件 - short top = 10; + uint32_t top = 10; + uint32_t freq = 49; bool trace_user = false; bool trace_kernel = false; } @@ -142,13 +143,7 @@ int main(int argc, char *argv[]) auto OnCpuOption = (clipp::option("on_cpu") .call([] { StackCollectorList.push_back(new OnCPUStackCollector()); }) % - COLLECTOR_INFO("on-cpu")) & - ((clipp::option("-f") & - clipp::value("freq", IntTmp) - .call([] - { static_cast(StackCollectorList.back()) - ->setScale(IntTmp); })) % - "Set sampling frequency; default is 49"); + COLLECTOR_INFO("on-cpu")); auto OffCpuOption = clipp::option("off_cpu") .call([] @@ -220,6 +215,9 @@ int main(int argc, char *argv[]) (clipp::option("-O") & clipp::value("top", MainConfig::top)) % "Set the top number; default is 10", + (clipp::option("-f") & + clipp::value("freq", MainConfig::freq)) % + "Set sampling frequency, 0 for close; default is 49", (clipp::option("-d") & clipp::value("interval", MainConfig::delay)) % "Set the output delay time (seconds); default is 5", @@ -324,6 +322,7 @@ int main(int argc, char *argv[]) (*Item)->tgid = MainConfig::target_tgid; (*Item)->cgroup = MainConfig::target_cgroup; (*Item)->top = MainConfig::top; + (*Item)->freq = MainConfig::freq; (*Item)->kstack = MainConfig::trace_kernel; (*Item)->ustack = MainConfig::trace_user; if ((*Item)->load() || (*Item)->attach()) From 917c2fcd98a1a956e3d5ecac2d31492723464ad0 Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Thu, 18 Apr 2024 05:20:07 +0000 Subject: [PATCH 25/47] add service name Signed-off-by: LiuLingze --- eBPF_Supermarket/Stack_Analyser/exporter/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/eBPF_Supermarket/Stack_Analyser/exporter/main.go b/eBPF_Supermarket/Stack_Analyser/exporter/main.go index 66a074f54..f81dbb098 100644 --- a/eBPF_Supermarket/Stack_Analyser/exporter/main.go +++ b/eBPF_Supermarket/Stack_Analyser/exporter/main.go @@ -292,6 +292,7 @@ func CollectProfiles(cb CollectProfilesCallback) error { for i, s := range scales { target := sd.NewTarget("", k.pid, sd.DiscoveryTarget{ "__container_id__": info[k.pid].cid, + "service_name": "Stack_Analyzer", labels.MetricName: s.Type, }) cb(target, group_trace, v[i], s, true) From 2ed56a11adb2522db769e0273f8f6a8a28544817 Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Thu, 18 Apr 2024 05:20:58 +0000 Subject: [PATCH 26/47] remove unused symbol Signed-off-by: LiuLingze --- eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/memleak.h | 1 - 1 file changed, 1 deletion(-) diff --git a/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/memleak.h b/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/memleak.h index 08afd79fb..544b3b7c6 100644 --- a/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/memleak.h +++ b/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/memleak.h @@ -60,7 +60,6 @@ class MemleakStackCollector : public StackCollector public: char *object = (char *)"libc.so.6"; bool percpu = false; - __u64 sample_rate = 1; bool wa_missing_free = false; protected: From 95b2dba40d9cfcf82920dbd53f4f45f1e322953b Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Thu, 18 Apr 2024 20:48:28 +0000 Subject: [PATCH 27/47] fix filter logic Signed-off-by: LiuLingze --- .../Stack_Analyser/bpf/off_cpu.bpf.c | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/off_cpu.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/off_cpu.bpf.c index 1387d0712..b79e894fb 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/off_cpu.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/off_cpu.bpf.c @@ -35,30 +35,42 @@ SEC("kprobe/finish_task_switch") // 动态挂载点finish_task_switch.isra.0 int BPF_KPROBE(do_stack, struct task_struct *curr) { CHECK_ACTIVE; - CHECK_FREQ; - bool record = true; - if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) - record = false; - u32 pid = BPF_CORE_READ(curr, pid); // 利用帮助函数获得当前进程的pid - if ((!pid) || (pid == self_pid) || (target_pid > 0 && pid != target_pid)) - record = false; - if (target_tgid > 0 && BPF_CORE_READ(curr, tgid) != target_tgid) - record = false; - { - SET_KNODE(curr, knode); - if (target_cgroupid > 0 && BPF_CORE_READ(knode, id) != target_cgroupid) - record = false; - } - if (record) + do { + if (freq) + { + __next_n = ((bpf_ktime_get_ns() & ((1ul << 30) - 1)) * freq) >> 30; + if (__last_n == __next_n) + { + if (__recorded) + break; + } + else + __last_n = __next_n; + } + + if (BPF_CORE_READ(curr, flags) & PF_KTHREAD) + break; + u32 pid = BPF_CORE_READ(curr, pid); // 利用帮助函数获得当前进程的pid + if ((!pid) || (pid == self_pid) || (target_pid > 0 && pid != target_pid)) + break; + if (target_tgid > 0 && BPF_CORE_READ(curr, tgid) != target_tgid) + break; + { + SET_KNODE(curr, knode); + if (target_cgroupid > 0 && BPF_CORE_READ(knode, id) != target_cgroupid) + break; + } + // record curr block time u64 ts = bpf_ktime_get_ns(); // ts=当前的时间戳（ns） bpf_map_update_elem(&pid_offTs_map, &pid, &ts, BPF_NOEXIST); // 如果start表中不存在pid对应的时间，则就创建pid-->ts - } + __recorded = true; + } while (false); // calculate time delta, next ready to run struct task_struct *next = (struct task_struct *)bpf_get_current_task(); // next指向换入进程结构体 - pid = BPF_CORE_READ(next, pid); // 利用帮助函数获取next指向的tsk的pid + u32 pid = BPF_CORE_READ(next, pid); // 利用帮助函数获取next指向的tsk的pid u64 *tsp = bpf_map_lookup_elem(&pid_offTs_map, &pid); // tsp指向start表中的pid的值 if (!tsp) return 0; From 1e76576c49c018af4ee5f985d6a072f115da2db9 Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Thu, 18 Apr 2024 21:05:12 +0000 Subject: [PATCH 28/47] fix some err Signed-off-by: LiuLingze --- .../Stack_Analyser/include/sa_ebpf.h | 16 ++++------ eBPF_Supermarket/Stack_Analyser/new_bpf.sh | 2 +- .../src/bpf_wapper/eBPFStackCollector.cpp | 2 +- .../src/bpf_wapper/llc_stat.cpp | 4 +-- .../Stack_Analyser/src/bpf_wapper/on_cpu.cpp | 2 +- .../Stack_Analyser/src/dt_symbol.cpp | 7 +++- eBPF_Supermarket/Stack_Analyser/src/main.cpp | 32 ++++++++----------- 7 files changed, 31 insertions(+), 34 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/include/sa_ebpf.h b/eBPF_Supermarket/Stack_Analyser/include/sa_ebpf.h index e3f409eed..925844796 100644 --- a/eBPF_Supermarket/Stack_Analyser/include/sa_ebpf.h +++ b/eBPF_Supermarket/Stack_Analyser/include/sa_ebpf.h @@ -86,10 +86,7 @@ if ((__last_n == __next_n) && __recorded) \ return 0; \ else \ - { \ - __recorded == true; \ __last_n = __next_n; \ - } \ } #define SET_KNODE(_task, _knode) \ @@ -110,11 +107,12 @@ &cgroup_name, BPF_NOEXIST); \ } -#define GET_COUNT_KEY(_pid, _ctx) \ - ((psid){ \ - .pid = _pid, \ - .usid = trace_user ? bpf_get_stackid(_ctx, &sid_trace_map, BPF_F_FAST_STACK_CMP | BPF_F_USER_STACK) : -1, \ - .ksid = trace_kernel ? bpf_get_stackid(_ctx, &sid_trace_map, BPF_F_FAST_STACK_CMP) : -1, \ - }) +#define GET_COUNT_KEY(_pid, _ctx) \ + (__recorded == true, \ + (psid){ \ + .pid = _pid, \ + .usid = trace_user ? bpf_get_stackid(_ctx, &sid_trace_map, BPF_F_FAST_STACK_CMP | BPF_F_USER_STACK) : -1, \ + .ksid = trace_kernel ? bpf_get_stackid(_ctx, &sid_trace_map, BPF_F_FAST_STACK_CMP) : -1, \ + }) #endif \ No newline at end of file diff --git a/eBPF_Supermarket/Stack_Analyser/new_bpf.sh b/eBPF_Supermarket/Stack_Analyser/new_bpf.sh index 8ed45cdb9..8aa1507a2 100755 --- a/eBPF_Supermarket/Stack_Analyser/new_bpf.sh +++ b/eBPF_Supermarket/Stack_Analyser/new_bpf.sh @@ -27,6 +27,6 @@ sed -i 's/template/'$origin_name'/g' bpf/$origin_name.bpf.c sed -i '/#include "bpf_wapper\/on_cpu.h"/a#include "bpf_wapper\/'$origin_name'.h"' src/main.cpp -sed -i '/auto MainOption = _GREEN "Some overall options" _RE %/iauto '$name'Option = clipp::option("'$origin_name'").call([]{ StackCollectorList.push_back(new '$class_name'()); }) % COLLECTOR_INFO("'$origin_name'") & (TraceOption);' src/main.cpp +sed -i '/auto MainOption = _GREEN "Some overall options" _RE %/iauto '$name'Option = clipp::option("'$origin_name'").call([]{ StackCollectorList.push_back(new '$class_name'()); }) % COLLECTOR_INFO("'$origin_name'");' src/main.cpp sed -i '/MainOption,/i'$name'Option,' src/main.cpp \ No newline at end of file diff --git a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp index 7fa24cc1c..d4f95de50 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/eBPFStackCollector.cpp @@ -146,7 +146,7 @@ StackCollector::operator std::string() else { std::stringstream ss(""); - ss << "0x" << std::hex << addr; + ss << "0x" << std::hex << sym.ip; sym.name = ss.str(); g_symbol_parser.putin_symbol_cache(id.pid, addr, sym.name); } diff --git a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/llc_stat.cpp b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/llc_stat.cpp index a95bd9362..ecedcc450 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/llc_stat.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/llc_stat.cpp @@ -78,7 +78,7 @@ int LlcStatStackCollector::attach(void) } /* Set up performance monitoring on a CPU/Core */ attr.config = PERF_COUNT_HW_CACHE_MISSES; - int pefd = syscall(SYS_perf_event_open, &attr, pid ? pid : -1, cpu, -1, 0); + int pefd = syscall(SYS_perf_event_open, &attr, tgid ? tgid : -1, cpu, -1, 0); CHECK_ERR(pefd < 0, "Fail to set up performance monitor on a CPU/Core"); mpefds[cpu] = pefd; /* Attach a BPF program on a CPU */ @@ -86,7 +86,7 @@ int LlcStatStackCollector::attach(void) CHECK_ERR(!mlinks[cpu], "Fail to attach bpf program"); attr.config = PERF_COUNT_HW_CACHE_REFERENCES; - pefd = syscall(SYS_perf_event_open, &attr, pid ? pid : -1, cpu, -1, 0); + pefd = syscall(SYS_perf_event_open, &attr, tgid ? tgid : -1, cpu, -1, 0); CHECK_ERR(pefd < 0, "Fail to set up performance monitor on a CPU/Core"); rpefds[cpu] = pefd; /* Attach a BPF program on a CPU */ diff --git a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/on_cpu.cpp b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/on_cpu.cpp index 5fa58a537..287bec493 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/on_cpu.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/on_cpu.cpp @@ -98,7 +98,7 @@ int OnCPUStackCollector::attach(void) continue; } /* Set up performance monitoring on a CPU/Core */ - int pefd = perf_event_open(&attr, pid ? pid : -1, cpu, -1, 0); + int pefd = perf_event_open(&attr, tgid ? tgid : -1, cpu, -1, 0); CHECK_ERR(pefd < 0, "Fail to set up performance monitor on a CPU/Core"); pefds[cpu] = pefd; /* Attach a BPF program on a CPU */ diff --git a/eBPF_Supermarket/Stack_Analyser/src/dt_symbol.cpp b/eBPF_Supermarket/Stack_Analyser/src/dt_symbol.cpp index 92feb0d7b..d3c848fbc 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/dt_symbol.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/dt_symbol.cpp @@ -49,7 +49,7 @@ bool symbol_parser::load_pid_maps(int pid) } proc_vma proc; - char fn[256]; + char fn[23]; sprintf(fn, "/proc/%d/maps", pid); FILE *fp = fopen(fn, "r"); if (!fp) { @@ -335,6 +335,11 @@ bool symbol_parser::putin_symbol_cache(int tgid, unsigned long addr, std::string return false; } +/// @brief 找到指定进程中的虚拟地址对应的在可执行文件中相对于文件开始的偏移，赋给sym.ip +/// @param pid 指定进程的pid +/// @param sym 提供进程虚拟地址 +/// @param file 文件信息 +/// @return 成功找到返回true bool symbol_parser::get_symbol_info(int pid, symbol &sym, elf_file &file) { std::map::iterator proc_vma_info; diff --git a/eBPF_Supermarket/Stack_Analyser/src/main.cpp b/eBPF_Supermarket/Stack_Analyser/src/main.cpp index 343e3ce75..178722504 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/main.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/main.cpp @@ -154,17 +154,11 @@ int main(int argc, char *argv[]) .call([] { StackCollectorList.push_back(new MemleakStackCollector()); }) % COLLECTOR_INFO("memleak")) & - ((clipp::option("-i") & - clipp::value("interval", IntTmp) - .call([] - { static_cast(StackCollectorList.back()) - ->sample_rate = IntTmp; })) % - "Set the sampling interval; default is 1", - clipp::option("-w") - .call([] - { static_cast(StackCollectorList.back()) - ->wa_missing_free = true; }) % - "Free when missing in kernel to alleviate misjudgments"); + (clipp::option("-W") + .call([] + { static_cast(StackCollectorList.back()) + ->wa_missing_free = true; }) % + "Free when missing in kernel to alleviate misjudgments"); auto IOOption = clipp::option("io") .call([] @@ -189,7 +183,7 @@ int main(int argc, char *argv[]) auto LlcStatOption = clipp::option("llc_stat").call([] { StackCollectorList.push_back(new LlcStatStackCollector()); }) % COLLECTOR_INFO("llc_stat") & - ((clipp::option("-i") & + ((clipp::option("-P") & clipp::value("period", IntTmp) .call([] { static_cast(StackCollectorList.back()) @@ -198,30 +192,30 @@ int main(int argc, char *argv[]) auto MainOption = _GREEN "Some overall options" _RE % (( - ((clipp::option("-G") & + ((clipp::option("-g") & clipp::value("cgroup path", StrTmp) .call([] { MainConfig::target_cgroup = helper::get_cgroupid(StrTmp.c_str()); printf("Trace cgroup %ld\n", MainConfig::target_cgroup); })) % "Set the cgroup of the process to be tracked; default is -1, which keeps track of all cgroups") | - ((clipp::option("-P") & + ((clipp::option("-p") & clipp::value("pid", MainConfig::target_tgid)) % "Set the pid of the process to be tracked; default is -1, which keeps track of all processes") | - ((clipp::option("-T") & + ((clipp::option("-t") & clipp::value("tid", MainConfig::target_pid)) % "Set the tid of the thread to be tracked; default is -1, which keeps track of all threads") | - ((clipp::option("-C") & + ((clipp::option("-c") & clipp::value("command", MainConfig::command)) % "Set the command to be run and sampled; defaults is none")), - (clipp::option("-O") & + (clipp::option("-o") & clipp::value("top", MainConfig::top)) % "Set the top number; default is 10", (clipp::option("-f") & clipp::value("freq", MainConfig::freq)) % "Set sampling frequency, 0 for close; default is 49", - (clipp::option("-d") & + (clipp::option("-i") & clipp::value("interval", MainConfig::delay)) % "Set the output delay time (seconds); default is 5", - (clipp::option("-t") & + (clipp::option("-d") & clipp::value("duration", MainConfig::run_time) .call([] { stop_time = time(NULL) + MainConfig::run_time; })) % From 19afafdebba3986ab71e3aacca7f217954d5c5b4 Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Thu, 18 Apr 2024 21:09:37 +0000 Subject: [PATCH 29/47] update docs Signed-off-by: LiuLingze --- .../Stack_Analyser/frame-use-guide.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename "eBPF_Supermarket/Stack_Analyser/\346\241\206\346\236\266\344\275\277\347\224\250\346\226\271\346\263\225.md" => eBPF_Supermarket/Stack_Analyser/frame-use-guide.md (87%) diff --git "a/eBPF_Supermarket/Stack_Analyser/\346\241\206\346\236\266\344\275\277\347\224\250\346\226\271\346\263\225.md" b/eBPF_Supermarket/Stack_Analyser/frame-use-guide.md similarity index 87% rename from "eBPF_Supermarket/Stack_Analyser/\346\241\206\346\236\266\344\275\277\347\224\250\346\226\271\346\263\225.md" rename to eBPF_Supermarket/Stack_Analyser/frame-use-guide.md index 2f393cbd5..8d829a678 100644 --- "a/eBPF_Supermarket/Stack_Analyser/\346\241\206\346\236\266\344\275\277\347\224\250\346\226\271\346\263\225.md" +++ b/eBPF_Supermarket/Stack_Analyser/frame-use-guide.md @@ -32,18 +32,18 @@ new_ebpf.sh 1. 实现包装类初始化函数，若数据标度`scale`的值可以确定，请在此初始化 2. 若标度无法确定，则实现`setScale`，对标度及包装类参数进行设置 -3. 实现一系列虚函数：`count_value, load, attach, detach, unload` +3. 实现一系列虚函数：`count_values, load, attach, detach, unload` 4. 可实现一些辅助函数 ## src/bpf/.bpf.c -1. 通过修改`DeclareCommonMaps(__u32)`中的`__u32`设置计数变量类型 +1. 通过修改`COMMON_MAPS(__u32)`中的`__u32`设置计数变量类型 2. 可声明额外的map和全局变量在eBPF程序内部使用，但不会被框架输出。 3. 实现eBPF程序，请使用通用的`eBPF map`进行数据存储，使用通用的全局变量进行进程和数据过滤，否则无法正确输出数据 通用的map分别为： - 1. psid_count：键为psid类型，值为 1. 中设置的计数变量类型 - 2. stack_trace：键为uint32类型，标识唯一的栈嗲用路径，值为void*[]类型，存储栈上的调用地址 + 1. psid_count_map：键为psid类型，值为 1. 中设置的计数变量类型 + 2. sid_trace_map：键为uint32类型，标识唯一的栈嗲用路径，值为void*[]类型，存储栈上的调用地址 3. pid_tgid：键为uint32类型，表示pid，值为uint32类型，表示tgid 4. pid_comm：键为uint32类型，表示pid，值为comm类型，表示进程名 From cf076534e6b1f2ae0aa75391713c5399b5933639 Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Mon, 22 Apr 2024 02:37:14 +0000 Subject: [PATCH 30/47] fix action Signed-off-by: LiuLingze --- .github/workflows/ebpf_stack_analyser.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ebpf_stack_analyser.yml b/.github/workflows/ebpf_stack_analyser.yml index 907288ffd..2e3fbc406 100644 --- a/.github/workflows/ebpf_stack_analyser.yml +++ b/.github/workflows/ebpf_stack_analyser.yml @@ -30,7 +30,7 @@ jobs: run: | cd eBPF_Supermarket/Stack_Analyser make - sudo ./stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe vfs_open -t 5 + sudo ./stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe vfs_open -d 5 magic-eyes-build-and-test: runs-on: ubuntu-22.04 @@ -48,4 +48,4 @@ jobs: cd MagicEyes/build cmake -DBUILD_STACK_ANALYZER=ON .. make - sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe vfs_open -t 5 \ No newline at end of file + sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe vfs_open -d 5 \ No newline at end of file From 99ffc433bddd40b984f2b5dd0d776cff12b5e9d0 Mon Sep 17 00:00:00 2001 From: gyxforeveryoung <1739037263@qq.com> Date: Mon, 22 Apr 2024 10:53:24 +0800 Subject: [PATCH 31/47] add flunclatency and user-defined probe --- .../Stack_Analyser/bpf/funclatency.bpf.c | 132 +++++++++++ .../include/bpf_wapper/funclatency.h | 59 +++++ .../src/bpf_wapper/funclatency.cpp | 220 ++++++++++++++++++ eBPF_Supermarket/Stack_Analyser/src/main.cpp | 12 +- 4 files changed, 422 insertions(+), 1 deletion(-) create mode 100644 eBPF_Supermarket/Stack_Analyser/bpf/funclatency.bpf.c create mode 100644 eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/funclatency.h create mode 100644 eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/funclatency.cpp diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/funclatency.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/funclatency.bpf.c new file mode 100644 index 000000000..e69f199bc --- /dev/null +++ b/eBPF_Supermarket/Stack_Analyser/bpf/funclatency.bpf.c @@ -0,0 +1,132 @@ +// Copyright 2024 The LMP Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/linuxkerneltravel/lmp/blob/develop/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: luiyanbing@foxmail.com +// +// 内核态bpf程序的模板代码 + +#include "vmlinux.h" +#include +#include +#include +#include + +#include "sa_ebpf.h" +#include "bpf_wapper/funclatency.h" +#include "task.h" + +COMMON_MAPS(time_tuple); +COMMON_VALS; +BPF_HASH(starts,psid,u64); +const volatile int target_pid = 0; + +static int entry(void *ctx) +{ + CHECK_ACTIVE; + struct task_struct *curr = (struct task_struct *)bpf_get_current_task(); // 利用bpf_get_current_task()获得当前的进程tsk + RET_IF_KERN(curr); + + u32 pid = get_task_ns_pid(curr); // 利用帮助函数获得当前进程的pid + if ((target_pid >= 0 && pid != target_pid) || !pid || pid == self_pid) + return 0; + + SAVE_TASK_INFO(pid, curr); + + psid a_psid =GET_COUNT_KEY(pid,ctx); + u64 nsec; + + nsec = bpf_ktime_get_ns(); + bpf_map_update_elem(&starts, &a_psid, &nsec, BPF_ANY); +} + + + +SEC("kprobe/dummy_kprobe") +int BPF_KPROBE(dummy_kprobe) +{ + entry(ctx); + return 0; +} +SEC("tp/sched/dummy_tp") +int tp_entry(void *ctx) +{ + entry(ctx); + return 0; +} +SEC("usdt") +int usdt_entry(void *ctx) +{ + entry(ctx); + return 0; +} +static int exit(void *ctx) +{ + u64 *start; + u64 nsec = bpf_ktime_get_ns(); + u64 delta; + + + + CHECK_ACTIVE; + struct task_struct *curr = (struct task_struct *)bpf_get_current_task(); // 利用bpf_get_current_task()获得当前的进程tsk + RET_IF_KERN(curr); + + u32 pid = get_task_ns_pid(curr); // 利用帮助函数获得当前进程的pid + if ((target_pid >= 0 && pid != target_pid) || !pid || pid == self_pid) + return 0; + + SAVE_TASK_INFO(pid, curr); + + psid b_psid = GET_COUNT_KEY(pid, ctx); + + start = bpf_map_lookup_elem(&starts, &b_psid); + if (!start) + return 0; + + delta = nsec - *start; + + time_tuple *d = bpf_map_lookup_elem(&psid_count_map, &b_psid); + if (!d) + { + time_tuple tmp = {.lat = delta,.count=1}; + bpf_map_update_elem(&psid_count_map, &b_psid, &tmp, BPF_NOEXIST); + } + else + { + d->lat+=delta; + d->count++; + } +} + + + +SEC("kretprobe/dummy_kretprobe") +int BPF_KRETPROBE(dummy_kretprobe) +{ + exit(ctx); + return 0; +} +SEC("tp/sched/dummy_tp") +int tp_exit(void *ctx) +{ + exit(ctx); + return 0; +} +SEC("usdt") +int usdt_exit(void *ctx) +{ + exit(ctx); + return 0; +} +const char LICENSE[] SEC("license") = "GPL"; \ No newline at end of file diff --git a/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/funclatency.h b/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/funclatency.h new file mode 100644 index 000000000..1f826942c --- /dev/null +++ b/eBPF_Supermarket/Stack_Analyser/include/bpf_wapper/funclatency.h @@ -0,0 +1,59 @@ +// Copyright 2024 The LMP Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/linuxkerneltravel/lmp/blob/develop/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: luiyanbing@foxmail.com +// +// ebpf程序的包装类的模板，声明接口和一些自定义方法，以及辅助结构 + +#ifndef _SA_FUNCLANTENCY_H__ +#define _SA_FUNCLANTENCY_H__ + +// ========== C code part ========== +#include +typedef struct +{ + __u64 lat; + __u64 count; +} time_tuple; +// ========== C code end ========== + +#ifdef __cplusplus +// ========== C++ code part ========== +#include "funclatency.skel.h" +#include "bpf_wapper/eBPFStackCollector.h" + +class FunclatencyStackCollector : public StackCollector +{ +private: + DECL_SKEL(funclatency); +public: + std::string probe; + +protected: + virtual uint64_t *count_values(void *); + +public: + void setScale(std::string probe); + FunclatencyStackCollector(); + virtual int load(void); + virtual int attach(void); + virtual void detach(void); + virtual void unload(void); + virtual void activate(bool tf); + virtual const char *getName(void); +}; +// ========== C++ code end ========== +#endif + +#endif \ No newline at end of file diff --git a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/funclatency.cpp b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/funclatency.cpp new file mode 100644 index 000000000..ffa9f8273 --- /dev/null +++ b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/funclatency.cpp @@ -0,0 +1,220 @@ +// Copyright 2024 The LMP Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/linuxkerneltravel/lmp/blob/develop/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: luiyanbing@foxmail.com +// +// ebpf程序包装类的模板，实现接口和一些自定义方法 + +#include "bpf_wapper/funclatency.h" +#include "trace_helpers.h" +#include "uprobe_helpers.h" + +// ========== implement virtual func ========== + +uint64_t *FunclatencyStackCollector::count_values(void *data) +{ + time_tuple *p = (time_tuple *)data; + return new uint64_t[scale_num]{ + p->lat, + p->count}; +}; + +void FunclatencyStackCollector::setScale(std::string probe) +{ + this->probe = probe; + scales->Type = probe + "Delay"; +}; +int FunclatencyStackCollector::load(void) +{ + EBPF_LOAD_OPEN_INIT(skel->rodata->target_pid = pid;); + return 0; +}; +void splitStr(std::string symbol, const char split, std::vector &res) +{ + if (symbol == "") + return; + std::string strs = symbol + split; + size_t pos = strs.find(split); + while (pos != strs.npos) + { + std::string temp = strs.substr(0, pos); + res.push_back(temp); + strs = strs.substr(pos + 1, strs.size()); + pos = strs.find(split); + } +} + +static int get_binpath(char *path, int pid) +{ + char mode[16], line[128], buf[64]; + size_t seg_start, seg_end, seg_off; + FILE *f; + int i = 0; + + sprintf(buf, "/proc/%d/maps", pid); + f = fopen(buf, "r"); + if (!f) + return -1; + + while (fscanf(f, "%zx-%zx %s %zx %*s %*d%[^\n]\n", + &seg_start, &seg_end, mode, &seg_off, line) == 5) + { + i = 0; + while (isblank(line[i])) + i++; + if (strstr(line + i, "libc.so.6")) + { + break; + } + } + + strcpy(path, line + i); + fclose(f); + return 0; +} +static int attach_kprobes(struct funclatency_bpf *skel, std::string func) +{ + skel->links.dummy_kprobe = + bpf_program__attach_kprobe(skel->progs.dummy_kprobe, false, func.c_str()); + CHECK_ERR(!skel->links.dummy_kprobe, "Fail to attach kprobe"); + skel->links.dummy_kretprobe = + bpf_program__attach_kprobe(skel->progs.dummy_kretprobe, true, func.c_str()); + CHECK_ERR(!skel->links.dummy_kretprobe, "Fail to attach ketprobe"); + return 0; +} +static int attach_uprobes(struct funclatency_bpf *skel, std::string probe, int pid) +{ + char *binary, *function; + char bin_path[128]; + std::string func = probe; + off_t func_off; + + binary = strdup(func.c_str()); + function = strchr(binary, ':'); // 查找：首次出现的位置 + *function = '\0'; + function++; + + if (resolve_binary_path(binary, pid, bin_path, sizeof(bin_path))) + free(binary); + + func_off = get_elf_func_offset(bin_path, function); + if (func_off < 0) + free(binary); + skel->links.dummy_kprobe = + bpf_program__attach_uprobe(skel->progs.dummy_kprobe, false, pid, + bin_path, func_off); + CHECK_ERR(!skel->links.dummy_kprobe, "Fail to attach uprobe"); + skel->links.dummy_kretprobe = + bpf_program__attach_uprobe(skel->progs.dummy_kretprobe, true, pid, + bin_path, func_off); + CHECK_ERR(!skel->links.dummy_kretprobe, "Fail to attach uprobe"); + return 0; +} + +static int attach_tp(struct funclatency_bpf *skel, std::string tp_class, std::string func) +{ + + skel->links.tp_entry = + bpf_program__attach_tracepoint(skel->progs.tp_entry, tp_class.c_str(), func.c_str()); + CHECK_ERR(!skel->links.tp_entry, "Fail to attach tracepoint"); + skel->links.tp_exit = + bpf_program__attach_tracepoint(skel->progs.tp_exit, tp_class.c_str(), func.c_str()); + CHECK_ERR(!skel->links.tp_exit, "Fail to attach tracepoint"); + return 0; +} +static int attach_usdt(struct funclatency_bpf *skel, std::string func, int pid) +{ + char bin_path[128]; + int err = get_binpath(bin_path, pid); + CHECK_ERR(err, "Fail to get lib path"); + skel->links.usdt_entry = + bpf_program__attach_usdt(skel->progs.usdt_entry, pid, bin_path, "libc", func.c_str(), NULL); + CHECK_ERR(!skel->links.usdt_entry, "Fail to attach usdt"); + skel->links.usdt_exit = + bpf_program__attach_usdt(skel->progs.usdt_exit, pid, bin_path, "libc", func.c_str(), NULL); + CHECK_ERR(!skel->links.usdt_exit, "Fail to attach usdt"); +} +int FunclatencyStackCollector::attach(void) +{ + // dynamic mounting + std::vector strList; + splitStr(probe, ':', strList); + std::string func = probe; + int err; + + if (strList.size() == 3 && strList[0] == "p" && strList[1] == "") + func = strList[2]; + if (strList.size() == 1 || (strList.size() == 3 && strList[0] == "p" && strList[1] == "")) + { + err = attach_kprobes(skel, func); + } + else if (strList.size() == 3 && strList[0] == "t") + { + err = attach_tp(skel, strList[1], strList[2]); + } + else if (strList.size() == 2 || (strList.size() == 3 && strList[0] == "p" && strList[1] != "")) + { + if (strList.size() == 3) + func = strList[1] + ":" + strList[2]; + err = attach_uprobes(skel, func, pid); + } + else if (strList.size() == 3 && strList[0] == "u") + { + char path[128]; + int err = get_binpath(path,pid); + CHECK_ERR(err, "Fail to get lib path"); + skel->links.usdt_entry = + bpf_program__attach_usdt(skel->progs.usdt_entry ,pid,path,"libc",strList[2].c_str(),NULL); + CHECK_ERR(!skel->links.usdt_entry , "Fail to attach usdt"); + } + else + { + printf("Type must be 'p', 't', or 'u' or too any args"); + } + + CHECK_ERR(err, "Fail to attach"); + + return 0; +}; + +void FunclatencyStackCollector::detach(void) +{ + DETACH_PROTO; +}; + +void FunclatencyStackCollector::unload(void) +{ + UNLOAD_PROTO; +}; + +void FunclatencyStackCollector::activate(bool tf) +{ + ACTIVE_SET(tf); +} + +const char *FunclatencyStackCollector::getName(void) +{ + return "FunclatencyStackCollector"; +} + +// ========== other implementations ========== + +FunclatencyStackCollector::FunclatencyStackCollector() +{ + scale_num = 2; + scales = new Scale[scale_num]{ + {"latencyTime", 1, "us"}, + {"latencyCount", 1, "counts"}, + }; +}; \ No newline at end of file diff --git a/eBPF_Supermarket/Stack_Analyser/src/main.cpp b/eBPF_Supermarket/Stack_Analyser/src/main.cpp index 842363627..6f3105212 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/main.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/main.cpp @@ -23,6 +23,7 @@ #include #include "bpf_wapper/on_cpu.h" +#include "bpf_wapper/funclatency.h" #include "bpf_wapper/llc_stat.h" #include "bpf_wapper/off_cpu.h" #include "bpf_wapper/memleak.h" @@ -151,7 +152,15 @@ int main(int argc, char *argv[]) ->setScale(IntTmp); })) % "Set sampling period; default is 100", TraceOption); - + auto FunclatencyOption = clipp::option("funclatency").call([] + { StackCollectorList.push_back(new FunclatencyStackCollector()); }) % + COLLECTOR_INFO("funclatency") & + (clipp::value("probe", StrTmp) + .call([] + { static_cast(StackCollectorList.back()) + ->setScale(StrTmp); }) % + "set the probe string to time specfic func; default period is ns" & + TraceOption); auto MainOption = _GREEN "Some overall options" _RE % (( ((clipp::option("-p") & @@ -194,6 +203,7 @@ int main(int argc, char *argv[]) ReadaheadOption, ProbeOption, LlcStatOption, + FunclatencyOption, MainOption, Info); } From fa3b350a68adf54a79e82e5c56a92004b236fcfd Mon Sep 17 00:00:00 2001 From: YYS Date: Thu, 25 Apr 2024 15:37:22 +0800 Subject: [PATCH 32/47] optimize code --- eBPF_Supermarket/kvm_watcher/docs/kvm_vcpu.md | 39 +++- .../kvm_watcher/src/kvm_watcher.c | 172 ++++++++---------- 2 files changed, 116 insertions(+), 95 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/docs/kvm_vcpu.md b/eBPF_Supermarket/kvm_watcher/docs/kvm_vcpu.md index e3733231b..39e1aefaf 100644 --- a/eBPF_Supermarket/kvm_watcher/docs/kvm_vcpu.md +++ b/eBPF_Supermarket/kvm_watcher/docs/kvm_vcpu.md @@ -2,7 +2,7 @@ ## 概述 -kvm watcher中的kvm vcpu子功能模块是设计用于监控和分析虚拟化环境中虚拟 CPU (VCPU) 活动的工具，它通过精确记录 VCPU 的唤醒/挂起事件、halt poll 时间的变化，以及 KVM 虚拟机在热迁移过程中产生的脏页信息，为优化虚拟机性能、提高系统响应速度、提供数据支持。 +kvm watcher中的kvm vcpu子功能模块是设计用于监控和分析虚拟化环境中虚拟 CPU (VCPU) 活动的工具，它通过精确记录 VCPU 的唤醒/挂起事件、halt poll 时间的变化，虚拟cpu进调度和出调度的时间记录，以及 KVM 虚拟机在热迁移过程中产生的脏页信息，为优化虚拟机性能、提高系统响应速度、提供数据支持。 ## 原理介绍 @@ -18,6 +18,9 @@ KVM 暂停轮询系统是 KVM 内的一项功能，其在某些情况下可以 在虚拟化环境中，脏页指的是自上次同步以来已经被修改的内存页。特别是在虚拟机热迁移过程中，源虚拟机上的内存页在复制到目标虚拟机的同时仍然处于活动状态，任何在此过程中对这些页的修改都会导致脏页的产生。监控这些脏页对于优化热迁移过程至关重要，因为过多的脏页生成可能会导致迁移延迟，甚至影响到虚拟机的运行性能。此监控功能特别适用于虚拟机热迁移的场景，其中脏页的精确监控和管理可以显著优化迁移过程。 +### load_vcpu + +加载虚拟 CPU（vCPU）是虚拟化环境中的一个重要步骤，它涉及为虚拟机创建和配置虚拟 CPU 实例，并将其加载到虚拟化平台中的运行环境中。加载 vCPU 将为客户机提供计算资源，允许其执行操作系统和应用程序代码。每个 vCPU 实例代表了一个独立的处理器核心，可以并行执行客户机指令。虚拟化环境中的多个 vCPU 可以并发执行客户机代码，从而支持虚拟机中的多任务和并发执行。加载多个 vCPU 允许虚拟机同时运行多个线程或进程。通过合理地分配物理资源并加载 vCPU，KVM可以优化虚拟机的性能和资源利用率。 ## 挂载点 ### wakeup @@ -39,6 +42,12 @@ KVM 暂停轮询系统是 KVM 内的一项功能，其在某些情况下可以 | :----- | :---------------------- | | kprobe | mark_page_dirty_in_slot | +### load_vcpu + +| 类型 | 名称 | +| :----- | :----------------------| +| kprobe | vmx_vcpu_load | +| kprobe | vmx_vcpu_put | ## 示例输出 ### wakeup @@ -124,7 +133,22 @@ PID GFN REL_GFN SLOT_ID COUNTS 630632 f0122 122 3 61 .... ``` +### load_vcpu +``` +sudo ./kvm_watcher -o +TIME:15:23:05 +pid tid total_time max_time min_time counts vcpuid pcpuid +------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ +49214 49224 1.2344 0.3670 0.0346 9 1 2 +49214 49225 0.5978 0.2171 0.1820 3 2 0 +49062 49072 2.4008 0.2656 0.0806 14 0 4 +49214 49223 4.5310 0.2794 0.0217 66 0 7 +54504 54514 6.9243 0.8872 0.0235 34 1 10 +49145 49166 2.8076 1.1653 0.0689 10 1 9 +54504 54513 3.8860 0.3099 0.0210 30 0 0 +49145 49165 1.4737 0.4106 0.0690 8 0 12 +``` ## 参数介绍 ### dirty page @@ -136,4 +160,15 @@ PID GFN REL_GFN SLOT_ID COUNTS - **REL_GFN**: 相对于 GFN 的偏移量。 - **NPAGES**: 内存槽的页面数量。 - **USERSPACE_ADDR**: 触发脏页的用户空间地址。 -- **SLOT_ID**: 内存插槽标识，指示哪个内存区域包含了脏页。 \ No newline at end of file +- **SLOT_ID**: 内存插槽标识，指示哪个内存区域包含了脏页。 + +### load_vcpu + +- **TIME(ms)**: 事件发生的时间，以毫秒为单位。 +- **PID/TID**: 进程或线程的标识符。 +- **total_time**: 2秒内调度vcpu的总时间。 +- **max_time**: 2秒内调度vcpu的最大时间。 +- **min_time**: 2秒内调度vcpu的最小时间。 +- **counts**: 2秒内调度vcpu的次数。 +- **vcpuid**: 虚拟CPU号。 +- **pcpuid**: vCPU对应绑定的物理CPU号。 \ No newline at end of file diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index bcb271968..f61bbece3 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -862,6 +862,80 @@ const char *getCurrentTimeFormatted() { return ts; // 返回指向静态字符串的指针 } +// In order to sort vm_exit maps +int sort_by_key(int fd, struct exit_key *keys, struct exit_value *values) { + int err = 0; + struct exit_key lookup_key = {}; + struct exit_key next_key = {}; + struct exit_value exit_value; + int first = 1; + int i = 0, j; + int count = 0; + while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { + count++; + if (first) { + first = 0; + bpf_map_lookup_elem(fd, &next_key, &exit_value); + keys[0] = next_key; + values[0] = exit_value; + i++; + lookup_key = next_key; + continue; + } + err = bpf_map_lookup_elem(fd, &next_key, &exit_value); + if (err < 0) { + fprintf(stderr, "failed to lookup exit_value: %d\n", err); + return -1; + } + // insert sort + j = i - 1; + struct exit_key temp_key = next_key; + struct exit_value temp_value = exit_value; + while (j >= 0 && + (keys[j].pid > temp_key.pid || (keys[j].tid > temp_key.tid))) { + keys[j + 1] = keys[j]; + values[j + 1] = values[j]; + j--; + } + i++; + keys[j + 1] = next_key; + values[j + 1] = temp_value; + // Move to the next key + lookup_key = next_key; + } + return count; +} + +// clear the specific map +int clear_map(void *lookup_key, void *next_key, enum EventType type, int fd) { + int err; + switch (type) { + case HYPERCALL: + memset(lookup_key, 0, sizeof(struct hc_key)); + break; + case TIMER: + memset(lookup_key, 0, sizeof(struct timer_key)); + break; + case VCPU_LOAD: + memset(lookup_key, 0, sizeof(struct load_key)); + break; + case EXIT: + memset(lookup_key, 0, sizeof(struct exit_key)); + break; + default: + return -1; + } + while (!bpf_map_get_next_key(fd, lookup_key, next_key)) { + err = bpf_map_delete_elem(fd, next_key); + if (err < 0) { + fprintf(stderr, "failed to cleanup hc_count: %d\n", err); + return -1; + } + lookup_key = next_key; + } + return 1; +} + int print_hc_map(struct kvm_watcher_bpf *skel) { int fd = bpf_map__fd(skel->maps.hc_map); int count_fd = bpf_map__fd(skel->maps.hc_count); @@ -894,24 +968,8 @@ int print_hc_map(struct kvm_watcher_bpf *skel) { // // Move to the next key lookup_key = next_key; } - memset(&lookup_key, 0, sizeof(struct hc_key)); - while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { - err = bpf_map_delete_elem(fd, &next_key); - if (err < 0) { - fprintf(stderr, "failed to cleanup hc_map: %d\n", err); - return -1; - } - lookup_key = next_key; - } - memset(&lookup_key, 0, sizeof(struct hc_key)); - while (!bpf_map_get_next_key(count_fd, &lookup_key, &next_key)) { - err = bpf_map_delete_elem(count_fd, &next_key); - if (err < 0) { - fprintf(stderr, "failed to cleanup hc_count: %d\n", err); - return -1; - } - lookup_key = next_key; - } + clear_map(&lookup_key, &next_key, HYPERCALL, fd); + clear_map(&lookup_key, &next_key, HYPERCALL, count_fd); return 0; } @@ -946,64 +1004,10 @@ int print_timer_map(struct kvm_watcher_bpf *skel) { // Move to the next key lookup_key = next_key; } - - // Clear the timer_map - memset(&lookup_key, 0, sizeof(struct timer_key)); - while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { - err = bpf_map_delete_elem(fd, &next_key); - if (err < 0) { - fprintf(stderr, "failed to cleanup timer_map: %d\n", err); - return -1; - } - lookup_key = next_key; - } + clear_map(&lookup_key, &next_key, TIMER, fd); return 0; } -// In order to sort vm_exit maps -int sort_by_key(int fd, struct exit_key *keys, struct exit_value *values) { - int err = 0; - struct exit_key lookup_key = {}; - struct exit_key next_key = {}; - struct exit_value exit_value; - int first = 1; - int i = 0, j; - int count = 0; - while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { - count++; - if (first) { - first = 0; - bpf_map_lookup_elem(fd, &next_key, &exit_value); - keys[0] = next_key; - values[0] = exit_value; - i++; - lookup_key = next_key; - continue; - } - err = bpf_map_lookup_elem(fd, &next_key, &exit_value); - if (err < 0) { - fprintf(stderr, "failed to lookup exit_value: %d\n", err); - return -1; - } - // insert sort - j = i - 1; - struct exit_key temp_key = next_key; - struct exit_value temp_value = exit_value; - while (j >= 0 && - (keys[j].pid > temp_key.pid || (keys[j].tid > temp_key.tid))) { - keys[j + 1] = keys[j]; - values[j + 1] = values[j]; - j--; - } - i++; - keys[j + 1] = next_key; - values[j + 1] = temp_value; - // Move to the next key - lookup_key = next_key; - } - return count; -} - int print_vcpu_load_map(struct kvm_watcher_bpf *skel) { int fd = bpf_map__fd(skel->maps.load_map); int err; @@ -1038,16 +1042,7 @@ int print_vcpu_load_map(struct kvm_watcher_bpf *skel) { load_value.vcpu_id, load_value.pcpu_id); lookup_key = next_key; } - // clear the maps - memset(&lookup_key, 0, sizeof(struct load_key)); - while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { - err = bpf_map_delete_elem(fd, &next_key); - if (err < 0) { - fprintf(stderr, "failed to cleanup counters: %d\n", err); - return -1; - } - lookup_key = next_key; - } + clear_map(&lookup_key, &next_key, VCPU_LOAD, fd); return 0; } @@ -1108,16 +1103,7 @@ void __print_exit_map(int fd, enum NameType name_type) { getName(keys[i].reason, name_type)); } } - // clear the maps - memset(&lookup_key, 0, sizeof(struct exit_key)); - while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { - int err = bpf_map_delete_elem(fd, &next_key); - if (err < 0) { - fprintf(stderr, "failed to cleanup counters: %d\n", err); - return; - } - lookup_key = next_key; - } + clear_map(&lookup_key, &next_key, EXIT, fd); } int print_exit_map(struct kvm_watcher_bpf *skel) { int exit_fd = bpf_map__fd(skel->maps.exit_map); From 9e1171f9a5e04d980256bfda9e54c5a02470ef60 Mon Sep 17 00:00:00 2001 From: LiuLingze Date: Sun, 28 Apr 2024 10:22:30 +0000 Subject: [PATCH 33/47] fix probe usdt Signed-off-by: LiuLingze --- .../Stack_Analyser/src/bpf_wapper/probe.cpp | 10 +++------- eBPF_Supermarket/Stack_Analyser/testdir/usdt_pthread.c | 8 +++++--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/probe.cpp b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/probe.cpp index 1f0306743..e77008f60 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/probe.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/probe.cpp @@ -149,6 +149,7 @@ static int attach_usdt(struct probe_bpf *skel, std::string func, int pid) skel->links.usdt_exit = bpf_program__attach_usdt(skel->progs.usdt_exit, pid, bin_path, "libc", func.c_str(), NULL); CHECK_ERR(!skel->links.usdt_exit, "Fail to attach usdt"); + return 0; } int ProbeStackCollector::attach(void) @@ -157,7 +158,7 @@ int ProbeStackCollector::attach(void) std::vector strList; splitStr(probe, ':', strList); std::string func = probe; - int err; + int err = 0; if (strList.size() == 3 && strList[0] == "p" && strList[1] == "") func = strList[2]; @@ -177,12 +178,7 @@ int ProbeStackCollector::attach(void) } else if (strList.size() == 3 && strList[0] == "u") { - char path[128]; - int err = get_binpath(path, pid); - CHECK_ERR(err, "Fail to get lib path"); - skel->links.usdt_entry = - bpf_program__attach_usdt(skel->progs.usdt_entry, pid, path, "libc", strList[2].c_str(), NULL); - CHECK_ERR(!skel->links.usdt_entry, "Fail to attach usdt"); + err = attach_usdt(skel, strList[2], pid); } else { diff --git a/eBPF_Supermarket/Stack_Analyser/testdir/usdt_pthread.c b/eBPF_Supermarket/Stack_Analyser/testdir/usdt_pthread.c index 6c9729b8d..47acf0b06 100644 --- a/eBPF_Supermarket/Stack_Analyser/testdir/usdt_pthread.c +++ b/eBPF_Supermarket/Stack_Analyser/testdir/usdt_pthread.c @@ -1,13 +1,15 @@ +#define _GNU_SOURCE #include #include #include #include +#include // 线程函数 void *thread_function(void *arg) { while (1) { // 打印当前线程的pid - printf("Thread PID: %d\n", getpid()); + printf("Thread ID: %d\n", gettid()); // 等待一秒 sleep(1); } @@ -29,14 +31,14 @@ void create_thread() { int main() { // 打印主进程的pid - printf("Main process PID: %d\n", getpid()); + printf("Main process ID: %d\n", gettid()); // 调用函数创建线程 create_thread(); // 主进程死循环打印pid while (1) { - printf("Main process PID: %d\n", getpid()); + printf("Main process ID: %d\n", getpid()); // 等待一秒 sleep(1); } From 3bb005825a3f35f88e0d8d2b5d798ca094be6076 Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 10 May 2024 11:40:10 +0800 Subject: [PATCH 34/47] modify sortFun --- eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md | 2 +- .../kvm_watcher/src/kvm_watcher.c | 32 ++++++------------- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md b/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md index 72d9a8034..65f954df8 100644 --- a/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md +++ b/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md @@ -48,7 +48,7 @@ ## 示例输出 4391为主机上的虚拟机进程，4508、4509、4510...分别是虚拟机中的vcpu子进程，每隔两秒输出虚拟机中产生的exit事件及其处理延时等信息。 - +结果会以进程号（VM的唯一标识）以及线程号（VM中每个VCPU的唯一标识）的优先级依次从小到大的顺序输出。 ``` ubuntu@rd350x:~/nans/lmp/eBPF_Supermarket/kvm_watcher$ sudo ./kvm_watcher -e diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index f24eed66f..5b2497b83 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -868,40 +868,26 @@ int sort_by_key(int fd, struct exit_key *keys, struct exit_value *values) { struct exit_key lookup_key = {}; struct exit_key next_key = {}; struct exit_value exit_value; - int first = 1; - int i = 0, j; - int count = 0; - while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { - count++; - if (first) { - first = 0; - bpf_map_lookup_elem(fd, &next_key, &exit_value); - keys[0] = next_key; - values[0] = exit_value; - i++; - lookup_key = next_key; - continue; - } + int i = 0,j=0,count=0; + while(!bpf_map_get_next_key(fd,&lookup_key,&next_key)){ + j = i-1; + struct exit_key temp_key = next_key; err = bpf_map_lookup_elem(fd, &next_key, &exit_value); if (err < 0) { fprintf(stderr, "failed to lookup exit_value: %d\n", err); return -1; } - // insert sort - j = i - 1; - struct exit_key temp_key = next_key; struct exit_value temp_value = exit_value; - while (j >= 0 && - (keys[j].pid > temp_key.pid || (keys[j].tid > temp_key.tid))) { + while(j >= 0 && (keys[j].pid > temp_key.pid || (keys[j].tid > temp_key.tid))){ keys[j + 1] = keys[j]; - values[j + 1] = values[j]; + values[j + 1] = values[j]; j--; } - i++; - keys[j + 1] = next_key; + keys[j + 1] = temp_key; values[j + 1] = temp_value; - // Move to the next key lookup_key = next_key; + count ++; + i++; } return count; } From 7725b237c3ace4447a71df17cb1331652092dddc Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 10 May 2024 12:52:58 +0800 Subject: [PATCH 35/47] Revert "modify sortFun" This reverts commit 3bb005825a3f35f88e0d8d2b5d798ca094be6076. --- eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md | 2 +- .../kvm_watcher/src/kvm_watcher.c | 32 +++++++++++++------ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md b/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md index 65f954df8..72d9a8034 100644 --- a/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md +++ b/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md @@ -48,7 +48,7 @@ ## 示例输出 4391为主机上的虚拟机进程，4508、4509、4510...分别是虚拟机中的vcpu子进程，每隔两秒输出虚拟机中产生的exit事件及其处理延时等信息。 -结果会以进程号（VM的唯一标识）以及线程号（VM中每个VCPU的唯一标识）的优先级依次从小到大的顺序输出。 + ``` ubuntu@rd350x:~/nans/lmp/eBPF_Supermarket/kvm_watcher$ sudo ./kvm_watcher -e diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 5b2497b83..f24eed66f 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -868,26 +868,40 @@ int sort_by_key(int fd, struct exit_key *keys, struct exit_value *values) { struct exit_key lookup_key = {}; struct exit_key next_key = {}; struct exit_value exit_value; - int i = 0,j=0,count=0; - while(!bpf_map_get_next_key(fd,&lookup_key,&next_key)){ - j = i-1; - struct exit_key temp_key = next_key; + int first = 1; + int i = 0, j; + int count = 0; + while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { + count++; + if (first) { + first = 0; + bpf_map_lookup_elem(fd, &next_key, &exit_value); + keys[0] = next_key; + values[0] = exit_value; + i++; + lookup_key = next_key; + continue; + } err = bpf_map_lookup_elem(fd, &next_key, &exit_value); if (err < 0) { fprintf(stderr, "failed to lookup exit_value: %d\n", err); return -1; } + // insert sort + j = i - 1; + struct exit_key temp_key = next_key; struct exit_value temp_value = exit_value; - while(j >= 0 && (keys[j].pid > temp_key.pid || (keys[j].tid > temp_key.tid))){ + while (j >= 0 && + (keys[j].pid > temp_key.pid || (keys[j].tid > temp_key.tid))) { keys[j + 1] = keys[j]; - values[j + 1] = values[j]; + values[j + 1] = values[j]; j--; } - keys[j + 1] = temp_key; + i++; + keys[j + 1] = next_key; values[j + 1] = temp_value; + // Move to the next key lookup_key = next_key; - count ++; - i++; } return count; } From ba56e3f8033d7d6306d24fed46bf4caab3f8c08a Mon Sep 17 00:00:00 2001 From: YYS Date: Fri, 10 May 2024 13:01:41 +0800 Subject: [PATCH 36/47] fix bug --- eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index f24eed66f..69f634a70 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -928,7 +928,7 @@ int clear_map(void *lookup_key, void *next_key, enum EventType type, int fd) { while (!bpf_map_get_next_key(fd, lookup_key, next_key)) { err = bpf_map_delete_elem(fd, next_key); if (err < 0) { - fprintf(stderr, "failed to cleanup hc_count: %d\n", err); + fprintf(stderr, "failed to cleanup map: %d\n", err); return -1; } lookup_key = next_key; From 6ef536acc16fabbdf9de62c2f7ef4942b309dbdf Mon Sep 17 00:00:00 2001 From: gyxforeveryoung <1739037263@qq.com> Date: Sat, 11 May 2024 22:54:32 +0800 Subject: [PATCH 37/47] fix usdt and tracepoint --- .github/workflows/ebpf_stack_analyser.yml | 8 ++++ .../Stack_Analyser/bpf/probe.bpf.c | 39 ++++++++++++------- .../Stack_Analyser/src/bpf_wapper/probe.cpp | 8 +--- eBPF_Supermarket/Stack_Analyser/src/main.cpp | 1 - 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ebpf_stack_analyser.yml b/.github/workflows/ebpf_stack_analyser.yml index 80522307c..8ad3aa57b 100644 --- a/.github/workflows/ebpf_stack_analyser.yml +++ b/.github/workflows/ebpf_stack_analyser.yml @@ -31,10 +31,15 @@ jobs: cd eBPF_Supermarket/Stack_Analyser make sudo ./stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe vfs_open -d 5 + sudo ./stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe p::vfs_open -d 5 + sudo ./stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe t:sched:sched_switch -d 5 gcc -o ./testdir/usdt_pthread ./testdir/usdt_pthread.c sudo ./stack_analyzer probe u:pthread:pthread_create -c "./testdir/usdt_pthread" -d 5 + gcc -o ./testdir/uprobe_malloc ./testdir/uprobe_malloc.c + sudo ./stack_analyzer probe c:malloc -c "./testdir/uprobe_malloc" -d 5 + magic-eyes-build-and-test: runs-on: ubuntu-22.04 steps: @@ -52,4 +57,7 @@ jobs: cmake -DBUILD_STACK_ANALYZER=ON .. make sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe vfs_open -d 5 + sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer probe p::vfs_open -d 5 + sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer probe t:sched:sched_switch -d 5 + sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer probe c:malloc -c "../../eBPF_Supermarket/Stack_Analyser/testdir/uprobe_malloc" -d 5 sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer probe u:pthread:pthread_create -c "../../eBPF_Supermarket/Stack_Analyser/testdir/usdt_pthread" -d 5 diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c index 258f4158f..e37fc26c9 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c @@ -60,18 +60,7 @@ int BPF_KPROBE(dummy_kprobe) entry(ctx); return 0; } -SEC("tp/sched/dummy_tp") -int tp_entry(void *ctx) -{ - entry(ctx); - return 0; -} -SEC("usdt") -int usdt_entry(void *ctx) -{ - entry(ctx); - return 0; -} + static int exit(void *ctx) { CHECK_ACTIVE; @@ -104,16 +93,38 @@ int BPF_KRETPROBE(dummy_kretprobe) exit(ctx); return 0; } + + +static int handleCounts(void *ctx) +{ + CHECK_ACTIVE; + u32 pid = bpf_get_current_pid_tgid() >> 32; + + + psid a_psid = GET_COUNT_KEY(pid, ctx); + time_tuple *d = bpf_map_lookup_elem(&psid_count_map, &a_psid); + if (!d) + { + time_tuple tmp = {.lat = 0, .count = 1}; + bpf_map_update_elem(&psid_count_map, &a_psid, &tmp, BPF_NOEXIST); + } + else + { + d->lat = 0; + d->count++; + } + return 0; +} SEC("tp/sched/dummy_tp") int tp_exit(void *ctx) { - exit(ctx); + handleCounts(ctx); return 0; } SEC("usdt") int usdt_exit(void *ctx) { - exit(ctx); + handleCounts(ctx); return 0; } const char LICENSE[] SEC("license") = "GPL"; \ No newline at end of file diff --git a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/probe.cpp b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/probe.cpp index e77008f60..d4db31cac 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/probe.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/probe.cpp @@ -40,7 +40,7 @@ void ProbeStackCollector::setScale(std::string probe) int ProbeStackCollector::load(void) { - EBPF_LOAD_OPEN_INIT(); + EBPF_LOAD_OPEN_INIT(skel->rodata->target_pid = pid;); return 0; }; @@ -129,9 +129,6 @@ static int attach_uprobes(struct probe_bpf *skel, std::string probe, int pid) static int attach_tp(struct probe_bpf *skel, std::string tp_class, std::string func) { - skel->links.tp_entry = - bpf_program__attach_tracepoint(skel->progs.tp_entry, tp_class.c_str(), func.c_str()); - CHECK_ERR(!skel->links.tp_entry, "Fail to attach tracepoint"); skel->links.tp_exit = bpf_program__attach_tracepoint(skel->progs.tp_exit, tp_class.c_str(), func.c_str()); CHECK_ERR(!skel->links.tp_exit, "Fail to attach tracepoint"); @@ -143,9 +140,6 @@ static int attach_usdt(struct probe_bpf *skel, std::string func, int pid) char bin_path[128]; int err = get_binpath(bin_path, pid); CHECK_ERR(err, "Fail to get lib path"); - skel->links.usdt_entry = - bpf_program__attach_usdt(skel->progs.usdt_entry, pid, bin_path, "libc", func.c_str(), NULL); - CHECK_ERR(!skel->links.usdt_entry, "Fail to attach usdt"); skel->links.usdt_exit = bpf_program__attach_usdt(skel->progs.usdt_exit, pid, bin_path, "libc", func.c_str(), NULL); CHECK_ERR(!skel->links.usdt_exit, "Fail to attach usdt"); diff --git a/eBPF_Supermarket/Stack_Analyser/src/main.cpp b/eBPF_Supermarket/Stack_Analyser/src/main.cpp index c3c38e81d..dfb270895 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/main.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/main.cpp @@ -23,7 +23,6 @@ #include #include "bpf_wapper/on_cpu.h" -#include "bpf_wapper/probe.h" #include "bpf_wapper/llc_stat.h" #include "bpf_wapper/off_cpu.h" #include "bpf_wapper/memleak.h" From 3b3c921606190cc5f5c446c3a3153bf025ef87c6 Mon Sep 17 00:00:00 2001 From: gyxforeveryoung <1739037263@qq.com> Date: Sat, 11 May 2024 22:58:39 +0800 Subject: [PATCH 38/47] fix workflow --- .github/workflows/ebpf_stack_analyser.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ebpf_stack_analyser.yml b/.github/workflows/ebpf_stack_analyser.yml index 8ad3aa57b..f1f6a4d83 100644 --- a/.github/workflows/ebpf_stack_analyser.yml +++ b/.github/workflows/ebpf_stack_analyser.yml @@ -31,8 +31,8 @@ jobs: cd eBPF_Supermarket/Stack_Analyser make sudo ./stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe vfs_open -d 5 - sudo ./stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe p::vfs_open -d 5 - sudo ./stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe t:sched:sched_switch -d 5 + sudo ./stack_analyzer probe p::vfs_open -d 5 + sudo ./stack_analyzer probe t:sched:sched_switch -d 5 gcc -o ./testdir/usdt_pthread ./testdir/usdt_pthread.c sudo ./stack_analyzer probe u:pthread:pthread_create -c "./testdir/usdt_pthread" -d 5 From d4203899edbdfbe9261a2417d7df4425b16e8e97 Mon Sep 17 00:00:00 2001 From: gyxforeveryoung <1739037263@qq.com> Date: Sat, 11 May 2024 23:03:47 +0800 Subject: [PATCH 39/47] add test prog Signed-off-by: gyxforeveryoung <1739037263@qq.com> --- .github/workflows/ebpf_stack_analyser.yml | 4 -- .../Stack_Analyser/testdir/usdt_malloc.c | 69 +++++++++++++++++++ 2 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 eBPF_Supermarket/Stack_Analyser/testdir/usdt_malloc.c diff --git a/.github/workflows/ebpf_stack_analyser.yml b/.github/workflows/ebpf_stack_analyser.yml index f1f6a4d83..8d4926cb9 100644 --- a/.github/workflows/ebpf_stack_analyser.yml +++ b/.github/workflows/ebpf_stack_analyser.yml @@ -37,9 +37,6 @@ jobs: gcc -o ./testdir/usdt_pthread ./testdir/usdt_pthread.c sudo ./stack_analyzer probe u:pthread:pthread_create -c "./testdir/usdt_pthread" -d 5 - gcc -o ./testdir/uprobe_malloc ./testdir/uprobe_malloc.c - sudo ./stack_analyzer probe c:malloc -c "./testdir/uprobe_malloc" -d 5 - magic-eyes-build-and-test: runs-on: ubuntu-22.04 steps: @@ -59,5 +56,4 @@ jobs: sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe vfs_open -d 5 sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer probe p::vfs_open -d 5 sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer probe t:sched:sched_switch -d 5 - sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer probe c:malloc -c "../../eBPF_Supermarket/Stack_Analyser/testdir/uprobe_malloc" -d 5 sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer probe u:pthread:pthread_create -c "../../eBPF_Supermarket/Stack_Analyser/testdir/usdt_pthread" -d 5 diff --git a/eBPF_Supermarket/Stack_Analyser/testdir/usdt_malloc.c b/eBPF_Supermarket/Stack_Analyser/testdir/usdt_malloc.c new file mode 100644 index 000000000..182a29555 --- /dev/null +++ b/eBPF_Supermarket/Stack_Analyser/testdir/usdt_malloc.c @@ -0,0 +1,69 @@ +#include +#include +#include +#include + +#define BUFFER_SIZE 1024 + +int main() { + // 打印当前进程号 + printf("Process ID: %d\n", getpid()); + + // 申请内存 + char *buffer = (char *)malloc(BUFFER_SIZE * sizeof(char)); + if (buffer == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + return 1; + } + + // 文件描述符 + int fd = open("data.txt", O_CREAT | O_WRONLY, 0644); + if (fd == -1) { + fprintf(stderr, "Failed to open file\n"); + free(buffer); + return 1; + } + + // 写数据 + printf("Writing data. Process ID: %d\n", getpid()); + write(fd, "Hello, World!\n", 14); + + // 关闭文件描述符 + close(fd); + + // 打开文件以读取数据 + fd = open("data.txt", O_RDONLY); + if (fd == -1) { + fprintf(stderr, "Failed to open file for reading\n"); + free(buffer); + return 1; + } + + // 读取数据 + printf("Reading data. Process ID: %d\n", getpid()); + ssize_t bytesRead = read(fd, buffer, BUFFER_SIZE); + if (bytesRead == -1) { + fprintf(stderr, "Failed to read data from file\n"); + free(buffer); + close(fd); + return 1; + } + + // 输出读取的数据 + printf("Data read: %s\n", buffer); + + // 关闭文件描述符 + close(fd); + + // 持续打印进程号 + while(1) { + printf("Still running. Process ID: %d\n", getpid()); + sleep(1); // 暂停一秒钟 + } + + // 释放内存 + free(buffer); + printf("Memory freed. Process ID: %d\n", getpid()); + + return 0; +} From ad936bd33c9ab3e0d2ba8dc30e59d50480281bb8 Mon Sep 17 00:00:00 2001 From: gyxforeveryoung <1739037263@qq.com> Date: Sat, 11 May 2024 23:08:23 +0800 Subject: [PATCH 40/47] modify workflow --- .github/workflows/ebpf_stack_analyser.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ebpf_stack_analyser.yml b/.github/workflows/ebpf_stack_analyser.yml index 8d4926cb9..ae4640ae6 100644 --- a/.github/workflows/ebpf_stack_analyser.yml +++ b/.github/workflows/ebpf_stack_analyser.yml @@ -33,10 +33,13 @@ jobs: sudo ./stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe vfs_open -d 5 sudo ./stack_analyzer probe p::vfs_open -d 5 sudo ./stack_analyzer probe t:sched:sched_switch -d 5 - + gcc -o ./testdir/usdt_pthread ./testdir/usdt_pthread.c sudo ./stack_analyzer probe u:pthread:pthread_create -c "./testdir/usdt_pthread" -d 5 + gcc -o ./testdir/uprobe_malloc ./testdir/uprobe_malloc.c + sudo ./stack_analyzer probe c:malloc -c "./testdir/uprobe_malloc" -d 5 + magic-eyes-build-and-test: runs-on: ubuntu-22.04 steps: @@ -56,4 +59,5 @@ jobs: sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe vfs_open -d 5 sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer probe p::vfs_open -d 5 sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer probe t:sched:sched_switch -d 5 + sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer probe c:malloc -c "../../eBPF_Supermarket/Stack_Analyser/testdir/uprobe_malloc" -d 5 sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer probe u:pthread:pthread_create -c "../../eBPF_Supermarket/Stack_Analyser/testdir/usdt_pthread" -d 5 From 6831b625e079f8589d3f8f7e5026e6e4bb53eedd Mon Sep 17 00:00:00 2001 From: gaoyixiang1 <45355878+gaoyixiang1@users.noreply.github.com> Date: Sat, 11 May 2024 23:12:28 +0800 Subject: [PATCH 41/47] Rename usdt_malloc.c to uprobe_malloc.c --- .../Stack_Analyser/testdir/{usdt_malloc.c => uprobe_malloc.c} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename eBPF_Supermarket/Stack_Analyser/testdir/{usdt_malloc.c => uprobe_malloc.c} (100%) diff --git a/eBPF_Supermarket/Stack_Analyser/testdir/usdt_malloc.c b/eBPF_Supermarket/Stack_Analyser/testdir/uprobe_malloc.c similarity index 100% rename from eBPF_Supermarket/Stack_Analyser/testdir/usdt_malloc.c rename to eBPF_Supermarket/Stack_Analyser/testdir/uprobe_malloc.c From d31944b6dca615daa305b61054666463e61fff58 Mon Sep 17 00:00:00 2001 From: YYS Date: Thu, 16 May 2024 15:38:41 +0800 Subject: [PATCH 42/47] improve code --- eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md | 2 +- .../kvm_watcher/src/kvm_watcher.c | 25 +++++-------------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md b/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md index 72d9a8034..65f954df8 100644 --- a/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md +++ b/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md @@ -48,7 +48,7 @@ ## 示例输出 4391为主机上的虚拟机进程，4508、4509、4510...分别是虚拟机中的vcpu子进程，每隔两秒输出虚拟机中产生的exit事件及其处理延时等信息。 - +结果会以进程号（VM的唯一标识）以及线程号（VM中每个VCPU的唯一标识）的优先级依次从小到大的顺序输出。 ``` ubuntu@rd350x:~/nans/lmp/eBPF_Supermarket/kvm_watcher$ sudo ./kvm_watcher -e diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 69f634a70..59dc42080 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -868,28 +868,15 @@ int sort_by_key(int fd, struct exit_key *keys, struct exit_value *values) { struct exit_key lookup_key = {}; struct exit_key next_key = {}; struct exit_value exit_value; - int first = 1; - int i = 0, j; - int count = 0; + int i = 0, j = 0, count = 0; while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { - count++; - if (first) { - first = 0; - bpf_map_lookup_elem(fd, &next_key, &exit_value); - keys[0] = next_key; - values[0] = exit_value; - i++; - lookup_key = next_key; - continue; - } + j = i - 1; + struct exit_key temp_key = next_key; err = bpf_map_lookup_elem(fd, &next_key, &exit_value); if (err < 0) { fprintf(stderr, "failed to lookup exit_value: %d\n", err); return -1; } - // insert sort - j = i - 1; - struct exit_key temp_key = next_key; struct exit_value temp_value = exit_value; while (j >= 0 && (keys[j].pid > temp_key.pid || (keys[j].tid > temp_key.tid))) { @@ -897,11 +884,11 @@ int sort_by_key(int fd, struct exit_key *keys, struct exit_value *values) { values[j + 1] = values[j]; j--; } - i++; - keys[j + 1] = next_key; + keys[j + 1] = temp_key; values[j + 1] = temp_value; - // Move to the next key lookup_key = next_key; + count++; + i++; } return count; } From d856b1ee14443f2d3f62bb53f56275beb4b3be4b Mon Sep 17 00:00:00 2001 From: YYS Date: Thu, 16 May 2024 15:41:44 +0800 Subject: [PATCH 43/47] Revert "improve code" This reverts commit d31944b6dca615daa305b61054666463e61fff58. --- eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md | 2 +- .../kvm_watcher/src/kvm_watcher.c | 25 ++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md b/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md index 65f954df8..72d9a8034 100644 --- a/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md +++ b/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md @@ -48,7 +48,7 @@ ## 示例输出 4391为主机上的虚拟机进程，4508、4509、4510...分别是虚拟机中的vcpu子进程，每隔两秒输出虚拟机中产生的exit事件及其处理延时等信息。 -结果会以进程号（VM的唯一标识）以及线程号（VM中每个VCPU的唯一标识）的优先级依次从小到大的顺序输出。 + ``` ubuntu@rd350x:~/nans/lmp/eBPF_Supermarket/kvm_watcher$ sudo ./kvm_watcher -e diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 59dc42080..69f634a70 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -868,15 +868,28 @@ int sort_by_key(int fd, struct exit_key *keys, struct exit_value *values) { struct exit_key lookup_key = {}; struct exit_key next_key = {}; struct exit_value exit_value; - int i = 0, j = 0, count = 0; + int first = 1; + int i = 0, j; + int count = 0; while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { - j = i - 1; - struct exit_key temp_key = next_key; + count++; + if (first) { + first = 0; + bpf_map_lookup_elem(fd, &next_key, &exit_value); + keys[0] = next_key; + values[0] = exit_value; + i++; + lookup_key = next_key; + continue; + } err = bpf_map_lookup_elem(fd, &next_key, &exit_value); if (err < 0) { fprintf(stderr, "failed to lookup exit_value: %d\n", err); return -1; } + // insert sort + j = i - 1; + struct exit_key temp_key = next_key; struct exit_value temp_value = exit_value; while (j >= 0 && (keys[j].pid > temp_key.pid || (keys[j].tid > temp_key.tid))) { @@ -884,11 +897,11 @@ int sort_by_key(int fd, struct exit_key *keys, struct exit_value *values) { values[j + 1] = values[j]; j--; } - keys[j + 1] = temp_key; + i++; + keys[j + 1] = next_key; values[j + 1] = temp_value; + // Move to the next key lookup_key = next_key; - count++; - i++; } return count; } From 27b9f8da5dfa3ac911437619b8cbdab3f07c65a4 Mon Sep 17 00:00:00 2001 From: YYS Date: Thu, 16 May 2024 15:47:14 +0800 Subject: [PATCH 44/47] Revert "Revert "improve code"" This reverts commit d856b1ee14443f2d3f62bb53f56275beb4b3be4b. --- eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md | 2 +- .../kvm_watcher/src/kvm_watcher.c | 25 +++++-------------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md b/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md index 72d9a8034..65f954df8 100644 --- a/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md +++ b/eBPF_Supermarket/kvm_watcher/docs/kvm_exit.md @@ -48,7 +48,7 @@ ## 示例输出 4391为主机上的虚拟机进程，4508、4509、4510...分别是虚拟机中的vcpu子进程，每隔两秒输出虚拟机中产生的exit事件及其处理延时等信息。 - +结果会以进程号（VM的唯一标识）以及线程号（VM中每个VCPU的唯一标识）的优先级依次从小到大的顺序输出。 ``` ubuntu@rd350x:~/nans/lmp/eBPF_Supermarket/kvm_watcher$ sudo ./kvm_watcher -e diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 69f634a70..59dc42080 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -868,28 +868,15 @@ int sort_by_key(int fd, struct exit_key *keys, struct exit_value *values) { struct exit_key lookup_key = {}; struct exit_key next_key = {}; struct exit_value exit_value; - int first = 1; - int i = 0, j; - int count = 0; + int i = 0, j = 0, count = 0; while (!bpf_map_get_next_key(fd, &lookup_key, &next_key)) { - count++; - if (first) { - first = 0; - bpf_map_lookup_elem(fd, &next_key, &exit_value); - keys[0] = next_key; - values[0] = exit_value; - i++; - lookup_key = next_key; - continue; - } + j = i - 1; + struct exit_key temp_key = next_key; err = bpf_map_lookup_elem(fd, &next_key, &exit_value); if (err < 0) { fprintf(stderr, "failed to lookup exit_value: %d\n", err); return -1; } - // insert sort - j = i - 1; - struct exit_key temp_key = next_key; struct exit_value temp_value = exit_value; while (j >= 0 && (keys[j].pid > temp_key.pid || (keys[j].tid > temp_key.tid))) { @@ -897,11 +884,11 @@ int sort_by_key(int fd, struct exit_key *keys, struct exit_value *values) { values[j + 1] = values[j]; j--; } - i++; - keys[j + 1] = next_key; + keys[j + 1] = temp_key; values[j + 1] = temp_value; - // Move to the next key lookup_key = next_key; + count++; + i++; } return count; } From dd09c837b6164ee29cff8a3546b05a4f904cb960 Mon Sep 17 00:00:00 2001 From: gyxforeveryoung <1739037263@qq.com> Date: Fri, 17 May 2024 18:03:24 +0800 Subject: [PATCH 45/47] kprobe transform fentry Signed-off-by: gyxforeveryoung <1739037263@qq.com> --- .../Stack_Analyser/bpf/probe.bpf.c | 19 ++- .../Stack_Analyser/src/bpf_wapper/probe.cpp | 112 ++++++++++++++---- 2 files changed, 106 insertions(+), 25 deletions(-) diff --git a/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c b/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c index e37fc26c9..23a2288c8 100644 --- a/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c +++ b/eBPF_Supermarket/Stack_Analyser/bpf/probe.bpf.c @@ -53,7 +53,12 @@ static int entry(void *ctx) bpf_map_update_elem(&starts, &pid, &nsec, BPF_ANY); return 0; } - +SEC("fentry/dummy_fentry") +int BPF_PROG(dummy_fentry) +{ + entry(ctx); + return 0; +} SEC("kprobe/dummy_kprobe") int BPF_KPROBE(dummy_kprobe) { @@ -64,8 +69,7 @@ int BPF_KPROBE(dummy_kprobe) static int exit(void *ctx) { CHECK_ACTIVE; - u32 pid = bpf_get_current_pid_tgid() >> 32; - + u32 pid = bpf_get_current_pid_tgid() >> 32; u64 *start = bpf_map_lookup_elem(&starts, &pid); if (!start) return 0; @@ -87,6 +91,14 @@ static int exit(void *ctx) return 0; } +SEC("fexit/dummy_fexit") +int BPF_PROG(dummy_fexit) +{ + exit(ctx); + return 0; +} + + SEC("kretprobe/dummy_kretprobe") int BPF_KRETPROBE(dummy_kretprobe) { @@ -94,7 +106,6 @@ int BPF_KRETPROBE(dummy_kretprobe) return 0; } - static int handleCounts(void *ctx) { CHECK_ACTIVE; diff --git a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/probe.cpp b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/probe.cpp index d4db31cac..db883abbb 100644 --- a/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/probe.cpp +++ b/eBPF_Supermarket/Stack_Analyser/src/bpf_wapper/probe.cpp @@ -21,7 +21,23 @@ #include "uprobe_helpers.h" // ========== implement virtual func ========== - +bool tryf; +std::vector strList; +std::string func; +void splitStr(std::string symbol, const char split, std::vector &res) +{ + if (symbol == "") + return; + std::string strs = symbol + split; + size_t pos = strs.find(split); + while (pos != strs.npos) + { + std::string temp = strs.substr(0, pos); + res.push_back(temp); + strs = strs.substr(pos + 1, strs.size()); + pos = strs.find(split); + } +} uint64_t *ProbeStackCollector::count_values(void *data) { time_tuple *p = (time_tuple *)data; @@ -34,31 +50,67 @@ uint64_t *ProbeStackCollector::count_values(void *data) void ProbeStackCollector::setScale(std::string probe) { this->probe = probe; + splitStr(probe, ':', strList); + func = probe; for (int i = 0; i < scale_num; i++) scales[i].Type = probe + scales[i].Type; }; -int ProbeStackCollector::load(void) +static bool try_fentry(struct probe_bpf *skel, std::string func) { - EBPF_LOAD_OPEN_INIT(skel->rodata->target_pid = pid;); - return 0; -}; + long err; -void splitStr(std::string symbol, const char split, std::vector &res) -{ - if (symbol == "") - return; - std::string strs = symbol + split; - size_t pos = strs.find(split); - while (pos != strs.npos) + if (!fentry_can_attach(func.c_str(), NULL)) { - std::string temp = strs.substr(0, pos); - res.push_back(temp); - strs = strs.substr(pos + 1, strs.size()); - pos = strs.find(split); + return false; } + err = bpf_program__set_attach_target(skel->progs.dummy_fentry, 0, func.c_str()); + if (err) + { + bpf_program__set_autoload(skel->progs.dummy_fentry, false); + bpf_program__set_autoload(skel->progs.dummy_fexit, false); + return false; + } + err = bpf_program__set_attach_target(skel->progs.dummy_fexit, 0, func.c_str()); + if (err) + { + bpf_program__set_autoload(skel->progs.dummy_fentry, false); + bpf_program__set_autoload(skel->progs.dummy_fexit, false); + + return false; + } + + bpf_program__set_autoload(skel->progs.dummy_kprobe, false); + bpf_program__set_autoload(skel->progs.dummy_kretprobe, false); + return true; } +int ProbeStackCollector::load(void) +{ + std::string str = func; + skel = skel->open(NULL); + CHECK_ERR(!skel, "Fail to open BPF skeleton"); + if (strList.size() == 3 && strList[0] == "p" && strList[1] == "") + str = strList[2]; + if (strList.size() == 1 || (strList.size() == 3 && strList[0] == "p" && strList[1] == "")){ + tryf = try_fentry(skel, str); + }else{ + bpf_program__set_autoload(skel->progs.dummy_fentry, false); + bpf_program__set_autoload(skel->progs.dummy_fexit, false); + } + skel->rodata->target_pid = pid; + skel->rodata->trace_user = ustack; + skel->rodata->trace_kernel = kstack; + skel->rodata->self_pid = self_pid; + skel->rodata->target_tgid = tgid; + skel->rodata->target_cgroupid = cgroup; + skel->rodata->freq = freq; + err = skel->load(skel); + CHECK_ERR(err, "Fail to load BPF skeleton"); + obj = skel->obj; + return 0; +}; + static int get_binpath(char *path, int pid) { char mode[16], line[128], buf[64]; @@ -97,6 +149,16 @@ static int attach_kprobes(struct probe_bpf *skel, std::string func) CHECK_ERR(!skel->links.dummy_kretprobe, "Fail to attach ketprobe"); return 0; } +static int attach_fentry(struct probe_bpf *skel) +{ + skel->links.dummy_fentry = + bpf_program__attach(skel->progs.dummy_fentry); + CHECK_ERR(!skel->links.dummy_fentry, "Fail to attach fentry"); + skel->links.dummy_fexit = + bpf_program__attach(skel->progs.dummy_fexit); + CHECK_ERR(!skel->links.dummy_fexit, "Fail to attach fexit"); + return 0; +} static int attach_uprobes(struct probe_bpf *skel, std::string probe, int pid) { char *binary, *function; @@ -149,16 +211,24 @@ static int attach_usdt(struct probe_bpf *skel, std::string func, int pid) int ProbeStackCollector::attach(void) { // dynamic mounting - std::vector strList; - splitStr(probe, ':', strList); - std::string func = probe; + // std::vector strList; + // splitStr(probe, ':', strList); + // std::string func = probe; int err = 0; - if (strList.size() == 3 && strList[0] == "p" && strList[1] == "") func = strList[2]; if (strList.size() == 1 || (strList.size() == 3 && strList[0] == "p" && strList[1] == "")) { - err = attach_kprobes(skel, func); + if (!tryf) + { + err = attach_kprobes(skel, func); + return 0; + } + else + { + err = attach_fentry(skel); + return 0; + } } else if (strList.size() == 3 && strList[0] == "t") { From 82925db948cb2a0ff167ff14ce544a8e7f930b5e Mon Sep 17 00:00:00 2001 From: shuaibo_nan <117452805+nanshuaibo@users.noreply.github.com> Date: Tue, 21 May 2024 14:14:33 +0800 Subject: [PATCH 46/47] =?UTF-8?q?kvm=5Fwatcher:=E4=BC=98=E5=85=88=E4=BD=BF?= =?UTF-8?q?=E7=94=A8fentry=E7=A8=8B=E5=BA=8F=E7=B1=BB=E5=9E=8B=20(#799)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 修改makefile * 更新目录结构 * 添加用户态帮助函数 * 更新目录结构，添加用户态帮助函数 * update * 更新目录结构，添加用户态帮助函数 * update * 优先使用fentry程序类型 * 格式化代码 * 优化代码 --- .../kvm_watcher/include/bpf/kvm_hypercall.h | 2 +- eBPF_Supermarket/kvm_watcher/include/common.h | 10 ++++ .../kvm_watcher/src/helpers/trace_helpers.c | 4 +- .../kvm_watcher/src/kvm_watcher.bpf.c | 48 +++++++++++++++++- .../kvm_watcher/src/kvm_watcher.c | 49 +++++++++++++------ 5 files changed, 93 insertions(+), 20 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/include/bpf/kvm_hypercall.h b/eBPF_Supermarket/kvm_watcher/include/bpf/kvm_hypercall.h index b4768d720..723200827 100644 --- a/eBPF_Supermarket/kvm_watcher/include/bpf/kvm_hypercall.h +++ b/eBPF_Supermarket/kvm_watcher/include/bpf/kvm_hypercall.h @@ -53,7 +53,7 @@ struct { __type(value, u32); } hc_count SEC(".maps"); -static int entry_emulate_hypercall(struct kvm_vcpu *vcpu, void *rb, +static int trace_emulate_hypercall(struct kvm_vcpu *vcpu, void *rb, struct common_event *e) { u32 pid = bpf_get_current_pid_tgid() >> 32; u64 nr, a0, a1, a2, a3; diff --git a/eBPF_Supermarket/kvm_watcher/include/common.h b/eBPF_Supermarket/kvm_watcher/include/common.h index d9435e584..8c5f89006 100644 --- a/eBPF_Supermarket/kvm_watcher/include/common.h +++ b/eBPF_Supermarket/kvm_watcher/include/common.h @@ -19,6 +19,16 @@ #ifndef __KVM_WATCHER_H #define __KVM_WATCHER_H +#define SET_KP_OR_FENTRY_LOAD(function_name, module_name) \ + do { \ + if (fentry_can_attach(#function_name, #module_name)) { \ + bpf_program__set_autoload(skel->progs.fentry_##function_name, \ + true); \ + } else { \ + bpf_program__set_autoload(skel->progs.kp_##function_name, true); \ + } \ + } while (0) + static const char binary_path[] = "/bin/qemu-system-x86_64"; #define __ATTACH_UPROBE(skel, sym_name, prog_name, is_retprobe) \ do { \ diff --git a/eBPF_Supermarket/kvm_watcher/src/helpers/trace_helpers.c b/eBPF_Supermarket/kvm_watcher/src/helpers/trace_helpers.c index c84ed19a0..72b78524b 100644 --- a/eBPF_Supermarket/kvm_watcher/src/helpers/trace_helpers.c +++ b/eBPF_Supermarket/kvm_watcher/src/helpers/trace_helpers.c @@ -976,7 +976,7 @@ bool is_kernel_module(const char *name) { return found; } -static bool fentry_try_attach(int id) { +bool fentry_try_attach(int id) { int prog_fd, attach_fd; char error[4096]; struct bpf_insn insns[] = { @@ -1023,7 +1023,7 @@ bool fentry_can_attach(const char *name, const char *mod) { btf__free(module_btf); btf__free(vmlinux_btf); - return id > 0 && fentry_try_attach(id); + return id > 0; } #define DEBUGFS "/sys/kernel/debug/tracing" diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c index 54352b8de..8e57cb7ae 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c @@ -45,6 +45,13 @@ int BPF_PROG(fentry_kvm_vcpu_halt, struct kvm_vcpu *vcpu) { CHECK_PID(vm_pid); return trace_kvm_vcpu_halt(vcpu); } + +SEC("kprobe/kvm_vcpu_halt") +int BPF_KPROBE(kp_kvm_vcpu_halt, struct kvm_vcpu *vcpu) { + CHECK_PID(vm_pid); + return trace_kvm_vcpu_halt(vcpu); +} + // 追踪vcpu运行信息 SEC("tp/kvm/kvm_vcpu_wakeup") int tp_vcpu_wakeup(struct vcpu_wakeup *ctx) { @@ -67,6 +74,7 @@ SEC("tp/kvm/kvm_entry") int tp_entry(struct exit *ctx) { return trace_kvm_entry(); } + // 记录VCPU调度的信息--进入 SEC("fentry/vmx_vcpu_load") int BPF_PROG(fentry_vmx_vcpu_load, struct kvm_vcpu *vcpu, int cpu) { @@ -74,12 +82,23 @@ int BPF_PROG(fentry_vmx_vcpu_load, struct kvm_vcpu *vcpu, int cpu) { return trace_vmx_vcpu_load(vcpu, cpu); } +SEC("kprobe/vmx_vcpu_load") +int BPF_KPROBE(kp_vmx_vcpu_load, struct kvm_vcpu *vcpu, int cpu) { + CHECK_PID(vm_pid); + return trace_vmx_vcpu_load(vcpu, cpu); +} + // 记录VCPU调度的信息--退出 SEC("fentry/vmx_vcpu_put") int BPF_PROG(fentry_vmx_vcpu_put) { return trace_vmx_vcpu_put(); } +SEC("kprobe/vmx_vcpu_put") +int BPF_KPROBE(kp_vmx_vcpu_put) { + return trace_vmx_vcpu_put(); +} + SEC("fentry/mark_page_dirty_in_slot") int BPF_PROG(fentry_mark_page_dirty_in_slot, struct kvm *kvm, const struct kvm_memory_slot *memslot, gfn_t gfn) { @@ -87,6 +106,13 @@ int BPF_PROG(fentry_mark_page_dirty_in_slot, struct kvm *kvm, return trace_mark_page_dirty_in_slot(kvm, memslot, gfn, &rb, e); } +SEC("kprobe/mark_page_dirty_in_slot") +int BPF_KPROBE(kp_mark_page_dirty_in_slot, struct kvm *kvm, + const struct kvm_memory_slot *memslot, gfn_t gfn) { + CHECK_PID(vm_pid); + return trace_mark_page_dirty_in_slot(kvm, memslot, gfn, &rb, e); +} + SEC("tp/kvm/kvm_page_fault") int tp_page_fault(struct page_fault *ctx) { CHECK_PID(vm_pid); @@ -164,9 +190,15 @@ int BPF_PROG(fexit_vmx_inject_irq, struct kvm_vcpu *vcpu, bool reinjected) { } SEC("fentry/kvm_emulate_hypercall") -int BPF_PROG(fentry_emulate_hypercall, struct kvm_vcpu *vcpu) { +int BPF_PROG(fentry_kvm_emulate_hypercall, struct kvm_vcpu *vcpu) { CHECK_PID(vm_pid); - return entry_emulate_hypercall(vcpu, &rb, e); + return trace_emulate_hypercall(vcpu, &rb, e); +} + +SEC("kprobe/kvm_emulate_hypercall") +int BPF_KPROBE(kp_kvm_emulate_hypercall, struct kvm_vcpu *vcpu) { + CHECK_PID(vm_pid); + return trace_emulate_hypercall(vcpu, &rb, e); } SEC("tp/syscalls/sys_enter_ioctl") @@ -194,8 +226,20 @@ int BPF_PROG(fentry_start_hv_timer, struct kvm_lapic *apic) { return trace_start_hv_timer(apic); } +SEC("kprobe/start_hv_timer") +int BPF_KPROBE(kp_start_hv_timer, struct kvm_lapic *apic) { + CHECK_PID(vm_pid); + return trace_start_hv_timer(apic); +} + SEC("fentry/start_sw_timer") int BPF_PROG(fentry_start_sw_timer, struct kvm_lapic *apic) { CHECK_PID(vm_pid); return trace_start_sw_timer(apic); } + +SEC("kprobe/start_sw_timer") +int BPF_KPROBE(kp_start_sw_timer, struct kvm_lapic *apic) { + CHECK_PID(vm_pid); + return trace_start_sw_timer(apic); +} diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index cc931f9c3..a1e89df5e 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -793,16 +793,43 @@ static int print_event_head(struct env *env) { return 0; } -/*通过env结构体的属性真值来判断是否加载某个挂载函数*/ static void set_disable_load(struct kvm_watcher_bpf *skel) { + bpf_program__set_autoload(skel->progs.fentry_vmx_vcpu_load, false); + bpf_program__set_autoload(skel->progs.kp_vmx_vcpu_load, false); + bpf_program__set_autoload(skel->progs.fentry_vmx_vcpu_put, false); + bpf_program__set_autoload(skel->progs.kp_vmx_vcpu_put, false); + bpf_program__set_autoload(skel->progs.fentry_kvm_vcpu_halt, false); + bpf_program__set_autoload(skel->progs.kp_kvm_vcpu_halt, false); + bpf_program__set_autoload(skel->progs.fentry_mark_page_dirty_in_slot, + false); + bpf_program__set_autoload(skel->progs.kp_mark_page_dirty_in_slot, false); + bpf_program__set_autoload(skel->progs.fentry_kvm_emulate_hypercall, false); + bpf_program__set_autoload(skel->progs.kp_kvm_emulate_hypercall, false); + bpf_program__set_autoload(skel->progs.fentry_start_hv_timer, false); + bpf_program__set_autoload(skel->progs.kp_start_hv_timer, false); + bpf_program__set_autoload(skel->progs.fentry_start_sw_timer, false); + bpf_program__set_autoload(skel->progs.kp_start_sw_timer, false); + + if (env.execute_vcpu_load) { + SET_KP_OR_FENTRY_LOAD(vmx_vcpu_load, kvm_intel); + SET_KP_OR_FENTRY_LOAD(vmx_vcpu_put, kvm_intel); + } + if (env.execute_vcpu_wakeup) { + SET_KP_OR_FENTRY_LOAD(kvm_vcpu_halt, kvm); + } + if (env.execute_mark_page_dirty) { + SET_KP_OR_FENTRY_LOAD(mark_page_dirty_in_slot, kvm); + } + if (env.execute_timer) { + SET_KP_OR_FENTRY_LOAD(start_hv_timer, kvm); + SET_KP_OR_FENTRY_LOAD(start_sw_timer, kvm); + } + if (env.execute_hypercall) { + SET_KP_OR_FENTRY_LOAD(kvm_emulate_hypercall, kvm); + } + bpf_program__set_autoload(skel->progs.tp_vcpu_wakeup, env.execute_vcpu_wakeup ? true : false); - bpf_program__set_autoload(skel->progs.fentry_vmx_vcpu_load, - env.execute_vcpu_load ? true : false); - bpf_program__set_autoload(skel->progs.fentry_vmx_vcpu_put, - env.execute_vcpu_load ? true : false); - bpf_program__set_autoload(skel->progs.fentry_kvm_vcpu_halt, - env.execute_vcpu_wakeup ? true : false); bpf_program__set_autoload(skel->progs.tp_exit, env.execute_exit ? true : false); bpf_program__set_autoload(skel->progs.tp_entry, @@ -813,8 +840,6 @@ static void set_disable_load(struct kvm_watcher_bpf *skel) { env.execute_exit ? true : false); bpf_program__set_autoload(skel->progs.tp_kvm_halt_poll_ns, env.execute_halt_poll_ns ? true : false); - bpf_program__set_autoload(skel->progs.fentry_mark_page_dirty_in_slot, - env.execute_mark_page_dirty ? true : false); bpf_program__set_autoload(skel->progs.tp_page_fault, env.execute_page_fault ? true : false); bpf_program__set_autoload(skel->progs.fexit_tdp_page_fault, @@ -839,14 +864,8 @@ static void set_disable_load(struct kvm_watcher_bpf *skel) { env.execute_irq_inject ? true : false); bpf_program__set_autoload(skel->progs.fexit_vmx_inject_irq, env.execute_irq_inject ? true : false); - bpf_program__set_autoload(skel->progs.fentry_emulate_hypercall, - env.execute_hypercall ? true : false); bpf_program__set_autoload(skel->progs.tp_ioctl, env.execute_ioctl ? true : false); - bpf_program__set_autoload(skel->progs.fentry_start_hv_timer, - env.execute_timer ? true : false); - bpf_program__set_autoload(skel->progs.fentry_start_sw_timer, - env.execute_timer ? true : false); } // 函数不接受参数，返回一个静态分配的字符串 From d94dc0ac01bc0e6c91474bde88cde1d61e49f767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=87=8C=E6=B3=BD?= <105694441+GorilaMond@users.noreply.github.com> Date: Tue, 21 May 2024 14:14:58 +0800 Subject: [PATCH 47/47] add a grafana dashboard (#798) --- .../grafana_stack_analyzer_dashboard.json | 1362 +++++++++++++++++ 1 file changed, 1362 insertions(+) create mode 100644 eBPF_Supermarket/Stack_Analyser/grafana_stack_analyzer_dashboard.json diff --git a/eBPF_Supermarket/Stack_Analyser/grafana_stack_analyzer_dashboard.json b/eBPF_Supermarket/Stack_Analyser/grafana_stack_analyzer_dashboard.json new file mode 100644 index 000000000..dfe7e0cab --- /dev/null +++ b/eBPF_Supermarket/Stack_Analyser/grafana_stack_analyzer_dashboard.json @@ -0,0 +1,1362 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "panels": [], + "title": "cpu", + "type": "row" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "OnCPUTime:OnCPUTime:nanoseconds::", + "queryType": "metrics", + "refId": "A", + "spanSelector": [] + } + ], + "title": "cpu metrics", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 2, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "OnCPUTime:OnCPUTime:nanoseconds::", + "queryType": "profile", + "refId": "A", + "spanSelector": [] + } + ], + "title": "cpu profile", + "type": "flamegraph" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 5, + "panels": [], + "title": "block", + "type": "row" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "OffCPUTime:OffCPUTime:nanoseconds::", + "queryType": "metrics", + "refId": "A", + "spanSelector": [] + } + ], + "title": "block time metrics", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 4, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "OffCPUTime:OffCPUTime:nanoseconds::", + "queryType": "profile", + "refId": "A", + "spanSelector": [] + } + ], + "title": "block time profile", + "type": "flamegraph" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 7, + "panels": [], + "title": "memory inuse", + "type": "row" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "LeakedSize:LeakedSize:bytes::", + "queryType": "metrics", + "refId": "A", + "spanSelector": [] + } + ], + "title": "memory-inuse_space metrics", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 9, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "LeakedSize:LeakedSize:bytes::", + "queryType": "profile", + "refId": "A", + "spanSelector": [] + } + ], + "title": "memory-inuse_space profile", + "type": "flamegraph" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "LeakedCount:LeakedCount:counts::", + "queryType": "metrics", + "refId": "A", + "spanSelector": [] + } + ], + "title": "memory-inuse_alloc metrics", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 12, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "LeakedCount:LeakedCount:counts::", + "queryType": "profile", + "refId": "A", + "spanSelector": [] + } + ], + "title": "memory-inuse_alloc profile", + "type": "flamegraph" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 13, + "panels": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "IOSize:IOSize:bytes::", + "queryType": "metrics", + "refId": "A", + "spanSelector": [] + } + ], + "title": "IO size metrics", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 20, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "IOSize:IOSize:bytes::", + "queryType": "profile", + "refId": "A", + "spanSelector": [] + } + ], + "title": "io size profile", + "type": "flamegraph" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "IOCount:IOCount:counts::", + "queryType": "metrics", + "refId": "A", + "spanSelector": [] + } + ], + "title": "IO count metrics", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 17, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "IOCount:IOCount:counts::", + "queryType": "profile", + "refId": "A", + "spanSelector": [] + } + ], + "title": "io count profile", + "type": "flamegraph" + } + ], + "title": "IO", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 18, + "panels": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "UnusedReadaheadPages:UnusedReadaheadPages:pages::", + "queryType": "metrics", + "refId": "A", + "spanSelector": [] + } + ], + "title": "readahead unused metrics", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 25, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "UnusedReadaheadPages:UnusedReadaheadPages:pages::", + "queryType": "profile", + "refId": "A", + "spanSelector": [] + } + ], + "title": "readahead unused profile", + "type": "flamegraph" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "UsedReadaheadPages:UsedReadaheadPages:pages::", + "queryType": "metrics", + "refId": "A", + "spanSelector": [] + } + ], + "title": "readahead used metrics", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 22, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "UsedReadaheadPages:UsedReadaheadPages:pages::", + "queryType": "profile", + "refId": "A", + "spanSelector": [] + } + ], + "title": "readahead used profile", + "type": "flamegraph" + } + ], + "title": "readahead", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 23, + "panels": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "vfs_openCount:vfs_openCount:counts::", + "queryType": "metrics", + "refId": "A", + "spanSelector": [] + } + ], + "title": "vfs_open count metrics", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 15, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "vfs_openCount:vfs_openCount:counts::", + "queryType": "profile", + "refId": "A", + "spanSelector": [] + } + ], + "title": "vfs_open count profile", + "type": "flamegraph" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "vfs_openTime:vfs_openTime:nanoseconds::", + "queryType": "metrics", + "refId": "A", + "spanSelector": [] + } + ], + "title": "vfs_open time metrics", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 27, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "ddlp0mnbltkw0c" + }, + "groupBy": [], + "labelSelector": "{}", + "profileTypeId": "vfs_openTime:vfs_openTime:nanoseconds::", + "queryType": "profile", + "refId": "A", + "spanSelector": [] + } + ], + "title": "vfs_open time profile", + "type": "flamegraph" + } + ], + "title": "vfs_open", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "stack_analyzer_dashboard", + "uid": "fdlqh6mke9hq8e", + "version": 6, + "weekStart": "" +} \ No newline at end of file