diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8ed044f..8b6aa3d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,3 +23,32 @@ jobs: with: name: perfspect path: dist/perfspect*.tgz + + build-hotspot: + runs-on: ubuntu-20.04 + container: + image: centos:7 + steps: + - uses: actions/checkout@v3 + - name: install dependencies + run: | + yum update -y + yum install -y make python3 gcc cmake gcc-c++ java-1.8.0-openjdk-devel.x86_64 git + python3 -m pip install --upgrade pip + curl -LJO https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl + curl -LJO https://raw.githubusercontent.com/brendangregg/FlameGraph/master/difffolded.pl + curl -LJO https://raw.githubusercontent.com/brendangregg/FlameGraph/master/stackcollapse-perf.pl + chmod +x *.pl + git clone https://github.com/jvm-profiling-tools/perf-map-agent.git + cd perf-map-agent + cmake . + make + - name: build + run: | + pip3 install -r requirements.txt + pyinstaller -F hotspot.py -n hotspot --bootloader-ignore-signals --add-data "perf-map-agent/out/*:." --add-data "flamegraph.pl:." --add-data "difffolded.pl:." --add-data "stackcollapse-perf.pl:." --runtime-tmpdir . --exclude-module readline + - name: upload artifact + uses: actions/upload-artifact@v3 + with: + name: hotspot + path: dist/hotspot \ No newline at end of file diff --git a/_version.txt b/_version.txt index e05cb33..d4c4950 100644 --- a/_version.txt +++ b/_version.txt @@ -1 +1 @@ -1.3.8 +1.3.9 diff --git a/hotspot.py b/hotspot.py new file mode 100644 index 0000000..f741cb3 --- /dev/null +++ b/hotspot.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 + +########################################################################################################### +# Copyright (C) 2021-2023 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +########################################################################################################### + +import logging +import os +import platform +import shlex +import shutil +import subprocess +import sys + +from argparse import ArgumentParser +from src.common import configure_logging, crash +from src.perf_helpers import get_perf_list + + +def fix_path(script): + return os.path.join( + getattr(sys, "_MEIPASS", os.path.dirname(os.path.abspath(__file__))), + script, + ) + + +def attach_perf_map_agent(): + # look for java processes + try: + pids = ( + subprocess.check_output(shlex.split("pgrep java"), encoding="UTF-8") + .strip() + .split("\n") + ) + except subprocess.CalledProcessError: + return + + if len(pids) > 0 and pids[0] != "": + logging.info("detected java processes: " + str(pids)) + + # setup tmp folder for storing perf-map-agent + if not os.path.exists("/tmp/perfspect"): + os.mkdir("/tmp/perfspect") + shutil.copy(fix_path("attach-main.jar"), "/tmp/perfspect") + shutil.copy(fix_path("libperfmap.so"), "/tmp/perfspect") + os.chmod("/tmp/perfspect/attach-main.jar", 0o666) + os.chmod("/tmp/perfspect/libperfmap.so", 0o666) + + for pid in pids: + uid = subprocess.check_output( + shlex.split("awk '/^Uid:/{print $2}' /proc/" + pid + "/status"), + encoding="UTF-8", + ) + gid = subprocess.check_output( + shlex.split("awk '/^Gid:/{print $2}' /proc/" + pid + "/status"), + encoding="UTF-8", + ) + JAVA_HOME = subprocess.check_output( + shlex.split('sed "s:bin/java::"'), + input=subprocess.check_output( + shlex.split("readlink -f /usr/bin/java"), encoding="UTF-8" + ), + encoding="UTF-8", + ).strip() + current_dir = os.getcwd() + try: + os.chdir("/tmp/perfspect/") + subprocess.check_call( + shlex.split( + f"sudo -u \\#{uid} -g \\#{gid} {JAVA_HOME}bin/java -cp /tmp/perfspect/attach-main.jar:{JAVA_HOME}lib/tools.jar net.virtualvoid.perf.AttachOnce {pid}" + ), + encoding="UTF-8", # type: ignore + ) + logging.info("Successfully attached perf-map-agent to: " + pid) + except subprocess.CalledProcessError: + logging.info("Failed to attach perf-map-agent to: " + pid) + os.chdir(current_dir) + + +if __name__ == "__main__": + configure_logging(".") + + parser = ArgumentParser( + description="hotspot: PMU based flamegraphs for hotspot analysis" + ) + parser.add_argument( + "-t", + "--timeout", + required=True, + type=int, + help="collection time", + ) + args = parser.parse_args() + if os.geteuid() != 0: + crash("Must run as root, please re-run") + if platform.system() != "Linux": + crash("PerfSpect currently supports Linux only") + get_perf_list() + + events = ["instructions", "cycles", "branch-misses", "cache-misses"] + + logging.info("collecting...") + + attach_perf_map_agent() + + subprocess.run( + shlex.split( + "sudo perf record -a -g -F 99 -e " + + ",".join(events) + + " sleep " + + str(args.timeout) + ) + ) + + logging.info("postprocessing...") + + script = subprocess.run( + shlex.split("perf script"), + stdout=subprocess.PIPE, + ) + cycles_collapse = "" + with open("cycles.col", "w") as c: + cycles_collapse = subprocess.run( + shlex.split(fix_path("stackcollapse-perf.pl") + ' --event-filter="cycles"'), + input=script.stdout, + stdout=c, + ) + for event, subtitle, differential in [ + ["branch-misses", "What is being stalled by poor prefetching", False], + ["cache-misses", "What is being stalled by poor caching", False], + ["instructions", "CPI: blue = vectorized, red = stalled", True], + ]: + with open(event + ".svg", "w") as f: + collapse = "" + with open(event + ".col", "w") as e: + collapse = subprocess.run( + shlex.split( + fix_path("stackcollapse-perf.pl") + + ' --event-filter="' + + event + + '"' + ), + input=script.stdout, + stdout=e, + ) + if differential: + with open("diff.col", "w") as e: + collapse = subprocess.run( + shlex.split( + fix_path("difffolded.pl") + " " + event + ".col cycles.col" + ), + stdout=e, + ) + with open("diff.col" if differential else event + ".col", "r") as e: + flamegraph = subprocess.run( + shlex.split( + fix_path("flamegraph.pl") + + ' --title="' + + event + + '" --subtitle="' + + subtitle + + '"' + ), + stdin=e, + stdout=f, + ) + os.remove(event + ".col") + if differential: + os.remove("diff.col") + os.chmod(event + ".svg", 0o666) + logging.info("generated " + event + ".svg") + + os.remove("cycles.col") diff --git a/instruction-mix.py b/instruction-mix.py deleted file mode 100644 index 1e4371c..0000000 --- a/instruction-mix.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 - -########################################################################################################### -# Copyright (C) 2021-2023 Intel Corporation -# SPDX-License-Identifier: BSD-3-Clause -########################################################################################################### - -import logging -import os -import platform -import shlex -import subprocess -import sys - -from iced_x86 import Decoder, Formatter, FormatterSyntax # type: ignore -from src.common import configure_logging, crash -from src.perf_helpers import get_perf_list - -formatter = Formatter(FormatterSyntax.NASM) - - -def get_insn(insn): - global formatter - insn = bytes.fromhex(insn.replace(" ", "")) - insnraw = formatter.format(Decoder(64, insn).decode()).split(" ", 1) - insn = insnraw[0] - reg = [] - if len(insnraw) > 1: - if "xmm" in insnraw[1]: - reg.append("AVX128") - if "ymm" in insnraw[1]: - reg.append("AVX256") - if "zmm" in insnraw[1]: - reg.append("AVX512") - if len(reg) > 0: - insn += ";" + ",".join(reg) - return insn - - -if __name__ == "__main__": - configure_logging(".") - - if len(sys.argv) != 2 or not sys.argv[1].isdigit(): - print('usage: "sudo ./instruction-mix 3 # run for 3 seconds"') - sys.exit() - if os.geteuid() != 0: - crash("Must run as root, please re-run") - if platform.system() != "Linux": - crash("PerfSpect currently supports Linux only") - get_perf_list() - - logging.info("collecting...") - - subprocess.run(shlex.split("perf record -a -F 99 sleep " + sys.argv[1])) - rawdata = ( - subprocess.Popen( - shlex.split("perf script -F comm,pid,insn"), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - .communicate()[0] - .decode() - .split("\n")[:-1] - ) - - logging.info("postprocessing...") - - processmap = {} - - for row in rawdata: - sides = row.split("insn:") - if len(sides) > 1: - insn = get_insn(sides[1]) - id = sides[0].split()[0] + ";" + insn - if id not in processmap: - processmap[id] = 0 - processmap[id] += 1 - - # generate freqs - col = "" - for p in processmap: - col += p + " " + str(processmap[p]) + "\n" - - with open("instruction-mix.svg", "w") as f: - subprocess.run( - shlex.split( - os.path.join( - getattr( - sys, "_MEIPASS", os.path.dirname(os.path.abspath(__file__)) - ), - "flamegraph.pl", - ) - ), - input=col.encode(), - stdout=f, - ) - - os.chmod("instruction-mix.svg", 0o666) - logging.info("generated instruction-mix.svg") diff --git a/requirements.txt b/requirements.txt index 73d9483..83dda8f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,7 @@ black flake8 -iced-x86 pytype simpleeval pandas -plotly -psutil pyinstaller -pytest -python-dateutil -XlsxWriter -yattag \ No newline at end of file +pytest \ No newline at end of file