diff --git a/.gitignore b/.gitignore index 1f0a1a1..f10d0bb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ build/* dist/* pmu-checker/pmu-checker src/libtsc.so +__pycache__ \ No newline at end of file diff --git a/Makefile b/Makefile index 95a4df2..105960e 100644 --- a/Makefile +++ b/Makefile @@ -4,13 +4,13 @@ VERSION_FILE := _version.txt VERSION_BASE := $(COMMIT_DATE)_$(COMMIT_ID) VERSION_NUMBER := $(shell cat ${VERSION_FILE}) VERSION_PUBLIC := $(VERSION_NUMBER) -PACKAGE_EXTERNAL := perfspect_$(VERSION_NUMBER).tgz +PACKAGE_EXTERNAL := perfspect.tgz BINARY_FINAL := perfspect BINARY_COLLECT := perf-collect BINARY_POSTPROCESS := perf-postprocess -default: all +default: dist -.PHONY: all test default dist clean format format_check security_scan flakes source_check checkmake dist/version_file dist/$(SOURCE_PACKAGE) +.PHONY: test default dist format format_check style_error_check check dist/version_file dist/$(SOURCE_PACKAGE) clean_dir: rm -rf build/* @@ -34,22 +34,9 @@ build-public/collect: mkdir -p $(TMPDIR)/src mkdir -p $(TMPDIR)/events cp src/* $(TMPDIR)/src && cp events/* $(TMPDIR)/events && cp *.py $(TMPDIR) - sed -i 's/PerfSpect_DEV_VERSION/$(VERSION_PUBLIC)/g' $(TMPDIR)/src/perf_helpers.py - cd $(TMPDIR) && pyinstaller -F perf-collect.py -n $(BINARY_COLLECT) \ - --add-data "./src/libtsc.so:." \ - --add-data "./events/bdx.txt:." \ - --add-data "./events/skx.txt:." \ - --add-data "./events/clx.txt:." \ - --add-data "./events/icx.txt:." \ - --add-data "./events/spr.txt:." \ - --add-data "./events/icx_aws.txt:." \ - --add-data "./events/spr_aws.txt:." \ - --add-data "./events/clx_aws.txt:." \ - --add-data "./events/skx_aws.txt:." \ - --add-binary "../build/pmu-checker:." \ - --runtime-tmpdir . \ - --exclude-module readline - + sed -i 's/PerfSpect_DEV_VERSION/$(VERSION_PUBLIC)/g' $(TMPDIR)/src/perf_helpers.py + cp perf-collect.spec $(TMPDIR) + cd $(TMPDIR) && pyinstaller perf-collect.spec cp $(TMPDIR)/dist/$(BINARY_COLLECT) build/ rm -rf $(TMPDIR) @@ -87,18 +74,16 @@ test: cd dist && tar -xvf perfspect.tgz && cp -r $(BINARY_FINAL) ../test/. cd test && pytest -format: - black src - black *.py - format_check: - black --check src - black --check perf-collect.py perf-postprocess.py + black --check *.py src + +format: + black *.py src -error_check: # ignore false positives - flake8 --ignore=E501,W503,F403,F405,E741 src - flake8 --ignore=E203,E501,E722,W503,F403,F405 *.py --exclude simpleeval.py,perfmon.py,average.py +style_error_check: + # ignore long lines and conflicts with black, i.e., black wins + flake8 *.py src --ignore=E501,W503,E203 -source_check: security_scan format_check error_check +check: format_check style_error_check -dist: source_check dist/$(PACKAGE_EXTERNAL) +dist: check dist/$(PACKAGE_EXTERNAL) diff --git a/README.md b/README.md index 0ece8fb..d2c6265 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Requires recent python and golang. ``` pip3 install -r requirements.txt -make dist +make ``` On successful build, binaries will be created in "dist" folder diff --git a/perf-collect.py b/perf-collect.py index c0394e9..9373cdd 100644 --- a/perf-collect.py +++ b/perf-collect.py @@ -519,9 +519,11 @@ def is_safe_file(fname, substr): False, ) + # reset nmi_watchdog to what it was before running perfspect if (int(nmi_watchdog) != 0) and supervisor: f_nmi = open("/proc/sys/kernel/nmi_watchdog", "w") f_nmi.write(nmi_watchdog) + f_nmi.close() if (args.muxinterval > 0) and supervisor: perf_helpers.set_perf_event_mux_interval(True, 1, mux_intervals) diff --git a/perf-collect.spec b/perf-collect.spec new file mode 100644 index 0000000..3d01f37 --- /dev/null +++ b/perf-collect.spec @@ -0,0 +1,49 @@ +# -*- mode: python ; coding: utf-8 -*- + + +block_cipher = None + + +a = Analysis( + ['perf-collect.py'], + pathex=[], + binaries=[('../build/pmu-checker', '.')], + datas=[('./src/libtsc.so', '.'), ('./events/bdx.txt', '.'), ('./events/skx.txt', '.'), ('./events/clx.txt', '.'), ('./events/icx.txt', '.'), ('./events/spr.txt', '.'), ('./events/icx_aws.txt', '.'), ('./events/spr_aws.txt', '.'), ('./events/clx_aws.txt', '.'), ('./events/skx_aws.txt', '.')], + hiddenimports=[], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=['readline'], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False, +) + +# exclude libtinfo shared library from distributed binaries due to warning observed on Ubuntu 16.04: +# "/bin/bash: ./_MEIuU3XMv/libtinfo.so.5: no version information available (required by /bin/bash)" +a.binaries = [bin for bin in a.binaries if not bin[0].startswith('libtinfo')] + +pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name='perf-collect', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir='.', + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) diff --git a/perf-postprocess.py b/perf-postprocess.py index a24782e..86e6b42 100644 --- a/perf-postprocess.py +++ b/perf-postprocess.py @@ -763,8 +763,6 @@ def write_perf_tmp_output(use_epoch): epoch = int(words[-1]) except ValueError: exit("Conversion error parsing timestamp") - except: - exit("Unkown error parsing timestamp") break # TO:DO remove "not_counted" and "not_supported" events from dat_file @@ -1433,7 +1431,7 @@ def is_safe_path(base_dir, path, follow_symlinks=True): if (args.outfile).endswith("xlsx"): try: import xlsxwriter - except: + except ImportError: raise SystemExit( "xlsxwriter not found to generate excel output. Install xlsxwriter or use .csv" ) diff --git a/pmu-checker/msr/msr.go b/pmu-checker/msr/msr.go index 591ddf6..4ddaf51 100644 --- a/pmu-checker/msr/msr.go +++ b/pmu-checker/msr/msr.go @@ -52,7 +52,7 @@ func (dpt retMSR) read(msr int64) (uint64, error) { rc, err := syscall.Pread(dpt.fd, buf, msr) if err != nil { log.Fatal(err) - panic(err) + return 0, err } if rc != 8 { diff --git a/src/perf_helpers.py b/src/perf_helpers.py index 86607ed..0ae5eab 100644 --- a/src/perf_helpers.py +++ b/src/perf_helpers.py @@ -14,8 +14,9 @@ import collections import psutil import subprocess # nosec +import logging from time import strptime -from ctypes import * # flake8: noqa +from ctypes import cdll, CDLL from datetime import datetime from dateutil import tz @@ -166,7 +167,7 @@ def get_version(): try: fo = open("/proc/version", "r") except EnvironmentError as e: - warnings.warn(str(e), UserWarning) + logging.warn(str(e), UserWarning) else: version = fo.read() version = version.split("#")[0] @@ -180,7 +181,7 @@ def get_cpuinfo(): try: fo = open("/proc/cpuinfo", "r") except EnvironmentError as e: - warnings.warn(str(e), UserWarning) + logging.warn(str(e), UserWarning) else: for line in fo: try: