From c1faf5a4a2f73b23b15703938db58ea24f427ef8 Mon Sep 17 00:00:00 2001 From: Manikandan Dhamodharan Date: Wed, 13 Mar 2024 00:23:21 -0400 Subject: [PATCH] Major dependency upgrade python to 3.x (deco 2.x) with latest jupyter compatibility --- .github/workflows/main.yml | 2 +- CMakeLists.txt | 1 + build.sh | 2 +- demo/Demo.H | 3 -- include/xpedite/probes/CallSite.H | 1 + include/xpedite/probes/ProbeKey.H | 1 + install.sh | 2 +- jni/build.gradle | 4 +-- lib/xpedite/framework/StorageMgr.H | 1 + scripts/bin/xpedite | 35 ++++++++++--------- scripts/lib/setup.py | 2 ++ scripts/lib/xpedite/analytics/__init__.py | 2 +- scripts/lib/xpedite/benchmark/__init__.py | 4 +-- scripts/lib/xpedite/benchmark/info.py | 2 +- scripts/lib/xpedite/jupyter/archive.py | 2 +- scripts/lib/xpedite/jupyter/commands.py | 2 +- .../jupyter/data/config/custom/custom.css | 12 +++---- .../data/config/jupyter_server_config.py | 18 ++++++++++ .../data/config/serverextensions/__init__.py | 0 .../__init__.py} | 33 ++++++++++------- scripts/lib/xpedite/jupyter/driver.py | 6 ++-- .../xpedite/jupyter/templates/initCell.fmt | 4 +-- scripts/lib/xpedite/profiler/app.py | 4 +-- scripts/lib/xpedite/profiler/environment.py | 2 +- scripts/lib/xpedite/profiler/profileInfo.py | 6 ++-- scripts/lib/xpedite/profiler/runtime.py | 6 ++-- scripts/lib/xpedite/requirements.txt | 5 ++- scripts/lib/xpedite/txn/extractor.py | 4 +-- scripts/lib/xpedite/txn/repo.py | 2 +- scripts/lib/xpedite/util/__init__.py | 2 +- test/runTest.sh | 31 +++++++++++++--- 31 files changed, 129 insertions(+), 72 deletions(-) create mode 100644 scripts/lib/xpedite/jupyter/data/config/jupyter_server_config.py delete mode 100644 scripts/lib/xpedite/jupyter/data/config/serverextensions/__init__.py rename scripts/lib/xpedite/jupyter/data/config/serverextensions/{tornadoExtension.py => tornadoExtension/__init__.py} (70%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4dedda6f..d7adca11 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [2.7, 3.8] + python-version: [3.8] # Steps represent a sequence of tasks that will be executed as part of the job steps: diff --git a/CMakeLists.txt b/CMakeLists.txt index 63c310a5..f459663f 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,6 +58,7 @@ install(TARGETS xpedite-pic DESTINATION "lib" COMPONENT libraries) ######################### Python bindings ############################# +set(PYBIND11_PYTHON_VERSION 3.8 CACHE STRING "") find_package(pybind11 REQUIRED) pybind11_add_module(xpediteBindings lib/xpedite/pybind/Bindings.cpp lib/xpedite/framework/SamplesLoader.C) install(TARGETS xpediteBindings DESTINATION "lib" COMPONENT libraries) diff --git a/build.sh b/build.sh index 2fb548f9..66d2c3a2 100755 --- a/build.sh +++ b/build.sh @@ -33,7 +33,7 @@ BUILD_TYPE=Release BUILD_VIVIFY=0 BUILD_JAVA=0 VERBOSE=0 -PYTHON_VERSION=2.7 +PYTHON_VERSION=3.8 while true ; do case "$1" in diff --git a/demo/Demo.H b/demo/Demo.H index 1fa80a43..72f8f383 100644 --- a/demo/Demo.H +++ b/demo/Demo.H @@ -30,9 +30,6 @@ namespace xpedite { namespace demo { using Indices = std::array; inline void initialize(size_t core_ = 0) { - if(mlockall(MCL_CURRENT | MCL_FUTURE)) { - std::cerr << "failed to mlock all pages" << std::endl; - } using namespace xpedite::framework; if(!xpedite::framework::initialize("xpedite-appinfo.txt", {AWAIT_PROFILE_BEGIN})) { throw std::runtime_error {"failed to init xpedite"}; diff --git a/include/xpedite/probes/CallSite.H b/include/xpedite/probes/CallSite.H index 89547310..1e18896e 100644 --- a/include/xpedite/probes/CallSite.H +++ b/include/xpedite/probes/CallSite.H @@ -12,6 +12,7 @@ #pragma once #include +#include namespace xpedite { namespace probes { diff --git a/include/xpedite/probes/ProbeKey.H b/include/xpedite/probes/ProbeKey.H index ff24d58a..357d3062 100644 --- a/include/xpedite/probes/ProbeKey.H +++ b/include/xpedite/probes/ProbeKey.H @@ -8,6 +8,7 @@ #pragma once #include +#include namespace xpedite { namespace probes { diff --git a/install.sh b/install.sh index 60485a52..25e6cd71 100755 --- a/install.sh +++ b/install.sh @@ -31,7 +31,7 @@ fi eval set -- "$ARGS" ENABLE_PMU=0 VERBOSE=0 -PYTHON_VERSION=2 +PYTHON_VERSION=3 while true ; do case "$1" in diff --git a/jni/build.gradle b/jni/build.gradle index 76de4d83..e23d8723 100644 --- a/jni/build.gradle +++ b/jni/build.gradle @@ -25,7 +25,7 @@ sourceSets { } task xpediteJar(type: Jar) { - baseName = 'xpedite' + archiveBaseName = 'xpedite' exclude(['**/demo/**']) manifest { attributes( @@ -42,7 +42,7 @@ task xpediteJar(type: Jar) { task demoJar(type: Jar) { include(['**/demo/**']) exclude(['**/com/xpedite/probes']) - baseName = 'xpediteDemo' + archiveBaseName = 'xpediteDemo' manifest { attributes( 'Implementation-Title': 'Xpedite Demo Jar', diff --git a/lib/xpedite/framework/StorageMgr.H b/lib/xpedite/framework/StorageMgr.H index e940d811..8288535c 100644 --- a/lib/xpedite/framework/StorageMgr.H +++ b/lib/xpedite/framework/StorageMgr.H @@ -11,6 +11,7 @@ #pragma once #include +#include namespace xpedite { namespace framework { diff --git a/scripts/bin/xpedite b/scripts/bin/xpedite index 30ade75c..97e6a5e9 100755 --- a/scripts/bin/xpedite +++ b/scripts/bin/xpedite @@ -26,8 +26,8 @@ logger.init() from xpedite.dependencies import Package, DEPENDENCY_LOADER DEPENDENCY_LOADER.load(Package.Cement) -from cement.core.foundation import CementApp -from cement.ext.ext_argparse import ArgparseController, expose +from cement import App, CaughtSignal, Controller, ex + import logging import xpedite @@ -54,15 +54,16 @@ def _loadDriver(name = None): return Driver() return driver -class AppController(ArgparseController): + +class AppController(Controller): class Meta: label = 'base' - @expose(hide=True) + @ex(hide=True) def default(self): self.probes() - @expose( + @ex( arguments=[ (['-p', '--profileInfo'], dict(action='store', required=True, help=profileInfoHelp)), (['-b', '--createBenchmark'], dict(action='store', help='store Live profile report at the given path, for future benchmarking')), @@ -94,7 +95,7 @@ class AppController(ArgparseController): driver = _loadDriver(self.app.pargs.driver) driver.render(profileInfo, report, leanReports = self.app.pargs.lean, cprofile = cprofile) - @expose( + @ex( arguments=[ (['-p', '--profileInfo'], dict(action='store', required=True, help=profileInfoHelp)), ], @@ -113,7 +114,7 @@ class AppController(ArgparseController): LOGGER.error('failed to locate probes in app %s. Have you instrumented any ?', profileInfo.appName) LOGGER.info('') - @expose( + @ex( arguments=[ (['-H', '--hostname'], dict(action='store', default='127.0.0.1', help='host where the app runs')), (['-a', '--appInfo'], dict(action='store', required=True, help='path to appInfo file, used in xpedite framework initilization')), @@ -124,7 +125,7 @@ class AppController(ArgparseController): from xpedite.profiler import Profiler Profiler.generate(self.app.pargs.appInfo, hostname=self.app.pargs.hostname) - @expose( + @ex( arguments=[ (['-H', '--home'], dict(action='store', required=False, help='Home dir for xpedite shell')), (['-z', '--zip'], dict(action='store', required=False, help='Archive a xpedite report. Takes path of a xpedite notebook as argument')), @@ -141,7 +142,7 @@ class AppController(ArgparseController): return home = self.app.pargs.home if not home: - LOGGER.warn('No home directory specified for shell - using current working directory') + LOGGER.warning('No home directory specified for shell - using current working directory') home = os.getcwd() if self.app.pargs.unzip: @@ -149,13 +150,13 @@ class AppController(ArgparseController): deflator.deflate(home) launchJupyter(home) -class FrameworkController(ArgparseController): +class FrameworkController(Controller): class Meta: label = 'FrameworkController' stacked_on = 'base' stacked_type = 'embedded' - @expose( + @ex( arguments=[ (['-p', '--profileInfo'], dict(action='store', required=True, help=profileInfoHelp)), (['-b', '--createBenchmark'], dict(action='store', help='store report at the given path, for future benchmarking')), @@ -191,7 +192,7 @@ class FrameworkController(ArgparseController): driver = _loadDriver(self.app.pargs.driver) driver.render(profileInfo, report, leanReports = self.app.pargs.lean, cprofile = cprofile) - @expose( + @ex( arguments=[ ], help='Print the CPU info for the current machine' @@ -201,7 +202,7 @@ class FrameworkController(ArgparseController): cpuId = getCpuId() LOGGER.info('%s\n', cpuId) - @expose( + @ex( arguments=[ (['-c', '--cpuId'], dict(action='store', required=False, help='list pmu counters for the give cpu identifier')), ], @@ -213,7 +214,7 @@ class FrameworkController(ArgparseController): LOGGER.info(e) LOGGER.info('') - @expose( + @ex( arguments=[ (['-n', '--node'], dict(action='store', help='list pmu events for node - enter "all" to list events for all nodes')), (['-c', '--cpuId'], dict(action='store', required=False, help='list pmu counters for the give cpu identifier')), @@ -227,7 +228,7 @@ class FrameworkController(ArgparseController): LOGGER.info('%s', topdown.metricsToString(n)) LOGGER.info('') - @expose( + @ex( arguments=[ (['-n', '--node'], dict(action='store', help='list pmu events for node - enter "all" to list events for all nodes')), (['-c', '--cpuId'], dict(action='store', required=False, help='list pmu counters for the give cpu identifier')), @@ -244,7 +245,7 @@ class FrameworkController(ArgparseController): return LOGGER.info('%s\n', topdown.hierarchy) - @expose( + @ex( arguments=[ (['-e', '--enable'], dict(action='store_true', help='load xpedite kernel module to ENABLE hardware performance counters')), (['-d', '--disable'], dict(action='store_true', help='unload xpedite kernel module to DISABLE hardware performance counters')), @@ -264,7 +265,7 @@ class FrameworkController(ArgparseController): extractor = subprocess.call(cmdList) LOGGER.info('pmc status - %s\n', 'enabled' if isDriverLoaded() else 'disabled') -class XpediteClient(CementApp): +class XpediteClient(App): class Meta: label = 'Xpedite' base_controller = 'base' diff --git a/scripts/lib/setup.py b/scripts/lib/setup.py index a5e57422..22912e00 100644 --- a/scripts/lib/setup.py +++ b/scripts/lib/setup.py @@ -65,5 +65,7 @@ 'py-cpuinfo', 'jupyter', 'six', + 'ipynbname', + 'iinit', ], zip_safe=False) diff --git a/scripts/lib/xpedite/analytics/__init__.py b/scripts/lib/xpedite/analytics/__init__.py index a2edeeed..8795e723 100644 --- a/scripts/lib/xpedite/analytics/__init__.py +++ b/scripts/lib/xpedite/analytics/__init__.py @@ -60,7 +60,7 @@ def buildElapsedTimeBundles(txnCollections, classifier): else: if category not in categorySet: categorySet.add(category) - LOGGER.warn('current run missing trasactions for category "%s"', category) + LOGGER.warning('current run missing trasactions for category "%s"', category) else: scopeList = ', '.join([probe.getCanonicalName() for probe in probes if not probe.isAnonymous]) errMsg = ( diff --git a/scripts/lib/xpedite/benchmark/__init__.py b/scripts/lib/xpedite/benchmark/__init__.py index d78b8151..e004749e 100644 --- a/scripts/lib/xpedite/benchmark/__init__.py +++ b/scripts/lib/xpedite/benchmark/__init__.py @@ -102,14 +102,14 @@ def gatherBenchmarks(self, count): benchmark.dataSource = dataSource benchmarks.append(benchmark) else: - LOGGER.warn('skip processing benchmark %s. failed to load benchmark info', path) + LOGGER.warning('skip processing benchmark %s. failed to load benchmark info', path) if len(benchmarks) >= count: if i + 1 < len(self.benchmarkPaths): LOGGER.debug('skip processing %s benchmarks. limit reached.', self.benchmarkPaths[i+1:]) break else: - LOGGER.warn('skip processing benchmark %s. failed to locate benchmark files', path) + LOGGER.warning('skip processing benchmark %s. failed to locate benchmark files', path) return benchmarks @staticmethod diff --git a/scripts/lib/xpedite/benchmark/info.py b/scripts/lib/xpedite/benchmark/info.py index 12401dfe..ebd638e4 100644 --- a/scripts/lib/xpedite/benchmark/info.py +++ b/scripts/lib/xpedite/benchmark/info.py @@ -73,7 +73,7 @@ def loadBenchmarkInfo(path): legend = configParser.get(BENCHMARK_SECTION, BENCHMARK_LEGEND_KEY) if not configParser.has_section(BENCHMARK_CPU_INFO_SECTION): - LOGGER.warn('failed to load benchmark %s - cpu info missing', benchmarkName) + LOGGER.warning('failed to load benchmark %s - cpu info missing', benchmarkName) return None cpuId = configParser.get(BENCHMARK_CPU_INFO_SECTION, BENCHMARK_CPU_ID_KEY) cpuFrequency = configParser.get(BENCHMARK_CPU_INFO_SECTION, BENCHMARK_CPU_FREQUENCY_KEY) diff --git a/scripts/lib/xpedite/jupyter/archive.py b/scripts/lib/xpedite/jupyter/archive.py index 9adc5e11..552da0db 100644 --- a/scripts/lib/xpedite/jupyter/archive.py +++ b/scripts/lib/xpedite/jupyter/archive.py @@ -93,7 +93,7 @@ def deflate(self, extractPath): """ if not extractPath: import tempfile - LOGGER.warn('No directory specified for extracting files, setting /tmp as notebook directory') + LOGGER.warning('No directory specified for extracting files, setting /tmp as notebook directory') extractPath = tempfile.mkdtemp(prefix=EXPORT_PREFIX, dir='/tmp') self.archive.extractall(path=extractPath) return extractPath diff --git a/scripts/lib/xpedite/jupyter/commands.py b/scripts/lib/xpedite/jupyter/commands.py index a21032e9..2bb682ae 100644 --- a/scripts/lib/xpedite/jupyter/commands.py +++ b/scripts/lib/xpedite/jupyter/commands.py @@ -241,7 +241,7 @@ def __repr__(self): ) if self.profile.current.isEventsEnabled() else '' threshold = 1000 if len(self.profile.current) > threshold: - LOGGER.warn('too many transaction - showing only %d out of %d', threshold, len(self.profile.current)) + LOGGER.warning('too many transaction - showing only %d out of %d', threshold, len(self.profile.current)) strRepr += str(ReportBuilder().buildTimelineTable( self.profile.current, self.profile.probes, ResultOrder.Chronological, threshold, uid )) diff --git a/scripts/lib/xpedite/jupyter/data/config/custom/custom.css b/scripts/lib/xpedite/jupyter/data/config/custom/custom.css index b94cc5ed..61ce2a7f 100644 --- a/scripts/lib/xpedite/jupyter/data/config/custom/custom.css +++ b/scripts/lib/xpedite/jupyter/data/config/custom/custom.css @@ -6,12 +6,12 @@ ** *******************************************************************************************/ -@import 'flot.css'; -@import 'notebook.css'; -@import 'sunburst.css'; -@import 'bipartite.css'; -@import 'xpedite.css'; -@import 'darkTheme.css'; +@import '/static/flot.css'; +@import '/static/notebook.css'; +@import '/static/sunburst.css'; +@import '/static/bipartite.css'; +@import '/static/xpedite.css'; +@import '/static/darkTheme.css'; body.notebook_app, body.notebook_app.command_mode { font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; diff --git a/scripts/lib/xpedite/jupyter/data/config/jupyter_server_config.py b/scripts/lib/xpedite/jupyter/data/config/jupyter_server_config.py new file mode 100644 index 00000000..a3bc8e0d --- /dev/null +++ b/scripts/lib/xpedite/jupyter/data/config/jupyter_server_config.py @@ -0,0 +1,18 @@ +# Configuration file for jupyter-notebook. + +# Add serverextension to sys.path for jupyter to load tornadoExtension module on startup +import os +import sys + +currDir = os.path.dirname(__file__) +packagePath = os.path.join(currDir, '../config/serverextensions') +sys.path.insert(0, packagePath) +c = get_config() +c.ServerApp.jpserver_extensions = { + 'tornadoExtension' : True, +} + +staticPath = os.path.join(currDir, '../config/custom') +c.ServerApp.extra_static_paths = [os.path.join(currDir, '../js'), staticPath] +c.ServerApp.allow_origin = '*' #allow all origins +c.ServerApp.ip = '0.0.0.0' # listen on all IPs diff --git a/scripts/lib/xpedite/jupyter/data/config/serverextensions/__init__.py b/scripts/lib/xpedite/jupyter/data/config/serverextensions/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/scripts/lib/xpedite/jupyter/data/config/serverextensions/tornadoExtension.py b/scripts/lib/xpedite/jupyter/data/config/serverextensions/tornadoExtension/__init__.py similarity index 70% rename from scripts/lib/xpedite/jupyter/data/config/serverextensions/tornadoExtension.py rename to scripts/lib/xpedite/jupyter/data/config/serverextensions/tornadoExtension/__init__.py index 902d6107..9cc3ea06 100644 --- a/scripts/lib/xpedite/jupyter/data/config/serverextensions/tornadoExtension.py +++ b/scripts/lib/xpedite/jupyter/data/config/serverextensions/tornadoExtension/__init__.py @@ -5,8 +5,10 @@ Author: Dhruv Shekhawat, Morgan Stanley """ -from notebook.utils import url_path_join -from notebook.base.handlers import IPythonHandler +#from notebook.utils import url_path_join +import jupyter_server +from jupyter_server.base.handlers import JupyterHandler +import tornado from tornado import template import tornado.web import json @@ -15,13 +17,15 @@ import os import sys -class HtmlReportHandler(tornado.web.RequestHandler): +class HtmlReportHandler(JupyterHandler): """Class to serve html reports through links with query params as notebook path and (cellId, reportId) as indices to read metadata from notebook """ + @tornado.web.authenticated def get(self): - xpeditePath = os.path.normpath(os.path.join(__file__, '../../../../../..')) + xpeditePath = os.path.normpath(os.path.join(__file__, '../../../../../../..')) + print('XXXXXXX Xpedite path ' + xpeditePath) sys.path.append(xpeditePath) from xpedite.jupyter.xpediteData import XpediteDataReader @@ -57,11 +61,16 @@ def get_init_cell(jsonReport): if(('isInit' in metadata) and (metadata['isInit'] == '0xFFFFFFFFA5A55A5DUL')): return cell -def load_jupyter_server_extension(nb_server_app): - """Method called first to load the - server extension on jupyter startup - """ - web_app = nb_server_app.web_app - host_pattern = '.*$' - route_pattern = url_path_join(web_app.settings['base_url'], '/xpedite') - web_app.add_handlers(host_pattern, [(route_pattern, HtmlReportHandler)]) +def _load_jupyter_server_extension(serverapp: jupyter_server.serverapp.ServerApp): + """ + This function is called when the extension is loaded. + """ + handlers = [('/xpedite', HtmlReportHandler)] + serverapp.web_app.add_handlers(".*$", handlers) + +def _jupyter_server_extension_points(): + """ + Returns a list of dictionaries with metadata describing + where to find the `_load_jupyter_server_extension` function. + """ + return [{"module": "tornadoExtension"}] diff --git a/scripts/lib/xpedite/jupyter/driver.py b/scripts/lib/xpedite/jupyter/driver.py index 4a22adb1..c5164eaa 100644 --- a/scripts/lib/xpedite/jupyter/driver.py +++ b/scripts/lib/xpedite/jupyter/driver.py @@ -138,7 +138,7 @@ def buildReportCells(nb, result, dataFilePath): nb['cells'].append( nbf.new_code_cell(source=cellCode, metadata={ - 'init_cell': True, 'hide_input': True, 'editable': False, 'deletable': True + 'init_cell': True, 'iinit' : True, 'hide_input': True, 'editable': True, 'deletable': True }) ) @@ -163,7 +163,7 @@ def buildInitCell(nb, numOfCategories, d3Flots, appName, runId): LOGGER.exception(typeErr) raise InvariantViloation(typeErr) - nb['cells'] = [nbf.new_code_cell(source=initCode, metadata={'init_cell': True, 'isInit': '0xFFFFFFFFA5A55A5DUL',\ + nb['cells'] = [nbf.new_code_cell(source=initCode, metadata={'init_cell': True, 'iinit' : True, 'isInit': '0xFFFFFFFFA5A55A5DUL',\ 'hide_input': True, 'editable': False, 'deletable': False,\ 'd3Flots': d3Flots})] + nb['cells'] @@ -217,7 +217,7 @@ def validatePath(homeDir, reportName): from xpedite.jupyter import DATA_DIR, DATA_FILE_EXT, TEMP_PREFIX, NOTEBOOK_EXT if homeDir is None: homeDir = tempfile.mkdtemp(prefix=TEMP_PREFIX, dir='/tmp') - LOGGER.warn('Xpedite home directory not found in profileInfo (using temp dir).\n' + LOGGER.warning('Xpedite home directory not found in profileInfo (using temp dir).\n' 'To keep all reports in one place, set variable homeDir in profileInfo to a valid path.') dataDir = os.path.join(homeDir, DATA_DIR) diff --git a/scripts/lib/xpedite/jupyter/templates/initCell.fmt b/scripts/lib/xpedite/jupyter/templates/initCell.fmt index c45522b7..916baa48 100644 --- a/scripts/lib/xpedite/jupyter/templates/initCell.fmt +++ b/scripts/lib/xpedite/jupyter/templates/initCell.fmt @@ -2,13 +2,13 @@ import os, sys from IPython.display import display, HTML sys.path.append(os.environ['XPEDITE_PATH']) import xpedite +import ipynbname from xpedite.jupyter.commands import routes, txns, plot, stat, filter, diff from xpedite.analytics.timelineTree import buildTimelineTree from xpedite.jupyter.templates.initCell import INTRO_FRMT from xpedite.jupyter.context import Context, context -thismodule = sys.modules[__name__] -notebookPath = getattr(thismodule, Context.notebookPathKey) +notebookPath = ipynbname.path() result = 'Failed to resolve jupyter notebook path' if notebookPath: context.initialize(notebookPath) diff --git a/scripts/lib/xpedite/profiler/app.py b/scripts/lib/xpedite/profiler/app.py index 5d6a1da9..345330a6 100644 --- a/scripts/lib/xpedite/profiler/app.py +++ b/scripts/lib/xpedite/profiler/app.py @@ -165,7 +165,7 @@ def pingApp(app): except socket.error as socketError: errMsg = 'encounter a socket error: {}'.format(str(socketError)) - LOGGER.warn('restarting xpedite client - application is not responding to ping - %s', errMsg) + LOGGER.warning('restarting xpedite client - application is not responding to ping - %s', errMsg) app.restart() try: @@ -189,7 +189,7 @@ def __init__(self, name, ip, appInfoPath, runId=None, dataSourcePath=None, works """Constructs an instance of XpediteDormantApp""" dataSource = CsvDataSourceFactory().gather(dataSourcePath) if dataSourcePath else None if dataSource: - LOGGER.warn('Data source detected. overriding appinfo to %s', dataSource.appInfoPath) + LOGGER.warning('Data source detected. overriding appinfo to %s', dataSource.appInfoPath) appInfoPath = dataSource.appInfoPath XpediteApp.__init__(self, name, ip, appInfoPath, dryRun=True, workspace=workspace) self.dataSource = dataSource diff --git a/scripts/lib/xpedite/profiler/environment.py b/scripts/lib/xpedite/profiler/environment.py index e265696c..b565ba2c 100644 --- a/scripts/lib/xpedite/profiler/environment.py +++ b/scripts/lib/xpedite/profiler/environment.py @@ -229,7 +229,7 @@ def enablePMU(self, eventsDb, cpuSet, events): if not self.isDriverLoaded(): (eventSet, request) = PMUCtrl.buildPerfEventsRequest(eventsDb, events) if eventSet and request: - LOGGER.warn('xpedite device driver not loaded - falling back to perf events api') + LOGGER.warning('xpedite device driver not loaded - falling back to perf events api') LOGGER.debug('sending request (%d bytes) to xpedite [%s]', len(request), request) rc = self.admin('ActivatePerfEvents --data {}'.format(request)) if rc: diff --git a/scripts/lib/xpedite/profiler/profileInfo.py b/scripts/lib/xpedite/profiler/profileInfo.py index 3c21d2e6..a97404de 100644 --- a/scripts/lib/xpedite/profiler/profileInfo.py +++ b/scripts/lib/xpedite/profiler/profileInfo.py @@ -9,7 +9,7 @@ import os import sys -import imp #pylint: disable=deprecated-module +import importlib.util import logging LOGGER = logging.getLogger(__name__) @@ -74,7 +74,9 @@ def loadProfileInfo(profilePath): path = os.path.abspath(profilePath) fileName = os.path.split(profilePath)[1] moduleName = str.split(fileName, '.')[0] - profileInfo = imp.load_source(moduleName, path) + spec = importlib.util.spec_from_file_location(moduleName, path) + profileInfo = importlib.util.module_from_spec(spec) + spec.loader.exec_module(profileInfo) benchmarkPaths = getattr(profileInfo, 'benchmarkPaths', None) pmc = getattr(profileInfo, 'pmc', None) cpuSet = getattr(profileInfo, 'cpuSet', None) diff --git a/scripts/lib/xpedite/profiler/runtime.py b/scripts/lib/xpedite/profiler/runtime.py index 06a6819f..58727f9f 100644 --- a/scripts/lib/xpedite/profiler/runtime.py +++ b/scripts/lib/xpedite/profiler/runtime.py @@ -82,7 +82,7 @@ def enableProbes(self, probes): LOGGER.error(msg) raise Exception(msg) else: - LOGGER.warn('failed to enable probes - Invalid or empty probes argument') + LOGGER.warning('failed to enable probes - Invalid or empty probes argument') def resolveProbes(self, probes): """ @@ -224,7 +224,7 @@ def __init__(self, app, probes, pmc=None, cpuSet=None, pollInterval=4, samplesFi else: if pmc: self.eventSet = self.resolveEvents(eventsDb, cpuSet, pmc) - LOGGER.warn('DRY Run selected - xpedite won\'t enable probes') + LOGGER.warning('DRY Run selected - xpedite won\'t enable probes') except Exception as ex: LOGGER.exception('failed to start profiling') raise ex @@ -263,7 +263,7 @@ def report(self, reportName=None, benchmarkPaths=None, classifier=DefaultClassif try: self.app.endProfile() except Exception as ex: - LOGGER.warn('Detected unclean profile termination - %s', ex) + LOGGER.warning('Detected unclean profile termination - %s', ex) if self.eventSet: self.app.disablePMU() diff --git a/scripts/lib/xpedite/requirements.txt b/scripts/lib/xpedite/requirements.txt index 3f313fd7..6107fd25 100644 --- a/scripts/lib/xpedite/requirements.txt +++ b/scripts/lib/xpedite/requirements.txt @@ -5,8 +5,11 @@ netifaces>=0.10.4 numpy>=1.15.2 pygments>=2.0.2 rpyc>=4.0.1 -cement==2.8.2 +cement>=3.0.10 termcolor>=1.1.0 py-cpuinfo>=0.1.2 jupyter>=1.0.0 six>=1.12.0 +ipynbname +setuptools +iinit diff --git a/scripts/lib/xpedite/txn/extractor.py b/scripts/lib/xpedite/txn/extractor.py index 3a6bdbb9..47fa6449 100644 --- a/scripts/lib/xpedite/txn/extractor.py +++ b/scripts/lib/xpedite/txn/extractor.py @@ -61,10 +61,10 @@ def gatherCounters(self, app, loader): elapsed = time.time() - begin self.logCounterFilterReport() if self.orphanedSamplesCount: - LOGGER.warn('detected mismatch in binary vs app info - %d counters ignored', self.orphanedSamplesCount) + LOGGER.warning('detected mismatch in binary vs app info - %d counters ignored', self.orphanedSamplesCount) LOGGER.completed('%d records | %d txns loaded in %0.2f sec.', recordCount-1, loader.getCount(), elapsed) if loader.isCompromised() or loader.getTxnCount() <= 0: - LOGGER.warn(loader.report()) + LOGGER.warning(loader.report()) elif loader.isNotAccounted(): LOGGER.debug(loader.report()) loader.endCollection() diff --git a/scripts/lib/xpedite/txn/repo.py b/scripts/lib/xpedite/txn/repo.py index 840359a2..d2d4cad2 100644 --- a/scripts/lib/xpedite/txn/repo.py +++ b/scripts/lib/xpedite/txn/repo.py @@ -96,7 +96,7 @@ def loaderFactory(loaderType, benchmark, probes, benchmarkProbes, topdownCache, benchmarkTopdownMetrics = None if benchmarkProbes and benchmark.name in benchmarkProbes: loaderProbes = benchmarkProbes[benchmark.name] - LOGGER.warn('overriding probes for benchmark run \'%s\'', benchmark.name) + LOGGER.warning('overriding probes for benchmark run \'%s\'', benchmark.name) if benchmark.events and topdownMetrics: from xpedite.pmu.event import TopdownMetrics benchmarkTopdownMetrics = TopdownMetrics() diff --git a/scripts/lib/xpedite/util/__init__.py b/scripts/lib/xpedite/util/__init__.py index dbd99234..3e006cae 100644 --- a/scripts/lib/xpedite/util/__init__.py +++ b/scripts/lib/xpedite/util/__init__.py @@ -40,7 +40,7 @@ def timeAction(action, delegate): retVal = delegate() elapsed = time.time() - begin if elapsed > 10: - LOGGER.warn('timed action exceeded threshold %s completed in %s.1f seconds', action, elapsed) + LOGGER.warning('timed action exceeded threshold %s completed in %s.1f seconds', action, elapsed) return retVal def shell(cmd, cwd=None, closeFds=True): diff --git a/test/runTest.sh b/test/runTest.sh index 73598900..82818dd0 100755 --- a/test/runTest.sh +++ b/test/runTest.sh @@ -36,6 +36,8 @@ usage: ${PROGRAM_NAME} [lgpw:cr:s:Pt:m:a:] -c|--cov check pytest code coverage -r|--remote set a remote hostname for the application to run on: ${PROGRAM_NAME} -r -s|--single choose a single test to run: ${PROGRAM_NAME} -s test_name +-L|--list List pytest +-k|--pattern Run pytest matching pattern -t|--transactions specify a number of transactions for the target application: ${PROGRAM_NAME} -t -m|--multithreaded specify the number of threads for the target application: ${PROGRAM_NAME} -m -a|--apps a comma separated list of binaries to test: ${PROGRAM_NAME} -a @@ -103,6 +105,10 @@ function doesDirectoryExist() { fi } +function listPytests() { + PYTHONPATH=${XPEDITE_DIR}:${PYTHONPATH} pytest ${PYTEST_DIR} --collect-only +} + function runPytests() { RUN_DIR=$(mktemp -d) @@ -128,8 +134,10 @@ function runPytests() { SCENARIO_TYPES="--scenarioTypes=Regular,Benchmark,PMC" fi - PYTEST_ARGS="${COV} ${TEST_NAME} -v ${APP_HOST} ${TRANSACTION_COUNT} ${THREAD_COUNT} ${WORKSPACE} ${RUN_DIR_ARG} ${APPS} ${SCENARIO_TYPES} ${RECORD_PMC}" - if ! PYTHONPATH=${XPEDITE_DIR}:${PYTHONPATH} pytest ${PYTEST_ARGS}; then + set -x + PYTEST_ARGS="${COV} ${TEST_NAME} ${TEST_PATTERN} -v ${APP_HOST} ${PYTEST_ARGS} ${TRANSACTION_COUNT} ${THREAD_COUNT}" + PYTEST_ARGS="${PYTEST_ARGS} ${WORKSPACE} ${RUN_DIR_ARG} ${APPS} ${SCENARIO_TYPES} ${RECORD_PMC}" + if ! PYTHONPATH=${XPEDITE_DIR}:${PYTHONPATH} FORCE_COLOR=true pytest ${PYTEST_ARGS}; then echo detected one or more pytest failures RC=$(($RC + 1)) fi @@ -173,7 +181,7 @@ function runAllTests() { runPytests } -ARGS=$(getopt -o lgpw:cr:s:t:m:a:S:P --long lint,gtest,pytest,workspace,cov,remote:,test:,transactions:,multithreaded:,apps:,scenarioTypes:recordPMC -- "$@") +ARGS=$(getopt -o lLgpw:cr:s:k:t:m:a:S:P --long lint,list,gtest,pytest,single,pattern,workspace,cov,remote:,test:,transactions:,multithreaded:,apps:,scenarioTypes:recordPMC -- "$@") if [ $? -ne 0 ]; then usage @@ -213,6 +221,15 @@ while true ; do echo ${TEST_NAME} shift 2 ;; + -k|--pattern) + TEST_PATTERN="-k $2" ; + echo 'Test pattern *******' ${TEST_PATTERN} + shift 2 + ;; + -L|--list) + LIST_PYTEST=true + shift + ;; -t|--transactions) TRANSACTION_COUNT="--transactions=$2" shift 2 @@ -252,7 +269,7 @@ if [ -d ${RUNTIME_DIR}/bin ]; then python -m pip --trusted-host pypi.org --trusted-host files.pythonhosted.org install pytest pylint pytest-cov fi -if [[ -z "${LINT}" && -z "${GTEST}" && -z "${PYTEST}" ]]; then +if [[ -z "${LINT}" && -z "${GTEST}" && -z "${PYTEST}" && -z "${LIST_PYTEST}" && -z "${TEST_PATTERN}" ]]; then runAllTests else if [ -z "${PYTEST}" ] && [[ "${APP_HOST}" || "${TRANSACTION_COUNT}" || "${THREAD_COUNT}" || "${WORKSPACE}" || "${COV}" || "${TEST_NAME}" || "${APPS}" || "${SCENARIO_TYPES}" || "${RECORD_PMC}" ]]; then @@ -267,9 +284,13 @@ else runLint fi - if [ "${PYTEST}" = true ]; then + if [ "${PYTEST}" = true -o -n "${TEST_PATTERN}" ]; then runPytests fi + + if [ "${LIST_PYTEST}" = true ]; then + listPytests + fi fi exit $RC