Major dependency upgrade python to 3.x (deco 2.x) with latest jupyter

compatibility

.github/workflows/main.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [2.7, 3.8]
+        python-version: [3.8]
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:

CMakeLists.txt

@@ -58,6 +58,7 @@ install(TARGETS xpedite-pic DESTINATION "lib" COMPONENT libraries)
 
 ######################### Python bindings #############################
 
+set(PYBIND11_PYTHON_VERSION 3.8 CACHE STRING "")
 find_package(pybind11 REQUIRED)
 pybind11_add_module(xpediteBindings lib/xpedite/pybind/Bindings.cpp lib/xpedite/framework/SamplesLoader.C)
 install(TARGETS xpediteBindings DESTINATION "lib" COMPONENT libraries)

build.sh

@@ -33,7 +33,7 @@ BUILD_TYPE=Release
 BUILD_VIVIFY=0
 BUILD_JAVA=0
 VERBOSE=0
-PYTHON_VERSION=2.7
+PYTHON_VERSION=3.8
 
 while true ; do
   case "$1" in

demo/Demo.H

@@ -30,9 +30,6 @@ namespace xpedite { namespace demo {
   using Indices = std::array<uint64_t, indexSize>;
 
   inline void initialize(size_t core_ = 0) {
-    if(mlockall(MCL_CURRENT | MCL_FUTURE)) {
-      std::cerr << "failed to mlock all pages" << std::endl;
-    }
     using namespace xpedite::framework;
     if(!xpedite::framework::initialize("xpedite-appinfo.txt", {AWAIT_PROFILE_BEGIN})) { 
       throw std::runtime_error {"failed to init xpedite"}; 

include/xpedite/probes/CallSite.H

@@ -12,6 +12,7 @@
 
 #pragma once
 #include <sstream>
+#include <cstdint>
 
 namespace xpedite { namespace probes {
 

include/xpedite/probes/ProbeKey.H

@@ -8,6 +8,7 @@
 
 #pragma once
 #include <string>
+#include <cstdint>
 
 namespace xpedite { namespace probes {
 

install.sh

@@ -31,7 +31,7 @@ fi
 eval set -- "$ARGS"
 ENABLE_PMU=0
 VERBOSE=0
-PYTHON_VERSION=2
+PYTHON_VERSION=3
 
 while true ; do
   case "$1" in

jni/build.gradle

@@ -25,7 +25,7 @@ sourceSets {
 }
 
 task xpediteJar(type: Jar) {
-  baseName = 'xpedite'
+  archiveBaseName = 'xpedite'
   exclude(['**/demo/**'])
   manifest {
     attributes(
@@ -42,7 +42,7 @@ task xpediteJar(type: Jar) {
 task demoJar(type: Jar) {
   include(['**/demo/**'])
   exclude(['**/com/xpedite/probes'])
-  baseName = 'xpediteDemo'
+  archiveBaseName = 'xpediteDemo'
   manifest {
     attributes(
       'Implementation-Title': 'Xpedite Demo Jar',

lib/xpedite/framework/StorageMgr.H

@@ -11,6 +11,7 @@
 
 #pragma once
 #include <string>
+#include <cstdint>
 
 namespace xpedite { namespace framework {
 

scripts/bin/xpedite

@@ -26,8 +26,8 @@ logger.init()
 
 from xpedite.dependencies               import Package, DEPENDENCY_LOADER
 DEPENDENCY_LOADER.load(Package.Cement)
-from cement.core.foundation             import CementApp
-from cement.ext.ext_argparse            import ArgparseController, expose
+from cement import App, CaughtSignal, Controller, ex
+
 import logging
 import xpedite
 
@@ -54,15 +54,16 @@ def _loadDriver(name = None):
     return Driver()
   return driver
 
-class AppController(ArgparseController):
+
+class AppController(Controller):
   class Meta:
     label = 'base'
 
-  @expose(hide=True)
+  @ex(hide=True)
   def default(self):
     self.probes()
 
-  @expose(
+  @ex(
     arguments=[
       (['-p', '--profileInfo'], dict(action='store', required=True, help=profileInfoHelp)),
       (['-b', '--createBenchmark'], dict(action='store', help='store Live profile report at the given path, for future benchmarking')),
@@ -94,7 +95,7 @@ class AppController(ArgparseController):
     driver = _loadDriver(self.app.pargs.driver)
     driver.render(profileInfo, report, leanReports = self.app.pargs.lean, cprofile = cprofile)
 
-  @expose(
+  @ex(
     arguments=[
       (['-p', '--profileInfo'], dict(action='store', required=True, help=profileInfoHelp)),
     ],
@@ -113,7 +114,7 @@ class AppController(ArgparseController):
       LOGGER.error('failed to locate probes in app %s. Have you instrumented any ?', profileInfo.appName)
     LOGGER.info('')
 
-  @expose(
+  @ex(
     arguments=[
       (['-H', '--hostname'], dict(action='store', default='127.0.0.1', help='host where the app runs')),
       (['-a', '--appInfo'], dict(action='store', required=True, help='path to appInfo file, used in xpedite framework initilization')),
@@ -124,7 +125,7 @@ class AppController(ArgparseController):
     from xpedite.profiler import Profiler
     Profiler.generate(self.app.pargs.appInfo, hostname=self.app.pargs.hostname)
 
-  @expose(
+  @ex(
     arguments=[
       (['-H', '--home'], dict(action='store', required=False, help='Home dir for xpedite shell')),
       (['-z', '--zip'], dict(action='store', required=False, help='Archive a xpedite report. Takes path of a xpedite notebook as argument')),
@@ -141,21 +142,21 @@ class AppController(ArgparseController):
         return
     home = self.app.pargs.home
     if not home:
-      LOGGER.warn('No home directory specified for shell - using current working directory')
+      LOGGER.warning('No home directory specified for shell - using current working directory')
       home = os.getcwd()
 
     if self.app.pargs.unzip:
       with Deflator(self.app.pargs.unzip) as deflator:
         deflator.deflate(home)
     launchJupyter(home)
 
-class FrameworkController(ArgparseController):
+class FrameworkController(Controller):
   class Meta:
     label = 'FrameworkController'
     stacked_on = 'base'
     stacked_type = 'embedded'
 
-  @expose(
+  @ex(
     arguments=[
       (['-p', '--profileInfo'], dict(action='store', required=True, help=profileInfoHelp)),
       (['-b', '--createBenchmark'], dict(action='store', help='store report at the given path, for future benchmarking')),
@@ -191,7 +192,7 @@ class FrameworkController(ArgparseController):
     driver = _loadDriver(self.app.pargs.driver)
     driver.render(profileInfo, report, leanReports = self.app.pargs.lean, cprofile = cprofile)
 
-  @expose(
+  @ex(
     arguments=[
     ],
     help='Print the CPU info for the current machine'
@@ -201,7 +202,7 @@ class FrameworkController(ArgparseController):
     cpuId = getCpuId()
     LOGGER.info('%s\n', cpuId)
 
-  @expose(
+  @ex(
     arguments=[
       (['-c', '--cpuId'], dict(action='store', required=False, help='list pmu counters for the give cpu identifier')),
     ],
@@ -213,7 +214,7 @@ class FrameworkController(ArgparseController):
       LOGGER.info(e)
     LOGGER.info('')
 
-  @expose(
+  @ex(
     arguments=[
       (['-n', '--node'], dict(action='store', help='list pmu events for node - enter "all" to list events for all nodes')),
       (['-c', '--cpuId'], dict(action='store', required=False, help='list pmu counters for the give cpu identifier')),
@@ -227,7 +228,7 @@ class FrameworkController(ArgparseController):
       LOGGER.info('%s', topdown.metricsToString(n))
     LOGGER.info('')
 
-  @expose(
+  @ex(
     arguments=[
       (['-n', '--node'], dict(action='store', help='list pmu events for node - enter "all" to list events for all nodes')),
       (['-c', '--cpuId'], dict(action='store', required=False, help='list pmu counters for the give cpu identifier')),
@@ -244,7 +245,7 @@ class FrameworkController(ArgparseController):
       return
     LOGGER.info('%s\n', topdown.hierarchy)
 
-  @expose(
+  @ex(
     arguments=[
       (['-e', '--enable'], dict(action='store_true', help='load xpedite kernel module to ENABLE hardware performance counters')),
       (['-d', '--disable'], dict(action='store_true', help='unload xpedite kernel module to DISABLE hardware performance counters')),
@@ -264,7 +265,7 @@ class FrameworkController(ArgparseController):
       extractor = subprocess.call(cmdList)
     LOGGER.info('pmc status - %s\n', 'enabled' if isDriverLoaded() else 'disabled')
 
-class XpediteClient(CementApp):
+class XpediteClient(App):
   class Meta:
     label = 'Xpedite'
     base_controller = 'base'

scripts/lib/setup.py

@@ -65,5 +65,7 @@
         'py-cpuinfo',
         'jupyter',
         'six',
+        'ipynbname',
+        'iinit',
       ],
       zip_safe=False)

scripts/lib/xpedite/analytics/__init__.py

@@ -60,7 +60,7 @@ def buildElapsedTimeBundles(txnCollections, classifier):
             else:
               if category not in categorySet:
                 categorySet.add(category)
-                LOGGER.warn('current run missing trasactions for category "%s"', category)
+                LOGGER.warning('current run missing trasactions for category "%s"', category)
       else:
         scopeList = ', '.join([probe.getCanonicalName() for probe in probes if not probe.isAnonymous])
         errMsg = (

scripts/lib/xpedite/benchmark/__init__.py

@@ -102,14 +102,14 @@ def gatherBenchmarks(self, count):
             benchmark.dataSource = dataSource
             benchmarks.append(benchmark)
         else:
-          LOGGER.warn('skip processing benchmark %s. failed to load benchmark info', path)
+          LOGGER.warning('skip processing benchmark %s. failed to load benchmark info', path)
 
         if len(benchmarks) >= count:
           if i + 1 < len(self.benchmarkPaths):
             LOGGER.debug('skip processing %s benchmarks. limit reached.', self.benchmarkPaths[i+1:])
           break
       else:
-        LOGGER.warn('skip processing benchmark %s. failed to locate benchmark files', path)
+        LOGGER.warning('skip processing benchmark %s. failed to locate benchmark files', path)
     return benchmarks
 
   @staticmethod

scripts/lib/xpedite/benchmark/info.py

@@ -73,7 +73,7 @@ def loadBenchmarkInfo(path):
     legend = configParser.get(BENCHMARK_SECTION, BENCHMARK_LEGEND_KEY)
 
     if not configParser.has_section(BENCHMARK_CPU_INFO_SECTION):
-      LOGGER.warn('failed to load benchmark %s - cpu info missing', benchmarkName)
+      LOGGER.warning('failed to load benchmark %s - cpu info missing', benchmarkName)
       return None
     cpuId = configParser.get(BENCHMARK_CPU_INFO_SECTION, BENCHMARK_CPU_ID_KEY)
     cpuFrequency = configParser.get(BENCHMARK_CPU_INFO_SECTION, BENCHMARK_CPU_FREQUENCY_KEY)

scripts/lib/xpedite/jupyter/archive.py

@@ -93,7 +93,7 @@ def deflate(self, extractPath):
     """
     if not extractPath:
       import tempfile
-      LOGGER.warn('No directory specified for extracting files, setting /tmp as notebook directory')
+      LOGGER.warning('No directory specified for extracting files, setting /tmp as notebook directory')
       extractPath = tempfile.mkdtemp(prefix=EXPORT_PREFIX, dir='/tmp')
     self.archive.extractall(path=extractPath)
     return extractPath

scripts/lib/xpedite/jupyter/commands.py

@@ -241,7 +241,7 @@ def __repr__(self):
       ) if self.profile.current.isEventsEnabled() else ''
       threshold = 1000
       if len(self.profile.current) > threshold:
-        LOGGER.warn('too many transaction - showing only %d out of %d', threshold, len(self.profile.current))
+        LOGGER.warning('too many transaction - showing only %d out of %d', threshold, len(self.profile.current))
       strRepr += str(ReportBuilder().buildTimelineTable(
         self.profile.current, self.profile.probes, ResultOrder.Chronological, threshold, uid
       ))

scripts/lib/xpedite/jupyter/data/config/custom/custom.css

@@ -6,12 +6,12 @@
 **
 *******************************************************************************************/
 
-@import 'flot.css';
-@import 'notebook.css';
-@import 'sunburst.css';
-@import 'bipartite.css';
-@import 'xpedite.css';
-@import 'darkTheme.css';
+@import '/static/flot.css';
+@import '/static/notebook.css';
+@import '/static/sunburst.css';
+@import '/static/bipartite.css';
+@import '/static/xpedite.css';
+@import '/static/darkTheme.css';
 
 body.notebook_app, body.notebook_app.command_mode {
   font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;

scripts/lib/xpedite/jupyter/data/config/jupyter_server_config.py

@@ -0,0 +1,18 @@
+# Configuration file for jupyter-notebook.
+
+# Add serverextension to sys.path for jupyter to load tornadoExtension module on startup
+import os
+import sys
+
+currDir = os.path.dirname(__file__)
+packagePath = os.path.join(currDir, '../config/serverextensions')
+sys.path.insert(0, packagePath)
+c = get_config()
+c.ServerApp.jpserver_extensions = {
+    'tornadoExtension' : True,
+}
+
+staticPath = os.path.join(currDir, '../config/custom')
+c.ServerApp.extra_static_paths = [os.path.join(currDir, '../js'), staticPath]
+c.ServerApp.allow_origin = '*'  #allow all origins
+c.ServerApp.ip = '0.0.0.0'      # listen on all IPs

scripts/lib/xpedite/jupyter/data/config/serverextensions/tornadoExtension.py → scripts/lib/xpedite/jupyter/data/config/serverextensions/tornadoExtension/__init__.py

@@ -5,8 +5,10 @@
 Author: Dhruv Shekhawat, Morgan Stanley
 """
 
-from notebook.utils          import url_path_join
-from notebook.base.handlers  import IPythonHandler
+#from notebook.utils          import url_path_join
+import jupyter_server
+from jupyter_server.base.handlers import JupyterHandler
+import tornado
 from tornado                 import template
 import tornado.web
 import json
@@ -15,13 +17,15 @@
 import os
 import sys
 
-class HtmlReportHandler(tornado.web.RequestHandler):
+class HtmlReportHandler(JupyterHandler):
   """Class to serve html reports through links with
   query params as notebook path and (cellId, reportId)
   as indices to read metadata from notebook
   """
+  @tornado.web.authenticated
   def get(self):
-    xpeditePath = os.path.normpath(os.path.join(__file__, '../../../../../..'))
+    xpeditePath = os.path.normpath(os.path.join(__file__, '../../../../../../..'))
+    print('XXXXXXX Xpedite path ' + xpeditePath)
     sys.path.append(xpeditePath)
 
     from xpedite.jupyter.xpediteData import XpediteDataReader
@@ -57,11 +61,16 @@ def get_init_cell(jsonReport):
     if(('isInit' in metadata) and (metadata['isInit'] == '0xFFFFFFFFA5A55A5DUL')):
       return cell
 
-def load_jupyter_server_extension(nb_server_app):
-  """Method called first to load the
-  server extension on jupyter startup
-  """
-  web_app = nb_server_app.web_app
-  host_pattern = '.*$'
-  route_pattern = url_path_join(web_app.settings['base_url'], '/xpedite')
-  web_app.add_handlers(host_pattern, [(route_pattern, HtmlReportHandler)])
+def _load_jupyter_server_extension(serverapp: jupyter_server.serverapp.ServerApp):
+    """
+    This function is called when the extension is loaded.
+    """
+    handlers = [('/xpedite', HtmlReportHandler)]
+    serverapp.web_app.add_handlers(".*$", handlers)
+
+def _jupyter_server_extension_points():
+    """
+    Returns a list of dictionaries with metadata describing
+    where to find the `_load_jupyter_server_extension` function.
+    """
+    return [{"module": "tornadoExtension"}]