General code cleanup part #1

.gitignore

@@ -91,4 +91,8 @@ ENV/
 # Rope project settings
 .ropeproject
 
-README.rst
+# Generated README
+README.rst
+
+# Extra development notebooks
+Untitled*.ipynb

.travis.yml

@@ -21,8 +21,7 @@ install:
   - python -m spylon_kernel install --user
 
 script:
-  - coverage run run_tests.py -vrsx --capture=sys --color=yes
-  - coverage report -m
+  - python run_tests.py -vxrs --capture=sys --color=yes
   # Ensure installability
   - python setup.py sdist
   - pip install --no-use-wheel dist/*.tar.gz

Makefile

@@ -37,4 +37,4 @@ release: clean ## Make a pypi release of a tagged build
 	$(SA) $(ENV) && python setup.py sdist register upload
 
 test: ## Make a test run
-	$(SA) $(ENV) && coverage run run_tests.py -vrsx --capture=sys --color=yes
+	$(SA) $(ENV) && python run_tests.py -vxrs --capture=sys --color=yes

run_tests.py

@@ -1,8 +1,22 @@
 #!/usr/bin/env python
-import sys
-import pytest
-
 if __name__ == '__main__':
-    # call pytest and exit with the return code from pytest so that
-    # travis will fail correctly if tests fail
-    sys.exit(pytest.main(sys.argv[1:]))
+    import coverage
+    cov = coverage.Coverage()
+    cov.start()
+
+    # Import required modules after coverage starts
+    import sys
+    import pytest
+
+    # Call pytest and exit with the return code from pytest so that
+    # CI systems will fail if tests fail.
+    ret = pytest.main(sys.argv[1:])
+
+    cov.stop()
+    cov.save()
+    # Save HTML coverage report to disk
+    cov.html_report()
+    # Emit coverage report to stdout
+    cov.report()
+
+    sys.exit(ret)

spylon_kernel/__main__.py

@@ -1,16 +1,7 @@
-from tornado.ioloop import IOLoop
-
+"""Entrypoint for running the kernel process."""
 from spylon_kernel import SpylonKernel
-import sys
+from tornado.ioloop import IOLoop
 
 if __name__ == '__main__':
-
-    # For testing purposes we want to be able to run our kernel with coverage on.
-    try:
-        import coverage
-        coverage.process_startup()
-    except ImportError:
-        pass
-
     IOLoop.configure("tornado.platform.asyncio.AsyncIOLoop")
     SpylonKernel.run_as_main()

spylon_kernel/init_spark_magic.py

@@ -1,7 +1,9 @@
+"""Metakernel magic for configuring and automatically initializing a Spark session."""
 import logging
 import spylon.spark
+
 from metakernel import Magic
-from spylon_kernel.scala_interpreter import init_spark_session
+from .scala_interpreter import init_spark
 
 try:
     import jedi
@@ -12,42 +14,60 @@
 
 
 class InitSparkMagic(Magic):
+    """Cell magic that supports configuration property autocompletion and
+    initializes a Spark session.
 
+    Attributes
+    ----------
+    env : __builtins__
+        Copy of the Python builtins
+    log : logging.Logger
+        Logger for this instance
+    """
     def __init__(self, kernel):
         super(InitSparkMagic, self).__init__(kernel)
         self.env = globals()['__builtins__'].copy()
         self.env['application_name'] = None
         self.env['launcher'] = spylon.spark.launcher.SparkConfiguration()
-        self.log = logging.Logger("InitSparkMagic")
+        self.log = logging.Logger(self.__class__.__name__)
 
     def cell_init_spark(self):
-        """
-        %%init_spark - start up a spark context with a custom configuration
+        """Starts a SparkContext with a custom configuration defined
+        using Python code.
 
-        Example:
-            %%init_spark
-            application_name = 'My Fancy App'
-            launcher.jars = ["file://some/jar.jar"]
-            launcher.master = "local[4]"
-            launcher.conf.spark.executor.cores = 8
+        Includes a `spylon.spark.launcher.SparkConfiguration` instance
+        in the variable `launcher`. Looks for an `application_name`
+        variable to use as the name of the Spark session.
 
-        This will evaluate the launcher args using spylon.
+        Example
+        -------
+        %%init_spark
+        application_name = "My Fancy App"
+        launcher.jars = ["file://some/jar.jar"]
+        launcher.master = "local[4]"
+        launcher.conf.spark.executor.cores = 8
         """
-        if "__builtins__" not in self.env:
-            # __builtins__ get generated after an eval:
-            eval("1", self.env)
-
-        globals_dict = self.env
-        exec(self.code, globals_dict)
-        application_name = globals_dict['application_name']
-        conf = globals_dict['launcher']
-        init_spark_session(conf, application_name=application_name)
+        # Evaluate the cell contents as Python
+        exec(self.code, self.env)
+        # Use the launcher and application_name as arguments to spylon to
+        # initialize a spark session
+        init_spark(conf=self.env['launcher'],
+                   application_name=self.env['application_name'])
+        # Do not evaluate the cell contents using the kernel
         self.evaluate = False
-        self.kernel.Display()
 
     def get_completions(self, info):
-        """Get Python completions."""
-        # https://github.com/davidhalter/jedi/blob/master/jedi/utils.py
+        """Gets Python completions based on the current cursor position
+        within the %%init_spark cell.
+
+        Based on
+        https://github.com/Calysto/metakernel/blob/master/metakernel/magics/python_magic.py
+
+        Parameters
+        ----------
+        info : dict
+            Information about the current caret position
+        """
         if jedi is None:
             return []
 
@@ -65,8 +85,4 @@ def get_completions(self, info):
         before = text[:len(text) - len(name)]
         completions = interpreter.completions()
         completions = [before + c.name_with_symbols for c in completions]
-        return [c[info['start']:] for c in completions]
-
-
-def register_magics(kernel):
-    kernel.register_magics(InitSparkMagic)
+        return [c[info['start']:] for c in completions]