[Draft] add stylechecker

client/api/assignment.py

@@ -195,7 +195,8 @@ def _load_tests(self):
                 for file in sorted(glob.glob(file_pattern)):
                     try:
                         module = importlib.import_module(self._TESTS_PACKAGE + '.' + source)
-                    except ImportError:
+                    except ImportError as e:
+                        print(e)
                         raise ex.LoadingException('Invalid test source: {}'.format(source))
 
                     test_name = file

client/cli/publish.py

@@ -14,7 +14,8 @@
 EXTRA_PACKAGES = [
     'requests', 'certifi', 'urllib3', 'chardet', 'idna', # requests/certifi and recursive deps
     'coverage', # coverage and recursive deps
-    'pytutor', 'ast_scope', 'attr' # pytutor and recursive deps
+    'pytutor', 'ast_scope', 'attr', # pytutor and recursive deps
+    'pycodestyle.py', # pycodestyle and recursive deps
 ]
 
 def abort(message):

client/sources/lint_test/__init__.py

@@ -0,0 +1,24 @@
+from client import exceptions as ex
+from client.sources.common import importing
+from client.sources.lint_test import models
+import logging
+import os
+import traceback
+
+log = logging.getLogger(__name__)
+
+def load(file, name, assign):
+    """Loads doctests from a specified filepath.
+
+    PARAMETERS:
+    file -- str; a filepath to a Python module containing OK-style
+            tests.
+    name -- str; optional parameter that specifies a particular function in
+            the file. If omitted, all doctests will be included.
+
+    RETURNS:
+    Test
+    """
+    if not os.path.isfile(file) or not file.endswith('.py'):
+        raise ex.LoadingException('Cannot run python linter on {}'.format(file))
+    return {name + "_lint" : models.LinterTest(file, name=name + "_lint", points=1)}

client/sources/lint_test/models.py

@@ -0,0 +1,64 @@
+from client import exceptions as ex
+from client.sources.common import core
+from client.sources.common import importing
+from client.sources.common import interpreter
+from client.sources.common import models
+from client.sources.common import pyconsole
+from client.utils import format
+from client.utils import output
+import re
+import textwrap
+
+import pycodestyle
+
+##########
+# Models #
+##########
+
+class LinterTest(models.Test):
+
+    def __init__(self, file, **fields):
+        super().__init__(**fields)
+        self.file = file
+        self.styleguide = None
+
+    def post_instantiation(self):
+        self.styleguide = pycodestyle.StyleGuide()
+
+    def _lint(self):
+        return self.styleguide.check_files([self.file])
+
+    def run(self, env):
+        """Runs the suites associated with this doctest.
+
+        NOTE: env is intended only for use with the programmatic API to support
+        Python OK tests. It is not used here.
+
+        RETURNS:
+        bool; True if the doctest completely passes, False otherwise.
+        """
+        result = self._lint()
+        if result.total_errors == 0:
+            return {'passed': 1, 'failed': 0, 'locked': 0}
+        else:
+            return {'passed': 0, 'failed': 1, 'locked': 0}
+
+    def score(self):
+        format.print_line('-')
+        print('Lint tests for {}'.format(self.file))
+        print()
+        success = self._lint().total_errors == 0
+        score = 1.0 if success else 0.0
+
+        print('Score: {}/1'.format(score))
+        print()
+        return score
+
+    def unlock(self, interact):
+        """Lint tests cannot be unlocked."""
+
+    def lock(self, hash_fn):
+        """Lint tests cannot be locked."""
+
+    def dump(self):
+        """Lint tests do not need to be dumped, since no state changes."""

requirements.txt

@@ -11,7 +11,8 @@ coverage==4.4
 pytutor==1.0.0
 ast-scope==0.3.1
 attrs==19.3.0
-
+## Linting
+pycodestyle==2.5.0
 
 # Tests
 nose==1.3.3

setup.py

@@ -46,6 +46,7 @@
         'coverage==4.4',
         'pytutor==1.0.0',
         'ast-scope==0.3.1',
-        'attrs==19.3.0'
+        'attrs==19.3.0',
+        'pycodestyle==2.5.0'
     ],
 )

tests/api/assignment_test.py

@@ -416,7 +416,7 @@ class AssignmentGradeTest(unittest.TestCase):
         "name": "Homework 1",
         "endpoint": "",
         "tests": {
-            "q1.py": "ok_test"
+            "q1.py": "ok_test,lint_test"
         },
         "protocols": []
     }

tests/end_to_end/smoke_test.py

@@ -7,6 +7,47 @@
 import shlex
 import sys
 
+TEST_OK_FILE = {
+    "name": "Test Assignment",
+    "endpoint": "cal/cs61a/fa19/test",
+    "src": [
+        "test.py"
+    ],
+    "tests": {
+        "test.py": "doctest,lint_test"
+    },
+    "default_tests": [
+        "test1"
+    ],
+    "protocols": [
+        "restore",
+        "file_contents",
+        "unlock",
+        "grading",
+        "analytics",
+        "backup"
+    ]
+}
+
+TEST_FILE = '''
+def f(x):
+    """
+    >>> f(2)
+    4
+    >>> f(3)
+    9
+    """
+    return x ** 2
+def g(x):
+    """
+    >>> g(2)
+    4
+    >>> g(3)
+    9
+    """
+    return x ** 3
+'''
+
 SCRIPT = """
 . {envloc}/{folder}/activate;
 python ok {args}
@@ -32,12 +73,16 @@ def add_file(self, name, contents):
         with open(os.path.join(self.directory.name, name), "w") as f:
             f.write(contents)
 
+    def add_ok_file(self):
+        self.add_file("test.ok", json.dumps(TEST_OK_FILE))
+
     def run_ok(self, *args):
         command_line = SCRIPT.format(
             envloc=shlex.quote(self.clean_env_dir.name),
             folder="Scripts" if sys.platform == "win32" else "bin",
             args=" ".join(shlex.quote(arg) for arg in args),
         )
+        print(command_line)
         with subprocess.Popen(
                 os.getenv('SHELL', 'sh'),
                 stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
@@ -56,27 +101,22 @@ def testUpdate(self):
         self.assertRegex(stdout, "Current version: v[0-9.]+\nChecking for software updates...\nOK is up to date")
 
     def testRunNoArgument(self):
-        self.add_file("test.ok", json.dumps(
-            {
-                "name": "Test Assignment",
-                "endpoint": "cal/cs61a/fa19/test",
-                "src": [
-                    "test.py"
-                ],
-                "tests": {
-                    "test.py": "doctest"
-                },
-                "default_tests": [],
-                "protocols": [
-                    "restore",
-                    "file_contents",
-                    "unlock",
-                    "grading",
-                    "analytics",
-                    "backup"
-                ]
-            }
-        ))
+        self.add_ok_file()
+        self.add_file("test.py", "")
         stdout, stderr = self.run_ok("--local")
         self.assertEqual(stderr, "")
         self.assertRegex(stdout, ".*0 test cases passed! No cases failed.*")
+
+    def testPassingTest(self):
+        self.add_ok_file()
+        self.add_file("test.py", TEST_FILE)
+        stdout, stderr = self.run_ok("-q", "f", "--local")
+        self.assertEqual(stderr, "")
+        self.assertRegex(stdout, ".*1 test cases passed! No cases failed.*")
+
+    def testFailingTest(self):
+        self.add_ok_file()
+        self.add_file("test.py", TEST_FILE)
+        stdout, stderr = self.run_ok("-q", "g", "--local")
+        self.assertEqual(stderr, "")
+        self.assertRegex(stdout, ".*0 test cases passed! No cases failed.*")