diff --git a/client/api/assignment.py b/client/api/assignment.py index 20b8b0e0..13af4125 100644 --- a/client/api/assignment.py +++ b/client/api/assignment.py @@ -195,7 +195,8 @@ def _load_tests(self): for file in sorted(glob.glob(file_pattern)): try: module = importlib.import_module(self._TESTS_PACKAGE + '.' + source) - except ImportError: + except ImportError as e: + print(e) raise ex.LoadingException('Invalid test source: {}'.format(source)) test_name = file diff --git a/client/cli/publish.py b/client/cli/publish.py index eb29569c..09659d96 100644 --- a/client/cli/publish.py +++ b/client/cli/publish.py @@ -14,7 +14,8 @@ EXTRA_PACKAGES = [ 'requests', 'certifi', 'urllib3', 'chardet', 'idna', # requests/certifi and recursive deps 'coverage', # coverage and recursive deps - 'pytutor', 'ast_scope', 'attr' # pytutor and recursive deps + 'pytutor', 'ast_scope', 'attr', # pytutor and recursive deps + 'pycodestyle.py', # pycodestyle and recursive deps ] def abort(message): diff --git a/client/sources/lint_test/__init__.py b/client/sources/lint_test/__init__.py new file mode 100644 index 00000000..b8e089bf --- /dev/null +++ b/client/sources/lint_test/__init__.py @@ -0,0 +1,24 @@ +from client import exceptions as ex +from client.sources.common import importing +from client.sources.lint_test import models +import logging +import os +import traceback + +log = logging.getLogger(__name__) + +def load(file, name, assign): + """Loads doctests from a specified filepath. + + PARAMETERS: + file -- str; a filepath to a Python module containing OK-style + tests. + name -- str; optional parameter that specifies a particular function in + the file. If omitted, all doctests will be included. + + RETURNS: + Test + """ + if not os.path.isfile(file) or not file.endswith('.py'): + raise ex.LoadingException('Cannot run python linter on {}'.format(file)) + return {name + "_lint" : models.LinterTest(file, name=name + "_lint", points=1)} diff --git a/client/sources/lint_test/models.py b/client/sources/lint_test/models.py new file mode 100644 index 00000000..4adf33b1 --- /dev/null +++ b/client/sources/lint_test/models.py @@ -0,0 +1,64 @@ +from client import exceptions as ex +from client.sources.common import core +from client.sources.common import importing +from client.sources.common import interpreter +from client.sources.common import models +from client.sources.common import pyconsole +from client.utils import format +from client.utils import output +import re +import textwrap + +import pycodestyle + +########## +# Models # +########## + +class LinterTest(models.Test): + + def __init__(self, file, **fields): + super().__init__(**fields) + self.file = file + self.styleguide = None + + def post_instantiation(self): + self.styleguide = pycodestyle.StyleGuide() + + def _lint(self): + return self.styleguide.check_files([self.file]) + + def run(self, env): + """Runs the suites associated with this doctest. + + NOTE: env is intended only for use with the programmatic API to support + Python OK tests. It is not used here. + + RETURNS: + bool; True if the doctest completely passes, False otherwise. + """ + result = self._lint() + if result.total_errors == 0: + return {'passed': 1, 'failed': 0, 'locked': 0} + else: + return {'passed': 0, 'failed': 1, 'locked': 0} + + def score(self): + format.print_line('-') + print('Lint tests for {}'.format(self.file)) + print() + success = self._lint().total_errors == 0 + score = 1.0 if success else 0.0 + + print('Score: {}/1'.format(score)) + print() + return score + + def unlock(self, interact): + """Lint tests cannot be unlocked.""" + + def lock(self, hash_fn): + """Lint tests cannot be locked.""" + + def dump(self): + """Lint tests do not need to be dumped, since no state changes.""" diff --git a/requirements.txt b/requirements.txt index ddc3aa48..8f333a28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,8 @@ coverage==4.4 pytutor==1.0.0 ast-scope==0.3.1 attrs==19.3.0 - +## Linting +pycodestyle==2.5.0 # Tests nose==1.3.3 diff --git a/setup.py b/setup.py index adecae6d..4d0afbea 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,7 @@ 'coverage==4.4', 'pytutor==1.0.0', 'ast-scope==0.3.1', - 'attrs==19.3.0' + 'attrs==19.3.0', + 'pycodestyle==2.5.0' ], ) diff --git a/tests/api/assignment_test.py b/tests/api/assignment_test.py index 60face50..fc6ff4bb 100644 --- a/tests/api/assignment_test.py +++ b/tests/api/assignment_test.py @@ -416,7 +416,7 @@ class AssignmentGradeTest(unittest.TestCase): "name": "Homework 1", "endpoint": "", "tests": { - "q1.py": "ok_test" + "q1.py": "ok_test,lint_test" }, "protocols": [] } diff --git a/tests/end_to_end/smoke_test.py b/tests/end_to_end/smoke_test.py index 0ac6364b..bc7123a8 100644 --- a/tests/end_to_end/smoke_test.py +++ b/tests/end_to_end/smoke_test.py @@ -7,6 +7,47 @@ import shlex import sys +TEST_OK_FILE = { + "name": "Test Assignment", + "endpoint": "cal/cs61a/fa19/test", + "src": [ + "test.py" + ], + "tests": { + "test.py": "doctest,lint_test" + }, + "default_tests": [ + "test1" + ], + "protocols": [ + "restore", + "file_contents", + "unlock", + "grading", + "analytics", + "backup" + ] +} + +TEST_FILE = ''' +def f(x): + """ + >>> f(2) + 4 + >>> f(3) + 9 + """ + return x ** 2 +def g(x): + """ + >>> g(2) + 4 + >>> g(3) + 9 + """ + return x ** 3 +''' + SCRIPT = """ . {envloc}/{folder}/activate; python ok {args} @@ -32,12 +73,16 @@ def add_file(self, name, contents): with open(os.path.join(self.directory.name, name), "w") as f: f.write(contents) + def add_ok_file(self): + self.add_file("test.ok", json.dumps(TEST_OK_FILE)) + def run_ok(self, *args): command_line = SCRIPT.format( envloc=shlex.quote(self.clean_env_dir.name), folder="Scripts" if sys.platform == "win32" else "bin", args=" ".join(shlex.quote(arg) for arg in args), ) + print(command_line) with subprocess.Popen( os.getenv('SHELL', 'sh'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -56,27 +101,22 @@ def testUpdate(self): self.assertRegex(stdout, "Current version: v[0-9.]+\nChecking for software updates...\nOK is up to date") def testRunNoArgument(self): - self.add_file("test.ok", json.dumps( - { - "name": "Test Assignment", - "endpoint": "cal/cs61a/fa19/test", - "src": [ - "test.py" - ], - "tests": { - "test.py": "doctest" - }, - "default_tests": [], - "protocols": [ - "restore", - "file_contents", - "unlock", - "grading", - "analytics", - "backup" - ] - } - )) + self.add_ok_file() + self.add_file("test.py", "") stdout, stderr = self.run_ok("--local") self.assertEqual(stderr, "") self.assertRegex(stdout, ".*0 test cases passed! No cases failed.*") + + def testPassingTest(self): + self.add_ok_file() + self.add_file("test.py", TEST_FILE) + stdout, stderr = self.run_ok("-q", "f", "--local") + self.assertEqual(stderr, "") + self.assertRegex(stdout, ".*1 test cases passed! No cases failed.*") + + def testFailingTest(self): + self.add_ok_file() + self.add_file("test.py", TEST_FILE) + stdout, stderr = self.run_ok("-q", "g", "--local") + self.assertEqual(stderr, "") + self.assertRegex(stdout, ".*0 test cases passed! No cases failed.*")