allow specification of tests with pytest -k expression (#43)

Co-authored-by: ldruschk <[email protected]>

README.md

@@ -5,12 +5,12 @@ Automatically test services/checker using the enochecker API
 `enochecker_test` can be used to run tests against a checker, optionally you can specify wich tests to run e.g. `enochecker_test test_getflag[0] test_exploit_per_exploit_id` will run only the first `getflag` test and all `exploit_per_exploit_id` tests.
 
 ```
-usage: enochecker_test [-h] [-a CHECKER_ADDRESS] [-p {1..65535}] [-A SERVICE_ADDRESS] [testcase [testcase ...]]
+usage: enochecker_test [-h] [-a CHECKER_ADDRESS] [-p {1..65535}] [-A SERVICE_ADDRESS] [testexpr]
 
 Utility for testing checkers that implement the enochecker API
 
 positional arguments:
-  testcase              Specify the tests that should be run in the syntax expected by pytest, e.g. test_getflag. If no test is specified, all tests will be run.
+  testexpr              Specify the tests that should be run in the syntax expected by pytests -k flag, e.g. 'test_getflag' or 'not exploit'. If no expr is specified, all tests will be run.
 
 optional arguments:
   -h, --help            show this help message and exit

enochecker_test/main.py

@@ -11,7 +11,7 @@
 from urllib3.util.retry import Retry
 
 
-def run_tests(host, port, service_address, test_methods):
+def run_tests(host, port, service_address, test_expr):
     s = requests.Session()
     retry_strategy = Retry(
         total=5,
@@ -46,13 +46,10 @@ def run_tests(host, port, service_address, test_methods):
         "-v",
     ]
 
-    if test_methods is None or len(test_methods) == 0:
-        test_args.append(os.path.join(os.path.dirname(__file__), "tests.py"))
-    else:
-        for method in test_methods:
-            test_args.append(
-                os.path.join(os.path.dirname(__file__), "tests.py") + "::" + method
-            )
+    if test_expr:
+        test_args.append("-k")
+        test_args.append(test_expr)
+    test_args.append(os.path.join(os.path.dirname(__file__), "tests.py"))
 
     sys.exit(pytest.main(test_args))
 
@@ -95,9 +92,9 @@ def main():
         default=os.environ.get("ENOCHECKER_TEST_SERVICE_ADDRESS"),
     )
     parser.add_argument(
-        "testcase",
-        help="Specify the tests that should be run in the syntax expected by pytest, e.g. test_getflag. If no test is specified, all tests will be run.",
-        nargs="*",
+        "testexpr",
+        help="Specify the tests that should be run in the syntax expected by pytests -k flag, e.g. 'test_getflag' or 'not exploit'. If no expr is specified, all tests will be run.",
+        nargs="?",
     )
 
     args = parser.parse_args()
@@ -120,5 +117,5 @@ def main():
 
     logging.basicConfig(level=logging.INFO)
     run_tests(
-        args.checker_address, args.checker_port, args.service_address, args.testcase
+        args.checker_address, args.checker_port, args.service_address, args.testexpr
     )