diff --git a/README.md b/README.md index 16c87e4..9feab37 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,12 @@ Automatically test services/checker using the enochecker API `enochecker_test` can be used to run tests against a checker, optionally you can specify wich tests to run e.g. `enochecker_test test_getflag[0] test_exploit_per_exploit_id` will run only the first `getflag` test and all `exploit_per_exploit_id` tests. ``` -usage: enochecker_test [-h] [-a CHECKER_ADDRESS] [-p {1..65535}] [-A SERVICE_ADDRESS] [testcase [testcase ...]] +usage: enochecker_test [-h] [-a CHECKER_ADDRESS] [-p {1..65535}] [-A SERVICE_ADDRESS] [testexpr] Utility for testing checkers that implement the enochecker API positional arguments: - testcase Specify the tests that should be run in the syntax expected by pytest, e.g. test_getflag. If no test is specified, all tests will be run. + testexpr Specify the tests that should be run in the syntax expected by pytests -k flag, e.g. 'test_getflag' or 'not exploit'. If no expr is specified, all tests will be run. optional arguments: -h, --help show this help message and exit diff --git a/enochecker_test/main.py b/enochecker_test/main.py index d1ce291..90d499b 100644 --- a/enochecker_test/main.py +++ b/enochecker_test/main.py @@ -11,7 +11,7 @@ from urllib3.util.retry import Retry -def run_tests(host, port, service_address, test_methods): +def run_tests(host, port, service_address, test_expr): s = requests.Session() retry_strategy = Retry( total=5, @@ -46,13 +46,10 @@ def run_tests(host, port, service_address, test_methods): "-v", ] - if test_methods is None or len(test_methods) == 0: - test_args.append(os.path.join(os.path.dirname(__file__), "tests.py")) - else: - for method in test_methods: - test_args.append( - os.path.join(os.path.dirname(__file__), "tests.py") + "::" + method - ) + if test_expr: + test_args.append("-k") + test_args.append(test_expr) + test_args.append(os.path.join(os.path.dirname(__file__), "tests.py")) sys.exit(pytest.main(test_args)) @@ -95,9 +92,9 @@ def main(): default=os.environ.get("ENOCHECKER_TEST_SERVICE_ADDRESS"), ) parser.add_argument( - "testcase", - help="Specify the tests that should be run in the syntax expected by pytest, e.g. test_getflag. If no test is specified, all tests will be run.", - nargs="*", + "testexpr", + help="Specify the tests that should be run in the syntax expected by pytests -k flag, e.g. 'test_getflag' or 'not exploit'. If no expr is specified, all tests will be run.", + nargs="?", ) args = parser.parse_args() @@ -120,5 +117,5 @@ def main(): logging.basicConfig(level=logging.INFO) run_tests( - args.checker_address, args.checker_port, args.service_address, args.testcase + args.checker_address, args.checker_port, args.service_address, args.testexpr )