diff --git a/dmoj/checkers/bridged.py b/dmoj/checkers/bridged.py index cc01e3229..ee77a2a57 100644 --- a/dmoj/checkers/bridged.py +++ b/dmoj/checkers/bridged.py @@ -10,13 +10,13 @@ from dmoj.utils.unicode import utf8text -def get_executor(problem_id, files, flags, lang, compiler_time_limit): +def get_executor(problem_id, storage_namespace, files, flags, lang, compiler_time_limit): if isinstance(files, str): filenames = [files] elif isinstance(files.unwrap(), list): filenames = list(files.unwrap()) - filenames = [os.path.join(get_problem_root(problem_id), f) for f in filenames] + filenames = [os.path.join(get_problem_root(problem_id, storage_namespace), f) for f in filenames] executor = compile_with_auxiliary_files(filenames, flags, lang, compiler_time_limit) return executor @@ -40,6 +40,7 @@ def check( input_name=None, output_name=None, treat_checker_points_as_percentage=False, + storage_namespace=None, **kwargs, ) -> CheckerResult: @@ -52,7 +53,7 @@ def check( flags.append('-DTHEMIS') elif type == 'cms': flags.append('-DCMS') - executor = get_executor(problem_id, files, flags, lang, compiler_time_limit) + executor = get_executor(problem_id, storage_namespace, files, flags, lang, compiler_time_limit) if type not in contrib_modules: raise InternalError('%s is not a valid contrib module' % type) diff --git a/dmoj/commands/rejudge.py b/dmoj/commands/rejudge.py index 7e47c7710..b54f0fa83 100644 --- a/dmoj/commands/rejudge.py +++ b/dmoj/commands/rejudge.py @@ -14,7 +14,7 @@ def execute(self, line: str) -> None: problem_id, lang, src, tl, ml = self.get_submission_data(args.submission_id) self.judge.begin_grading( - Submission(self.judge.submission_id_counter, problem_id, lang, src, tl, ml, False, {}), + Submission(self.judge.submission_id_counter, problem_id, None, lang, src, tl, ml, False, {}), blocking=True, report=print, ) diff --git a/dmoj/commands/resubmit.py b/dmoj/commands/resubmit.py index c8e4827dd..4290e6a47 100644 --- a/dmoj/commands/resubmit.py +++ b/dmoj/commands/resubmit.py @@ -47,7 +47,7 @@ def execute(self, line: str) -> None: self.judge.graded_submissions.append((problem_id, lang, src, tl, ml)) try: self.judge.begin_grading( - Submission(self.judge.submission_id_counter, problem_id, lang, src, tl, ml, False, {}), + Submission(self.judge.submission_id_counter, problem_id, None, lang, src, tl, ml, False, {}), blocking=True, report=print, ) diff --git a/dmoj/commands/submit.py b/dmoj/commands/submit.py index 4eb7844a7..7cea6d738 100644 --- a/dmoj/commands/submit.py +++ b/dmoj/commands/submit.py @@ -70,7 +70,15 @@ def execute(self, line: str) -> None: try: self.judge.begin_grading( Submission( - self.judge.submission_id_counter, problem_id, language_id, src, time_limit, memory_limit, False, {} + self.judge.submission_id_counter, + problem_id, + None, + language_id, + src, + time_limit, + memory_limit, + False, + {}, ), blocking=True, report=print, diff --git a/dmoj/graders/bridged.py b/dmoj/graders/bridged.py index a9b62479c..e5085143d 100644 --- a/dmoj/graders/bridged.py +++ b/dmoj/graders/bridged.py @@ -129,7 +129,7 @@ def _generate_interactor_binary(self) -> BaseExecutor: filenames = [files] elif isinstance(files.unwrap(), list): filenames = list(files.unwrap()) - problem_root = get_problem_root(self.problem.id) + problem_root = get_problem_root(self.problem.id, self.problem.storage_namespace) assert problem_root is not None filenames = [os.path.join(problem_root, f) for f in filenames] flags = self.handler_data.get('flags', []) diff --git a/dmoj/graders/communication.py b/dmoj/graders/communication.py index 98266e4a1..360640a84 100644 --- a/dmoj/graders/communication.py +++ b/dmoj/graders/communication.py @@ -228,7 +228,7 @@ def _generate_manager_binary(self) -> BaseExecutor: filenames = [files] elif isinstance(files.unwrap(), list): filenames = list(files.unwrap()) - problem_root = get_problem_root(self.problem.id) + problem_root = get_problem_root(self.problem.id, self.problem.storage_namespace) assert problem_root is not None filenames = [os.path.join(problem_root, f) for f in filenames] flags = self.handler_data.manager.get('flags', []) diff --git a/dmoj/graders/custom.py b/dmoj/graders/custom.py index 29b8b4eb9..8956fd339 100644 --- a/dmoj/graders/custom.py +++ b/dmoj/graders/custom.py @@ -7,7 +7,9 @@ class CustomGrader: def __init__(self, judge, problem, language, source): self.judge = judge - self.mod = load_module_from_file(os.path.join(get_problem_root(problem.id), problem.config['custom_judge'])) + self.mod = load_module_from_file( + os.path.join(get_problem_root(problem.id, problem.storage_namespace), problem.config['custom_judge']) + ) self._grader = self.mod.Grader(judge, problem, language, source) def __getattr__(self, item): diff --git a/dmoj/judge.py b/dmoj/judge.py index 2c710ed61..a2a4d9112 100644 --- a/dmoj/judge.py +++ b/dmoj/judge.py @@ -61,6 +61,7 @@ class IPC(Enum): [ ('id', int), ('problem_id', str), + ('storage_namespace', Optional[str]), ('language', str), ('source', str), ('time_limit', float), @@ -448,7 +449,11 @@ def _report_unhandled_exception() -> None: def _grade_cases(self) -> Generator[Tuple[IPC, tuple], None, None]: problem = Problem( - self.submission.problem_id, self.submission.time_limit, self.submission.memory_limit, self.submission.meta + self.submission.problem_id, + self.submission.time_limit, + self.submission.memory_limit, + self.submission.meta, + storage_namespace=self.submission.storage_namespace, ) try: diff --git a/dmoj/judgeenv.py b/dmoj/judgeenv.py index 7c14b14e4..2ffc23d0b 100644 --- a/dmoj/judgeenv.py +++ b/dmoj/judgeenv.py @@ -3,6 +3,7 @@ import logging import os import ssl +from collections import defaultdict from fnmatch import fnmatch from operator import itemgetter from typing import Dict, Iterable, List, Optional, Set, Tuple @@ -15,6 +16,7 @@ from dmoj.utils.glob_ext import find_glob_root from dmoj.utils.unicode import utf8text +storage_namespaces: Dict[Optional[str], List[str]] = {} problem_globs: List[str] = [] problem_watches: List[str] = [] env: ConfigNode = ConfigNode( @@ -74,13 +76,18 @@ only_executors: Set[str] = set() exclude_executors: Set[str] = set() -_problem_root_cache: Dict[str, str] = {} -_problem_roots_cache: Optional[List[str]] = None -_supported_problems_cache = None + +class StorageNamespaceCache: + problem_root_cache: Dict[str, str] = {} + problem_roots_cache: Optional[List[str]] = None + supported_problems_cache: Optional[List[Tuple[str, float]]] = None + + +_storage_namespace_cache: Dict[Optional[str], StorageNamespaceCache] = defaultdict(StorageNamespaceCache) def load_env(cli: bool = False, testsuite: bool = False) -> None: # pragma: no cover - global problem_globs, only_executors, exclude_executors, log_file, server_host, server_port, no_ansi, no_ansi_emu, skip_self_test, env, startup_warnings, no_watchdog, problem_regex, case_regex, api_listen, secure, no_cert_check, cert_store, problem_watches, cli_history_file, cli_command, log_level + global storage_namespaces, problem_globs, only_executors, exclude_executors, log_file, server_host, server_port, no_ansi, no_ansi_emu, skip_self_test, env, startup_warnings, no_watchdog, problem_regex, case_regex, api_listen, secure, no_cert_check, cert_store, problem_watches, cli_history_file, cli_command, log_level if cli: description = 'Starts a shell for interfacing with a local judge instance.' @@ -207,15 +214,23 @@ def load_env(cli: bool = False, testsuite: bool = False) -> None: # pragma: no env['key'] = os.environ['DMOJ_JUDGE_KEY'] if not testsuite: - problem_globs = env.problem_storage_globs - if problem_globs is None: - raise SystemExit(f'`problem_storage_globs` not specified in "{model_file}"; no problems available to grade') + storage_namespaces[None] = env.problem_storage_globs or [] + storage_namespaces.update(env.storage_namespaces or {}) + all_problem_globs = [] + for globs in storage_namespaces.values(): + all_problem_globs.extend(globs) + + if not all_problem_globs: + raise SystemExit('no problems available to grade') + + problem_globs = storage_namespaces[None] problem_watches = problem_globs else: if not os.path.isdir(args.tests_dir): raise SystemExit('Invalid tests directory') problem_globs = [os.path.join(args.tests_dir, '*')] + storage_namespaces[None] = problem_globs import re @@ -230,25 +245,29 @@ def load_env(cli: bool = False, testsuite: bool = False) -> None: # pragma: no except re.error: raise SystemExit('Invalid case regex') + for namespace in storage_namespaces: + _storage_namespace_cache[namespace] = StorageNamespaceCache() + skip_first_scan = False if cli else args.skip_first_scan if not skip_first_scan: # Populate cache and send warnings get_supported_problems_and_mtimes() else: - global _problem_roots_cache - global _supported_problems_cache - _problem_roots_cache = [str(root) for root in map(find_glob_root, problem_globs)] - _supported_problems_cache = [] + for namespace, globs in storage_namespaces.items(): + cache = _storage_namespace_cache[namespace] + cache.problem_roots_cache = [str(root) for root in map(find_glob_root, globs)] + cache.supported_problems_cache = [] def get_problem_watches(): return problem_watches -def get_problem_root(problem_id) -> Optional[str]: - cached_root = _problem_root_cache.get(problem_id) +def get_problem_root(problem_id, namespace=None) -> Optional[str]: + cache = _storage_namespace_cache[namespace] + cached_root = cache.problem_root_cache.get(problem_id) if cached_root is None or not os.path.isfile(os.path.join(cached_root, 'init.yml')): - for root_dir in get_problem_roots(): + for root_dir in get_problem_roots(namespace): problem_root_dir = os.path.join(root_dir, problem_id) problem_config = os.path.join(problem_root_dir, 'init.yml') if os.path.isfile(problem_config): @@ -256,18 +275,18 @@ def get_problem_root(problem_id) -> Optional[str]: fnmatch(problem_config, os.path.join(problem_glob, 'init.yml')) for problem_glob in problem_globs ): continue - _problem_root_cache[problem_id] = problem_root_dir + cache.problem_root_cache[problem_id] = problem_root_dir break else: return None - return _problem_root_cache[problem_id] + return cache.problem_root_cache[problem_id] -def get_problem_roots() -> List[str]: - global _problem_roots_cache - assert _problem_roots_cache is not None - return _problem_roots_cache +def get_problem_roots(namespace=None) -> List[str]: + cache = _storage_namespace_cache[namespace] + assert cache.problem_roots_cache is not None + return cache.problem_roots_cache def get_supported_problems_and_mtimes(warnings: bool = True, force_update: bool = False) -> List[Tuple[str, float]]: @@ -277,11 +296,10 @@ def get_supported_problems_and_mtimes(warnings: bool = True, force_update: bool A list of all problems in tuple format: (problem id, mtime) """ - global _problem_roots_cache - global _supported_problems_cache + cache = _storage_namespace_cache[None] - if _supported_problems_cache is not None and not force_update: - return _supported_problems_cache + if cache.supported_problems_cache is not None and not force_update: + return cache.supported_problems_cache problems = [] root_dirs = [] @@ -309,8 +327,8 @@ def get_supported_problems_and_mtimes(warnings: bool = True, force_update: bool problem_dirs[problem] = problem_dir problems.append((problem, os.path.getmtime(problem_dir))) - _problem_roots_cache = root_dirs - _supported_problems_cache = problems + cache.problem_roots_cache = root_dirs + cache.supported_problems_cache = problems return problems diff --git a/dmoj/packet.py b/dmoj/packet.py index 9680a8d0a..f0657d8ec 100644 --- a/dmoj/packet.py +++ b/dmoj/packet.py @@ -260,6 +260,7 @@ def _receive_packet(self, packet: dict): Submission( id=packet['submission-id'], problem_id=packet['problem-id'], + storage_namespace=packet.get('storage-namespace', None), language=packet['language'], source=packet['source'], time_limit=float(packet['time-limit']), diff --git a/dmoj/problem.py b/dmoj/problem.py index c3aa5d248..1afbac26a 100644 --- a/dmoj/problem.py +++ b/dmoj/problem.py @@ -50,6 +50,7 @@ class BaseTestCase: class Problem: id: str + storage_namespace: Optional[str] time_limit: float memory_limit: int meta: ConfigNode @@ -58,8 +59,11 @@ class Problem: problem_data: 'ProblemDataManager' config: 'ProblemConfig' - def __init__(self, problem_id: str, time_limit: float, memory_limit: int, meta: dict) -> None: + def __init__( + self, problem_id: str, time_limit: float, memory_limit: int, meta: dict, storage_namespace: Optional[str] = None + ) -> None: self.id = problem_id + self.storage_namespace = storage_namespace self.time_limit = time_limit self.memory_limit = memory_limit self.meta = ConfigNode(meta) @@ -68,7 +72,7 @@ def __init__(self, problem_id: str, time_limit: float, memory_limit: int, meta: self._testcase_counter = 0 # Cache root dir so that we don't need to scan all roots (potentially very slow on networked mount). - root_dir = get_problem_root(problem_id) + root_dir = get_problem_root(problem_id, storage_namespace) assert root_dir is not None self.root_dir = root_dir self.problem_data = ProblemDataManager(self.root_dir) @@ -386,7 +390,7 @@ def _run_generator(self, gen: Union[str, ConfigNode], args: Optional[Iterable[st compiler_time_limit = env.generator_compiler_time_limit lang = None # Default to C/C++ - base = get_problem_root(self.problem.id) + base = get_problem_root(self.problem.id, self.problem.storage_namespace) assert base is not None filenames: Union[str, list] if isinstance(gen, str): @@ -489,11 +493,14 @@ def checker(self) -> partial: if not hasattr(checker, 'check') or not callable(checker.check): raise InvalidInitException('malformed checker: no check method found') + params['storage_namespace'] = self.problem.storage_namespace + # Themis checker need input name and output name if self.config['in']: params['input_name'] = self.config['in'] if self.config['out']: params['output_name'] = self.config['out'] + return partial(checker.check, **params) def free_data(self) -> None: diff --git a/dmoj/testsuite.py b/dmoj/testsuite.py index f3e0e7ec3..c2cc28342 100644 --- a/dmoj/testsuite.py +++ b/dmoj/testsuite.py @@ -290,7 +290,7 @@ def output_case(data): extended_feedback_cases, ) self.judge.begin_grading( - Submission(self.sub_id, problem, language, source, time, memory, False, meta), + Submission(self.sub_id, problem, None, language, source, time, memory, False, meta), blocking=True, report=output_case, )