From e3a57c55d513833c51e4797c765edd970c10508d Mon Sep 17 00:00:00 2001 From: Tomasz Kwiatkowski Date: Thu, 28 Nov 2024 12:02:56 +0100 Subject: [PATCH 1/9] Package identification --- src/sio3pack/packages/__init__.py | 2 ++ src/sio3pack/packages/package/model.py | 7 ++++ src/sio3pack/packages/sinolpack/model.py | 15 ++++++++- src/sio3pack/utils.py | 41 ++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 src/sio3pack/utils.py diff --git a/src/sio3pack/packages/__init__.py b/src/sio3pack/packages/__init__.py index 22919aa..fd5539f 100644 --- a/src/sio3pack/packages/__init__.py +++ b/src/sio3pack/packages/__init__.py @@ -1 +1,3 @@ from sio3pack.packages.sinolpack import Sinolpack + +all_packages = [Sinolpack] diff --git a/src/sio3pack/packages/package/model.py b/src/sio3pack/packages/package/model.py index fa7a215..00aa8c2 100644 --- a/src/sio3pack/packages/package/model.py +++ b/src/sio3pack/packages/package/model.py @@ -4,6 +4,7 @@ from sio3pack.graph.graph import Graph from sio3pack.test.test import Test +from sio3pack.packages import all_packages class Package: """ @@ -13,6 +14,12 @@ class Package: def __init__(self): pass + @classmethod + def from_file(cls, file: File): + for package_type in all_packages: + if package_type.identify(file): + return package_type(file) + def get_task_id(self) -> str: pass diff --git a/src/sio3pack/packages/sinolpack/model.py b/src/sio3pack/packages/sinolpack/model.py index a31f567..404887c 100644 --- a/src/sio3pack/packages/sinolpack/model.py +++ b/src/sio3pack/packages/sinolpack/model.py @@ -1,3 +1,6 @@ +import os +from sio3pack import utils +from sio3pack.files.file import File from sio3pack.packages.package import Package @@ -6,4 +9,14 @@ class Sinolpack(Package): Represents a OIOIOI's standard problem package. """ - pass + @classmethod + def identify(cls, file: File) -> bool: + """ + Identifies whether file is a Sinolpack. + + :param file: File with package. + :return: True when file is a Sinolpack, otherwise False. + """ + if utils.is_archive(file): + return utils.has_dir(file, "in") and utils.has_dir(file, "out") + return os.path.exists(os.path.join(file, "in")) and os.path.exists(os.path.join(file, "out")) diff --git a/src/sio3pack/utils.py b/src/sio3pack/utils.py new file mode 100644 index 0000000..a439192 --- /dev/null +++ b/src/sio3pack/utils.py @@ -0,0 +1,41 @@ +import tarfile +import zipfile + +from sio3pack.files.file import File + + +def is_archive(file: str|File): + if isinstance(file, File): + file_path = file.path + else: + file_path = file + + try: + if zipfile.is_zipfile(file_path): + return True + + if tarfile.is_tarfile(file_path): + return True + except: + pass + + return False + +def has_dir(file: str|File, dir_name: str): + if isinstance(file, File): + file_path = file.path + else: + file_path = file + + try: + if zipfile.is_zipfile(file_path): + archive = zipfile.ZipFile(file_path) + return dir_name in map(lambda f: f.filename, archive.infolist()) + + if tarfile.is_tarfile(file_path): + archive = tarfile.TarFile(file_path) + return dir_name in map(lambda f: f.name, archive.getmembers()) + except: + pass + + return False From e69e62d543baed82ace873ee871aa1ac4dce437a Mon Sep 17 00:00:00 2001 From: MasloMaslane Date: Sat, 30 Nov 2024 16:47:26 +0100 Subject: [PATCH 2/9] Code --- src/sio3pack/__init__.py | 15 ++ src/sio3pack/files/file.py | 29 +++ src/sio3pack/graph/graph.py | 10 + src/sio3pack/graph/graph_manager.py | 29 +++ src/sio3pack/graph/graph_op.py | 23 ++ src/sio3pack/packages/__init__.py | 2 - src/sio3pack/packages/exceptions.py | 2 + src/sio3pack/packages/package/model.py | 25 +- src/sio3pack/packages/sinolpack/enums.py | 17 ++ src/sio3pack/packages/sinolpack/model.py | 306 ++++++++++++++++++++++- src/sio3pack/util.py | 9 + src/sio3pack/utils.py | 41 --- src/sio3pack/utils/__init__.py | 0 src/sio3pack/utils/archive.py | 250 ++++++++++++++++++ src/sio3pack/utils/classinit.py | 90 +++++++ 15 files changed, 794 insertions(+), 54 deletions(-) create mode 100644 src/sio3pack/graph/graph_manager.py create mode 100644 src/sio3pack/graph/graph_op.py create mode 100644 src/sio3pack/packages/exceptions.py create mode 100644 src/sio3pack/packages/sinolpack/enums.py create mode 100644 src/sio3pack/util.py delete mode 100644 src/sio3pack/utils.py create mode 100644 src/sio3pack/utils/__init__.py create mode 100644 src/sio3pack/utils/archive.py create mode 100644 src/sio3pack/utils/classinit.py diff --git a/src/sio3pack/__init__.py b/src/sio3pack/__init__.py index f102a9c..51fe899 100644 --- a/src/sio3pack/__init__.py +++ b/src/sio3pack/__init__.py @@ -1 +1,16 @@ __version__ = "0.0.1" + +from sio3pack.files.file import File +from sio3pack.packages.package import Package + + +def from_file(file: str | File, django_settings=None) -> Package: + """ + Initialize a package object from a file (archive or directory). + :param file: The file path or File object. + :param django_settings: Django settings object. + :return: The package object. + """ + if isinstance(file, str): + file = File(file) + return Package.from_file(file, django_settings) diff --git a/src/sio3pack/files/file.py b/src/sio3pack/files/file.py index 4548836..779ec3a 100644 --- a/src/sio3pack/files/file.py +++ b/src/sio3pack/files/file.py @@ -1,7 +1,36 @@ +import os.path + + class File: """ Base class for all files in a package. """ + @classmethod + def get_file_matching_extension(cls, dir: str, filename: str, extensions: list[str]) -> "File": + """ + Get the file with the given filename and one of the given extensions. + :param dir: The directory to search in. + :param filename: The filename. + :param extensions: The extensions. + :return: The file object. + """ + for ext in extensions: + path = os.path.join(dir, filename + ext) + if os.path.exists(path): + return cls(path) + raise FileNotFoundError + def __init__(self, path: str): + if not os.path.exists(path): + raise FileNotFoundError self.path = path + self.filename = os.path.basename(path) + + def read(self) -> str: + with open(self.path, "r") as f: + return f.read() + + def write(self, text: str): + with open(self.path, "w") as f: + f.write(text) diff --git a/src/sio3pack/graph/graph.py b/src/sio3pack/graph/graph.py index 166df8b..cdd50b2 100644 --- a/src/sio3pack/graph/graph.py +++ b/src/sio3pack/graph/graph.py @@ -3,5 +3,15 @@ class Graph: A class to represent a job graph. """ + @classmethod + def from_dict(cls, data: dict): + raise NotImplemented + def __init__(self, name: str): self.name = name + + def get_prog_files(self) -> list[str]: + """ + Get all program files in the graph. + """ + raise NotImplemented diff --git a/src/sio3pack/graph/graph_manager.py b/src/sio3pack/graph/graph_manager.py new file mode 100644 index 0000000..585ba8b --- /dev/null +++ b/src/sio3pack/graph/graph_manager.py @@ -0,0 +1,29 @@ +import json + +from sio3pack import File +from sio3pack.graph.graph import Graph + + +class GraphManager: + @classmethod + def from_file(cls, file: File): + graphs = {} + content = json.loads(file.read()) + for name, graph in content.items(): + graphs[name] = Graph.from_dict(graph) + return cls(graphs) + + def __init__(self, graphs: dict[str, Graph]): + self.graphs = graphs + + def get_prog_files(self) -> list[str]: + """ + Get all program files used in all graphs. + """ + files = set() + for graph in self.graphs.values(): + files.update(graph.get_prog_files()) + return list(files) + + def get(self, name: str) -> Graph: + return self.graphs[name] diff --git a/src/sio3pack/graph/graph_op.py b/src/sio3pack/graph/graph_op.py new file mode 100644 index 0000000..f5d4994 --- /dev/null +++ b/src/sio3pack/graph/graph_op.py @@ -0,0 +1,23 @@ +from sio3pack.graph.graph import Graph + + +class GraphOperation: + """ + A class to represent a graph that should be run on workers. + Allows for returning results. + """ + + def __init__(self, graph: Graph, return_results: bool = False, return_func: callable = None): + """ + :param graph: The graph to run on workers. + :param return_results: Whether to return the results. + :param return_func: The function to use to return the + results, if return_results is True. + """ + self.graph = graph + self.return_results = return_results + self.return_func = return_func + + def return_results(self, data: dict): + if self.return_func: + return self.return_func(data) diff --git a/src/sio3pack/packages/__init__.py b/src/sio3pack/packages/__init__.py index fd5539f..22919aa 100644 --- a/src/sio3pack/packages/__init__.py +++ b/src/sio3pack/packages/__init__.py @@ -1,3 +1 @@ from sio3pack.packages.sinolpack import Sinolpack - -all_packages = [Sinolpack] diff --git a/src/sio3pack/packages/exceptions.py b/src/sio3pack/packages/exceptions.py new file mode 100644 index 0000000..ca91db2 --- /dev/null +++ b/src/sio3pack/packages/exceptions.py @@ -0,0 +1,2 @@ +class UnknownPackageType(Exception): + pass diff --git a/src/sio3pack/packages/package/model.py b/src/sio3pack/packages/package/model.py index 00aa8c2..cea7afd 100644 --- a/src/sio3pack/packages/package/model.py +++ b/src/sio3pack/packages/package/model.py @@ -2,23 +2,34 @@ from sio3pack.files.file import File from sio3pack.graph.graph import Graph +from sio3pack.packages.exceptions import UnknownPackageType from sio3pack.test.test import Test from sio3pack.packages import all_packages +from sio3pack.utils.archive import Archive +from sio3pack.utils.classinit import RegisteredSubclassesBase -class Package: + +class Package(RegisteredSubclassesBase): """ Base class for all packages. """ + abstract = True - def __init__(self): - pass + def __init__(self, file: File): + super().__init__() + self.file = file + if Archive.is_archive(file.path): + self.is_archive = True + else: + self.is_archive = False @classmethod - def from_file(cls, file: File): - for package_type in all_packages: - if package_type.identify(file): - return package_type(file) + def from_file(cls, file: File, django_settings=None): + for subclass in cls.subclasses: + if subclass.identify(file): + return subclass(file, django_settings) + raise UnknownPackageType def get_task_id(self) -> str: pass diff --git a/src/sio3pack/packages/sinolpack/enums.py b/src/sio3pack/packages/sinolpack/enums.py new file mode 100644 index 0000000..6f81e47 --- /dev/null +++ b/src/sio3pack/packages/sinolpack/enums.py @@ -0,0 +1,17 @@ +from enum import Enum + + +class ModelSolutionKind(Enum): + NORMAL = 0 + SLOW = 1 + INCORRECT = 2 + + @classmethod + def from_regex(cls, group): + if group == '': + return cls.NORMAL + if group == 's': + return cls.SLOW + if group == 'b': + return cls.INCORRECT + raise ValueError(f"Invalid model solution kind: {group}") diff --git a/src/sio3pack/packages/sinolpack/model.py b/src/sio3pack/packages/sinolpack/model.py index 404887c..374f5e1 100644 --- a/src/sio3pack/packages/sinolpack/model.py +++ b/src/sio3pack/packages/sinolpack/model.py @@ -1,7 +1,17 @@ +import json import os -from sio3pack import utils +import re +import yaml +import tempfile + from sio3pack.files.file import File +from sio3pack.graph.graph import Graph +from sio3pack.graph.graph_manager import GraphManager +from sio3pack.graph.graph_op import GraphOperation from sio3pack.packages.package import Package +from sio3pack.packages.sinolpack.enums import ModelSolutionKind +from sio3pack.util import naturalsort_key +from sio3pack.utils.archive import Archive, UnrecognizedArchiveFormat class Sinolpack(Package): @@ -9,6 +19,23 @@ class Sinolpack(Package): Represents a OIOIOI's standard problem package. """ + @classmethod + def _find_main_dir(cls, archive: Archive) -> str | None: + dirs = list(map(os.path.normcase, archive.dirnames())) + dirs = list(map(os.path.normpath, dirs)) + toplevel_dirs = list(set(f.split(os.sep)[0] for f in dirs)) + problem_dirs = [] + for dir in toplevel_dirs: + for required_subdir in ('in', 'out'): + if all(f.split(os.sep)[:2] != [dir, required_subdir] for f in dirs): + break + else: + problem_dirs.append(dir) + if len(problem_dirs) == 1: + return problem_dirs[0] + + return None + @classmethod def identify(cls, file: File) -> bool: """ @@ -17,6 +44,277 @@ def identify(cls, file: File) -> bool: :param file: File with package. :return: True when file is a Sinolpack, otherwise False. """ - if utils.is_archive(file): - return utils.has_dir(file, "in") and utils.has_dir(file, "out") - return os.path.exists(os.path.join(file, "in")) and os.path.exists(os.path.join(file, "out")) + path = file.path + try: + archive = Archive(path) + return cls._find_main_dir(archive) is not None + except UnrecognizedArchiveFormat: + return os.path.exists(os.path.join(path, "in")) and os.path.exists(os.path.join(path, "out")) + + def __del__(self): + if hasattr(self, "tmpdir"): + self.tmpdir.cleanup() + + def __init__(self, file: File, django_settings=None): + super().__init__(file) + if self.is_archive: + archive = Archive(file.path) + self.short_name = self._find_main_dir(archive) + self.tmpdir = tempfile.TemporaryDirectory() + archive.extract(to_path=self.tmpdir.name) + self.rootdir = os.path.join(self.tmpdir.name, self.short_name) + else: + self.short_name = os.path.basename(file.path) + self.rootdir = file.path + + try: + graph_file = self.get_in_root('graph.json') + self.graph_manager = GraphManager.from_file(graph_file) + except FileNotFoundError: + self.has_custom_graph = False + + self.django_settings = django_settings + + def _default_graph_manager(self) -> GraphManager: + return GraphManager({ + "unpack": Graph.from_dict({ + "name": "unpack", + # ... + }) + }) + + def _get_from_django_settings(self, key: str, default=None): + if self.django_settings is None: + return default + return getattr(self.django_settings, key, default) + + def get_doc_dir(self) -> str: + """ + Returns the path to the directory containing the problem's documents. + """ + return os.path.join(self.rootdir, 'doc') + + def get_in_doc_dir(self, filename: str) -> File: + """ + Returns the path to the input file in the documents' directory. + """ + return File(os.path.join(self.get_doc_dir(), filename)) + + def get_in_root(self, filename: str) -> File: + """ + Returns the path to the input file in the root directory. + """ + return File(os.path.join(self.rootdir, filename)) + + def get_prog_dir(self) -> str: + """ + Returns the path to the directory containing the problem's program files. + """ + return os.path.join(self.rootdir, 'prog') + + def get_in_prog_dir(self, filename: str) -> File: + """ + Returns the path to the input file in the program directory. + """ + return File(os.path.join(self.get_prog_dir(), filename)) + + def get_attachments_dir(self) -> str: + """ + Returns the path to the directory containing the problem's attachments. + """ + return os.path.join(self.rootdir, 'attachments') + + def _process_package(self): + self._process_config_yml() + self._detect_full_name() + self._detect_full_name_translations() + self._process_prog_files() + self._process_statements() + self._process_attachments() + + if not self.has_custom_graph: + # Create the graph with processed files. + self.graph_manager = self._default_graph_manager() + + def _process_config_yml(self): + """ + Process the config.yml file. If it exists, it will be loaded into the config attribute. + """ + try: + config = self.get_in_root('config.yml') + self.config = yaml.safe_load(config.read()) + except FileNotFoundError: + self.config = {} + + def _detect_full_name(self): + """ + Sets the problem's full name from the ``config.yml`` (key ``title``) + or from the ``title`` tag in the LaTeX source file (backwards compatibility). + The ``config.yml`` file takes precedence over the LaTeX source. + + Example of how the ``title`` tag may look like: + \title{A problem} + """ + if 'title' in self.config: + self.full_name = self.config['title'] + return + + try: + source = self.get_in_doc_dir(self.short_name + 'zad.tex') + text = source.read() + r = re.search(r'^[^%]*\\title{(.+)}', text, re.MULTILINE) + if r is not None: + self.full_name = r.group(1) + except FileNotFoundError: + pass + + def _detect_full_name_translations(self): + """Creates problem's full name translations from the ``config.yml`` + (keys matching the pattern ``title_[a-z]{2}``, where ``[a-z]{2}`` represents + two-letter language code defined in ``settings.py``), if any such key is given. + """ + self.lang_titles = {} + for lang_code, lang in self._get_from_django_settings('LANGUAGES', [('en', 'English')]): + key = 'title_%s' % lang_code + if key in self.config: + self.lang_titles[lang_code] = self.config[key] + + def get_submittable_extensions(self): + """ + Returns a list of extensions that are submittable. + """ + return self.config.get( + 'submittable_langs', + self._get_from_django_settings('SUBMITTABLE_LANGUAGES', ['c', 'cpp', 'cxx', 'py']) + ) + + def get_model_solution_regex(self): + """ + Returns the regex used to determine model solutions. + """ + extensions = self.get_submittable_extensions() + return rf'^{self.short_name}[0-9]*([bs]?)[0-9]*(_.*)?\.(' + '|'.join(extensions) + ')' + + def _get_model_solutions(self) -> list[tuple[ModelSolutionKind, str]]: + """ + Returns a list of model solutions, where each element is a tuple of model solution kind and filename. + """ + regex = self.get_model_solution_regex() + model_solutions = [] + for file in os.listdir(self.get_prog_dir()): + match = re.match(regex, file) + if re.match(regex, file) and os.path.isfile(os.path.join(self.get_prog_dir(), file)): + model_solutions.append((ModelSolutionKind.from_regex(match.group(1)), file)) + + return model_solutions + + def sort_model_solutions(self, model_solutions: list[tuple[ModelSolutionKind, str]]) -> list[tuple[ModelSolutionKind, str]]: + """ + Sorts model solutions by kind. + """ + def sort_key(model_solution): + kind, name = model_solution + return kind.value, naturalsort_key(name[: name.index(".")]) + return list(sorted(model_solutions, key=sort_key)) + + def _process_prog_files(self): + """ + Process all files in the problem's program directory that are used. + Saves all models solution files. If the problem has a custom graph file, + takes the files that are used in the graph. Otherwise, ingen, inwer and + files in `extra_compilation_files` and `extra_execution_files` from config + are saved. + """ + + # Process model solutions. + self.model_solutions = self.sort_model_solutions(self._get_model_solutions()) + + if self.has_custom_graph: + self.additional_files = self.graph_manager.get_prog_files() + else: + self.additional_files = [] + self.additional_files.extend(self.config.get('extra_compilation_files', [])) + self.additional_files.extend(self.config.get('extra_execution_files', [])) + extensions = self.get_submittable_extensions() + self.special_files: dict[str, bool] = {} + for file in ('ingen', 'inwer', 'soc', 'chk'): + try: + self.additional_files.append(File.get_file_matching_extension(self.get_prog_dir(), self.short_name + file, extensions).filename) + self.special_files[file] = True + except FileNotFoundError: + self.special_files[file] = False + + def _process_statements(self): + """ + Creates a problem statement from html or pdf source. + + TODO: we have to support this somehow, but we can't use makefiles. Probably a job for sio3workers. + If `USE_SINOLPACK_MAKEFILES` is set to True in the OIOIOI settings, + the pdf file will be compiled from a LaTeX source. + """ + docdir = self.get_doc_dir() + if not os.path.exists(docdir): + return + + lang_prefs = [''] + ['-' + l[0] for l in self._get_from_django_settings('LANGUAGES', [('en', 'English'), ('pl', 'Polish')])] + + self.lang_statements = {} + for lang in lang_prefs: + try: + htmlzipfile = self.get_in_doc_dir(self.short_name + 'zad' + lang + '.html.zip') + # TODO: what to do with html? + # if self._html_disallowed(): + # raise ProblemPackageError( + # _( + # "You cannot upload package with " + # "problem statement in HTML. " + # "Try again using PDF format." + # ) + # ) + # + # self._force_index_encoding(htmlzipfile) + # statement = ProblemStatement(problem=self.problem, language=lang[1:]) + # statement.content.save( + # self.short_name + lang + '.html.zip', File(open(htmlzipfile, 'rb')) + # ) + except FileNotFoundError: + pass + + try: + pdffile = self.get_in_doc_dir(self.short_name + 'zad' + lang + '.pdf') + if lang == '': + self.statement = pdffile + else: + self.lang_statements[lang[1:]] = pdffile + except: + pass + + def _process_attachments(self): + """ + + """ + attachments_dir = self.get_attachments_dir() + if not os.path.isdir(attachments_dir): + return + self.attachments = [ + attachment + for attachment in os.listdir(attachments_dir) + if os.path.isfile(os.path.join(attachments_dir, attachment)) + ] + + def get_unpack_graph(self) -> GraphOperation | None: + try: + return GraphOperation( + self.graph_manager.get('unpack'), + True, + self._unpack_return_data, + ) + except KeyError: + return None + + def _unpack_return_data(self, data: dict): + """ + Adds data received from the unpack operation to the package. + """ + # TODO: implement. The unpack will probably return tests, so we need to process them. + pass diff --git a/src/sio3pack/util.py b/src/sio3pack/util.py new file mode 100644 index 0000000..0bc26a3 --- /dev/null +++ b/src/sio3pack/util.py @@ -0,0 +1,9 @@ +import re +import tarfile +import zipfile + +from sio3pack.files.file import File + +def naturalsort_key(key): + convert = lambda text: int(text) if text.isdigit() else text + return [convert(c) for c in re.split('([0-9]+)', key)] diff --git a/src/sio3pack/utils.py b/src/sio3pack/utils.py deleted file mode 100644 index a439192..0000000 --- a/src/sio3pack/utils.py +++ /dev/null @@ -1,41 +0,0 @@ -import tarfile -import zipfile - -from sio3pack.files.file import File - - -def is_archive(file: str|File): - if isinstance(file, File): - file_path = file.path - else: - file_path = file - - try: - if zipfile.is_zipfile(file_path): - return True - - if tarfile.is_tarfile(file_path): - return True - except: - pass - - return False - -def has_dir(file: str|File, dir_name: str): - if isinstance(file, File): - file_path = file.path - else: - file_path = file - - try: - if zipfile.is_zipfile(file_path): - archive = zipfile.ZipFile(file_path) - return dir_name in map(lambda f: f.filename, archive.infolist()) - - if tarfile.is_tarfile(file_path): - archive = tarfile.TarFile(file_path) - return dir_name in map(lambda f: f.name, archive.getmembers()) - except: - pass - - return False diff --git a/src/sio3pack/utils/__init__.py b/src/sio3pack/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/sio3pack/utils/archive.py b/src/sio3pack/utils/archive.py new file mode 100644 index 0000000..1a86127 --- /dev/null +++ b/src/sio3pack/utils/archive.py @@ -0,0 +1,250 @@ +# Taken from +# https://github.com/gdub/python-archive/blob/master/archive/__init__.py + +# Copyright (c) Gary Wilson Jr. and contributors. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import os +import tarfile +import zipfile + + +class ArchiveException(RuntimeError): + """Base exception class for all archive errors.""" + + +class UnrecognizedArchiveFormat(ArchiveException): + """Error raised when passed file is not a recognized archive format.""" + + +class UnsafeArchive(ArchiveException): + """ + Error raised when passed file contains paths that would be extracted + outside of the target directory. + """ + + +def extract(path, to_path='', ext='', **kwargs): + """ + Unpack the tar or zip file at the specified path to the directory + specified by to_path. + """ + Archive(path, ext=ext).extract(to_path, **kwargs) + + +class Archive(object): + """ + The external API class that encapsulates an archive implementation. + """ + + def __init__(self, file, ext=''): + """ + Arguments: + * 'file' can be a string path to a file or a file-like object. + * Optional 'ext' argument can be given to override the file-type + guess that is normally performed using the file extension of the + given 'file'. Should start with a dot, e.g. '.tar.gz'. + """ + self.filename = file + self._archive = self._archive_cls(self.filename, ext=ext)(self.filename) + + @staticmethod + def _archive_cls(file, ext=''): + """ + Return the proper Archive implementation class, based on the file type. + """ + cls = None + filename = None + if isinstance(file, str): + filename = file + else: + try: + filename = file.name + except AttributeError: + raise UnrecognizedArchiveFormat( + "File object not a recognized archive format." + ) + lookup_filename = filename + ext + base, tail_ext = os.path.splitext(lookup_filename.lower()) + cls = extension_map.get(tail_ext) + if not cls: + base, ext = os.path.splitext(base) + cls = extension_map.get(ext) + if not cls: + raise UnrecognizedArchiveFormat( + "Path not a recognized archive format: %s" % filename + ) + return cls + + @classmethod + def is_archive(cls, file: str) -> bool: + """ + Check if the file is a recognized archive format. + """ + try: + cls._archive_cls(file) + return True + except UnrecognizedArchiveFormat: + return False + + def extract(self, *args, **kwargs): + self._archive.extract(*args, **kwargs) + + def filenames(self): + return self._archive.filenames() + + def dirnames(self): + return self._archive.dirnames() + + def extracted_size(self): + return self._archive.extracted_size() + + +class BaseArchive(object): + """ + Base Archive class. Implementations should inherit this class. + """ + + def __del__(self): + if hasattr(self, "_archive"): + self._archive.close() + + def filenames(self): + """ + Return a list of the filenames contained in the archive. + """ + raise NotImplementedError() + + def dirnames(self): + """ + Return a list of the dirnames contained in the archive. + """ + raise NotImplementedError() + + def extracted_size(self): + """ + Return total file size of extracted files in bytes. + """ + raise NotImplementedError() + + def _extract(self, to_path): + """ + Performs the actual extraction. Separate from 'extract' method so that + we don't recurse when subclasses don't declare their own 'extract' + method. + """ + self._archive.extractall(to_path) + + def extract(self, to_path='', method='safe'): + if method == 'safe': + self.check_files(to_path) + elif method == 'insecure': + pass + else: + raise ValueError("Invalid method option") + self._extract(to_path) + + def check_files(self, to_path=None): + """ + Check that all of the files contained in the archive are within the + target directory. + """ + if to_path: + target_path = os.path.normpath(os.path.realpath(to_path)) + else: + target_path = os.getcwd() + for filename in self.filenames(): + extract_path = os.path.join(target_path, filename) + extract_path = os.path.normpath(os.path.realpath(extract_path)) + if not extract_path.startswith(target_path): + raise UnsafeArchive( + "Archive member destination is outside the target" + " directory. member: %s" % filename + ) + + +class TarArchive(BaseArchive): + def __init__(self, file): + # tarfile's open uses different parameters for file path vs. file obj. + if isinstance(file, str): + self._archive = tarfile.open(name=file) + else: + self._archive = tarfile.open(fileobj=file) + + def filenames(self): + return [ + tarinfo.name for tarinfo in self._archive.getmembers() if tarinfo.isfile() + ] + + def dirnames(self): + return [ + tarinfo.name for tarinfo in self._archive.getmembers() if tarinfo.isdir() + ] + + def extracted_size(self): + total = 0 + for member in self._archive: + total += member.size + return total + + def check_files(self, to_path=None): + BaseArchive.check_files(self, to_path) + + for finfo in self._archive: + if finfo.issym(): + raise UnsafeArchive("Archive contains symlink: " + finfo.name) + if finfo.islnk(): + raise UnsafeArchive("Archive contains hardlink: " + finfo.name) + + +class ZipArchive(BaseArchive): + def __init__(self, file): + # ZipFile's 'file' parameter can be path (string) or file-like obj. + self._archive = zipfile.ZipFile(file) + + def extracted_size(self): + total = 0 + for member in self._archive.infolist(): + total += member.file_size + return total + + def filenames(self): + return [ + zipinfo.filename + for zipinfo in self._archive.infolist() + if not zipinfo.is_dir() + ] + + def dirnames(self): + return [ + zipinfo.filename + for zipinfo in self._archive.infolist() + if zipinfo.is_dir() + ] + + +extension_map = { + '.tar': TarArchive, + '.tar.bz2': TarArchive, + '.tar.gz': TarArchive, + '.tgz': TarArchive, + '.tz2': TarArchive, + '.zip': ZipArchive, +} diff --git a/src/sio3pack/utils/classinit.py b/src/sio3pack/utils/classinit.py new file mode 100644 index 0000000..5330916 --- /dev/null +++ b/src/sio3pack/utils/classinit.py @@ -0,0 +1,90 @@ +# From oioioi/base/utils/__init__.py + +class ClassInitMeta(type): + """Meta class triggering __classinit__ on class intialization.""" + + def __init__(cls, class_name, bases, new_attrs): + super(ClassInitMeta, cls).__init__(class_name, bases, new_attrs) + cls.__classinit__() + + +class ClassInitBase(object, metaclass=ClassInitMeta): + """Abstract base class injecting ClassInitMeta meta class.""" + + @classmethod + def __classinit__(cls): + """ + Empty __classinit__ implementation. + + This must be a no-op as subclasses can't reliably call base class's + __classinit__ from their __classinit__s. + + Subclasses of __classinit__ should look like: + + .. python:: + + class MyClass(ClassInitBase): + + @classmethod + def __classinit__(cls): + # Need globals().get as MyClass may be still undefined. + super(globals().get('MyClass', cls), + cls).__classinit__() + ... + + class Derived(MyClass): + + @classmethod + def __classinit__(cls): + super(globals().get('Derived', cls), + cls).__classinit__() + ... + """ + pass + + +class RegisteredSubclassesBase(ClassInitBase): + """A base class for classes which should have a list of subclasses + available. + + The list of subclasses is available in their :attr:`subclasses` class + attributes. Classes which have *explicitly* set :attr:`abstract` class + attribute to ``True`` are not added to :attr:`subclasses`. + """ + + _subclasses_loaded = False + + @classmethod + def __classinit__(cls): + this_cls = globals().get('RegisteredSubclassesBase', cls) + super(this_cls, cls).__classinit__() + if this_cls is cls: + # This is RegisteredSubclassesBase class. + return + + assert 'subclasses' not in cls.__dict__, ( + '%s defines attribute subclasses, but has ' + 'RegisteredSubclassesMeta metaclass' % (cls,) + ) + cls.subclasses = [] + cls.abstract = cls.__dict__.get('abstract', False) + + def find_superclass(cls): + superclasses = [c for c in cls.__bases__ if issubclass(c, this_cls)] + if not superclasses: + return None + if len(superclasses) > 1: + raise AssertionError( + '%s derives from more than one ' + 'RegisteredSubclassesBase' % (cls.__name__,) + ) + superclass = superclasses[0] + return superclass + + # Add the class to all superclasses' 'subclasses' attribute, including + # self. + superclass = cls + while superclass is not this_cls: + if not cls.abstract: + superclass.subclasses.append(cls) + superclass = find_superclass(superclass) From c151cc44349931d056020106b43c91e58c46017b Mon Sep 17 00:00:00 2001 From: MasloMaslane Date: Sat, 30 Nov 2024 16:48:02 +0100 Subject: [PATCH 3/9] Add testing framework --- setup.cfg | 3 +- tests/__init__.py | 0 tests/fixtures.py | 99 ++++++++++++++++++++++ tests/packages/__init__.py | 0 tests/packages/sinolpack/__init__.py | 0 tests/packages/sinolpack/test_sinolpack.py | 21 +++++ tests/test_packages/simple/__init__.py | 2 + tests/test_packages/simple/config.yml | 1 + tests/test_packages/simple/in/.gitkeep | 0 tests/test_packages/simple/out/.gitkeep | 0 tests/test_packages/simple/prog/.gitkeep | 0 11 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 tests/__init__.py create mode 100644 tests/fixtures.py create mode 100644 tests/packages/__init__.py create mode 100644 tests/packages/sinolpack/__init__.py create mode 100644 tests/packages/sinolpack/test_sinolpack.py create mode 100644 tests/test_packages/simple/__init__.py create mode 100644 tests/test_packages/simple/config.yml create mode 100644 tests/test_packages/simple/in/.gitkeep create mode 100644 tests/test_packages/simple/out/.gitkeep create mode 100644 tests/test_packages/simple/prog/.gitkeep diff --git a/setup.cfg b/setup.cfg index 47979f0..da4d6d0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,7 +21,8 @@ packages = find_namespace: packages_dir = src include_package_data = True python_requires = >=3.9 -install_requires = +install_requires = + PyYAML [options.packages.find] where = src diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures.py b/tests/fixtures.py new file mode 100644 index 0000000..5da0f01 --- /dev/null +++ b/tests/fixtures.py @@ -0,0 +1,99 @@ +import importlib.util +import os.path +import shutil +import tarfile +import tempfile +import zipfile +from enum import Enum + +import pytest + + +class Compression(Enum): + NONE = "" + ZIP = "zip" + TAR_GZ = "tar.gz" + TGZ = "tgz" + + +all_compressions = [c.value for c in Compression if c != Compression.NONE] + + +def _tar_archive(dir, dest, compression=None): + """ + Create a tar archive of the specified directory. + """ + if compression is None: + mode = 'w' + else: + mode = f'w:{compression}' + with tarfile.open(dest, mode) as tar: + tar.add(dir, arcname=os.path.basename(dir)) + + +def _zip_archive(dir, dest): + """ + Create a zip archive of the specified directory. + """ + with zipfile.ZipFile(dest, 'w') as zip: + for root, dirs, files in os.walk(dir): + for file in files: + zip.write(os.path.join(root, file), os.path.relpath(os.path.join(root, file), dir)) + + +def _create_package(package_name, tmpdir, archive=False, extension="zip"): + packages = os.path.join(os.path.dirname(__file__), "test_packages") + if not os.path.exists(os.path.join(packages, package_name)): + raise FileNotFoundError(f"Package {package_name} does not exist") + + spec = importlib.util.spec_from_file_location(package_name, os.path.join(packages, package_name, "__init__.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + task_id = module.TASK_ID + type = module.TYPE + package_path = os.path.join(tmpdir.name, task_id) + shutil.copytree(os.path.join(packages, package_name), package_path) + shutil.rmtree(os.path.join(package_path, "__pycache__"), ignore_errors=True) + os.unlink(os.path.join(package_path, "__init__.py")) + + if archive: + if extension == "zip": + _zip_archive(package_path, os.path.join(tmpdir.name, f"{task_id}.zip")) + elif extension == "tar": + _tar_archive(package_path, os.path.join(tmpdir.name, f"{task_id}.tar")) + elif extension == "tar.gz" or extension == "tgz": + _tar_archive(package_path, os.path.join(tmpdir.name, f"{task_id}.{extension}"), "gz") + else: + raise ValueError(f"Unknown extension {extension}") + package_path = os.path.join(tmpdir.name, f"{task_id}.{extension}") + + return package_path, type + +@pytest.fixture +def package(request): + """ + Fixture to create a temporary directory with specified package. + """ + package_name = request.param + tmpdir = tempfile.TemporaryDirectory() + package_path, type = _create_package(package_name, tmpdir) + + yield package_path, type + + tmpdir.cleanup() + + +@pytest.fixture +def package_archived(request): + """ + Fixture to create a temporary directory with specified package, but archived. + """ + package_name, extension = request.param + archive = extension != Compression.NONE + tmpdir = tempfile.TemporaryDirectory() + package_path, type = _create_package(package_name, tmpdir, archive, extension) + + yield package_path, type + + tmpdir.cleanup() diff --git a/tests/packages/__init__.py b/tests/packages/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/packages/sinolpack/__init__.py b/tests/packages/sinolpack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/packages/sinolpack/test_sinolpack.py b/tests/packages/sinolpack/test_sinolpack.py new file mode 100644 index 0000000..f4c5351 --- /dev/null +++ b/tests/packages/sinolpack/test_sinolpack.py @@ -0,0 +1,21 @@ +import os.path + +import pytest + +from tests.fixtures import package, package_archived, Compression, all_compressions + + +@pytest.mark.parametrize("package", ["simple"], indirect=True) +def test_simple(package): + package_path, type = package + assert type == "sinolpack" + print(os.listdir(package_path)) + assert os.path.isdir(package_path) + + +@pytest.mark.parametrize("package_archived", [("simple", c) for c in all_compressions], indirect=True) +def test_archive(package_archived): + package_path, type = package_archived + assert type == "sinolpack" + print(package_path) + assert os.path.isfile(package_path) diff --git a/tests/test_packages/simple/__init__.py b/tests/test_packages/simple/__init__.py new file mode 100644 index 0000000..b2b61e9 --- /dev/null +++ b/tests/test_packages/simple/__init__.py @@ -0,0 +1,2 @@ +TASK_ID = 'abc' +TYPE = 'sinolpack' diff --git a/tests/test_packages/simple/config.yml b/tests/test_packages/simple/config.yml new file mode 100644 index 0000000..020745c --- /dev/null +++ b/tests/test_packages/simple/config.yml @@ -0,0 +1 @@ +title: Simple package diff --git a/tests/test_packages/simple/in/.gitkeep b/tests/test_packages/simple/in/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_packages/simple/out/.gitkeep b/tests/test_packages/simple/out/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_packages/simple/prog/.gitkeep b/tests/test_packages/simple/prog/.gitkeep new file mode 100644 index 0000000..e69de29 From 747532dd8c0e922fcc9d898b6a2ee1fa2337d744 Mon Sep 17 00:00:00 2001 From: MasloMaslane Date: Sat, 30 Nov 2024 16:48:11 +0100 Subject: [PATCH 4/9] Add workflows for tests and release --- .github/workflows/arch.yml | 28 +++++++++++++++++++++++++ .github/workflows/macOS.yml | 39 +++++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 37 +++++++++++++++++++++++++++++++++ .github/workflows/ubuntu.yml | 28 +++++++++++++++++++++++++ tests/.gitkeep | 0 5 files changed, 132 insertions(+) create mode 100644 .github/workflows/arch.yml create mode 100644 .github/workflows/macOS.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/ubuntu.yml delete mode 100644 tests/.gitkeep diff --git a/.github/workflows/arch.yml b/.github/workflows/arch.yml new file mode 100644 index 0000000..0c4a281 --- /dev/null +++ b/.github/workflows/arch.yml @@ -0,0 +1,28 @@ +name: pytest-arch +run-name: Run pytest on Arch Linux +on: + push: + branches: 'main' + pull_request: +jobs: + pytest: + runs-on: archlinux:latest + strategy: + matrix: + python-version: ["3.9", "3.13"] + name: pytest-arch-python-${{ matrix.python-version }} + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip3 install .[tests] + - name: Run pytest + env: + PYTEST_ADDOPTS: "--color=yes" + run: | + python3 -m pytest -v -n auto diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml new file mode 100644 index 0000000..0d379c1 --- /dev/null +++ b/.github/workflows/macOS.yml @@ -0,0 +1,39 @@ +name: pytest-macos +run-name: Run pytest on macOS +on: + push: + branches: 'main' + pull_request: + +jobs: + pytest: + runs-on: macos-latest + strategy: + matrix: + python-version: ["3.9", "3.13"] + name: pytest-macos-python-${{ matrix.python-version }} + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Set up Homebrew + uses: Homebrew/actions/setup-homebrew@master + - name: Install Homebrew dependencies + run: | + rm -f /usr/local/bin/2to3* /usr/local/bin/python3* /usr/local/bin/idle3* \ + /usr/local/bin/pydoc3* # Homebrew will fail if these exist + brew install virtualenv + - name: Install Python dependencies + run: | + python3 -m venv .venv + source .venv/bin/activate + pip install .[tests] + - name: Run pytest + env: + PYTEST_ADDOPTS: "--color=yes" + run: | + source .venv/bin/activate + pytest -v -n auto diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..fb96a52 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,37 @@ +name: publish +run-name: Publish to PyPi +on: + release: + types: [published] + +permissions: + contents: write + +jobs: + build-and-publish: + name: Build and publish to PyPi + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.11 + - name: Install build + run: | + python3 -m pip install build + python3 -m pip install setuptools --upgrade + - name: Build + run: | + python3 -m build --sdist --wheel --outdir dist/ . + - name: Upload to release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload ${{ github.event.release.tag_name }} dist/* + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.TOKEN }} diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml new file mode 100644 index 0000000..9256238 --- /dev/null +++ b/.github/workflows/ubuntu.yml @@ -0,0 +1,28 @@ +name: pytest-ubuntu +run-name: Run pytest on Ubuntu +on: + push: + branches: 'main' + pull_request: +jobs: + pytest: + runs-on: ubuntu:latest + strategy: + matrix: + python-version: ["3.9", "3.13"] + name: pytest-ubuntu-python-${{ matrix.python-version }} + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip3 install .[tests] + - name: Run pytest + env: + PYTEST_ADDOPTS: "--color=yes" + run: | + python3 -m pytest -v -n auto diff --git a/tests/.gitkeep b/tests/.gitkeep deleted file mode 100644 index e69de29..0000000 From c577483fd2c015c6731c87782fed87cf6c4f7270 Mon Sep 17 00:00:00 2001 From: MasloMaslane Date: Sat, 30 Nov 2024 16:50:45 +0100 Subject: [PATCH 5/9] Reformat --- src/sio3pack/packages/package/model.py | 3 +- src/sio3pack/packages/sinolpack/enums.py | 6 +- src/sio3pack/packages/sinolpack/model.py | 83 ++++++++++++---------- src/sio3pack/util.py | 5 +- src/sio3pack/utils/archive.py | 55 +++++--------- src/sio3pack/utils/classinit.py | 17 ++--- tests/fixtures.py | 7 +- tests/packages/sinolpack/test_sinolpack.py | 2 +- tests/test_packages/simple/__init__.py | 4 +- 9 files changed, 85 insertions(+), 97 deletions(-) diff --git a/src/sio3pack/packages/package/model.py b/src/sio3pack/packages/package/model.py index cea7afd..85972e1 100644 --- a/src/sio3pack/packages/package/model.py +++ b/src/sio3pack/packages/package/model.py @@ -4,8 +4,6 @@ from sio3pack.graph.graph import Graph from sio3pack.packages.exceptions import UnknownPackageType from sio3pack.test.test import Test - -from sio3pack.packages import all_packages from sio3pack.utils.archive import Archive from sio3pack.utils.classinit import RegisteredSubclassesBase @@ -14,6 +12,7 @@ class Package(RegisteredSubclassesBase): """ Base class for all packages. """ + abstract = True def __init__(self, file: File): diff --git a/src/sio3pack/packages/sinolpack/enums.py b/src/sio3pack/packages/sinolpack/enums.py index 6f81e47..0f44533 100644 --- a/src/sio3pack/packages/sinolpack/enums.py +++ b/src/sio3pack/packages/sinolpack/enums.py @@ -8,10 +8,10 @@ class ModelSolutionKind(Enum): @classmethod def from_regex(cls, group): - if group == '': + if group == "": return cls.NORMAL - if group == 's': + if group == "s": return cls.SLOW - if group == 'b': + if group == "b": return cls.INCORRECT raise ValueError(f"Invalid model solution kind: {group}") diff --git a/src/sio3pack/packages/sinolpack/model.py b/src/sio3pack/packages/sinolpack/model.py index 374f5e1..65fb3b8 100644 --- a/src/sio3pack/packages/sinolpack/model.py +++ b/src/sio3pack/packages/sinolpack/model.py @@ -1,9 +1,9 @@ -import json import os import re -import yaml import tempfile +import yaml + from sio3pack.files.file import File from sio3pack.graph.graph import Graph from sio3pack.graph.graph_manager import GraphManager @@ -26,7 +26,7 @@ def _find_main_dir(cls, archive: Archive) -> str | None: toplevel_dirs = list(set(f.split(os.sep)[0] for f in dirs)) problem_dirs = [] for dir in toplevel_dirs: - for required_subdir in ('in', 'out'): + for required_subdir in ("in", "out"): if all(f.split(os.sep)[:2] != [dir, required_subdir] for f in dirs): break else: @@ -68,7 +68,7 @@ def __init__(self, file: File, django_settings=None): self.rootdir = file.path try: - graph_file = self.get_in_root('graph.json') + graph_file = self.get_in_root("graph.json") self.graph_manager = GraphManager.from_file(graph_file) except FileNotFoundError: self.has_custom_graph = False @@ -76,12 +76,16 @@ def __init__(self, file: File, django_settings=None): self.django_settings = django_settings def _default_graph_manager(self) -> GraphManager: - return GraphManager({ - "unpack": Graph.from_dict({ - "name": "unpack", - # ... - }) - }) + return GraphManager( + { + "unpack": Graph.from_dict( + { + "name": "unpack", + # ... + } + ) + } + ) def _get_from_django_settings(self, key: str, default=None): if self.django_settings is None: @@ -92,7 +96,7 @@ def get_doc_dir(self) -> str: """ Returns the path to the directory containing the problem's documents. """ - return os.path.join(self.rootdir, 'doc') + return os.path.join(self.rootdir, "doc") def get_in_doc_dir(self, filename: str) -> File: """ @@ -110,7 +114,7 @@ def get_prog_dir(self) -> str: """ Returns the path to the directory containing the problem's program files. """ - return os.path.join(self.rootdir, 'prog') + return os.path.join(self.rootdir, "prog") def get_in_prog_dir(self, filename: str) -> File: """ @@ -122,7 +126,7 @@ def get_attachments_dir(self) -> str: """ Returns the path to the directory containing the problem's attachments. """ - return os.path.join(self.rootdir, 'attachments') + return os.path.join(self.rootdir, "attachments") def _process_package(self): self._process_config_yml() @@ -141,7 +145,7 @@ def _process_config_yml(self): Process the config.yml file. If it exists, it will be loaded into the config attribute. """ try: - config = self.get_in_root('config.yml') + config = self.get_in_root("config.yml") self.config = yaml.safe_load(config.read()) except FileNotFoundError: self.config = {} @@ -155,14 +159,14 @@ def _detect_full_name(self): Example of how the ``title`` tag may look like: \title{A problem} """ - if 'title' in self.config: - self.full_name = self.config['title'] + if "title" in self.config: + self.full_name = self.config["title"] return try: - source = self.get_in_doc_dir(self.short_name + 'zad.tex') + source = self.get_in_doc_dir(self.short_name + "zad.tex") text = source.read() - r = re.search(r'^[^%]*\\title{(.+)}', text, re.MULTILINE) + r = re.search(r"^[^%]*\\title{(.+)}", text, re.MULTILINE) if r is not None: self.full_name = r.group(1) except FileNotFoundError: @@ -174,8 +178,8 @@ def _detect_full_name_translations(self): two-letter language code defined in ``settings.py``), if any such key is given. """ self.lang_titles = {} - for lang_code, lang in self._get_from_django_settings('LANGUAGES', [('en', 'English')]): - key = 'title_%s' % lang_code + for lang_code, lang in self._get_from_django_settings("LANGUAGES", [("en", "English")]): + key = "title_%s" % lang_code if key in self.config: self.lang_titles[lang_code] = self.config[key] @@ -184,8 +188,7 @@ def get_submittable_extensions(self): Returns a list of extensions that are submittable. """ return self.config.get( - 'submittable_langs', - self._get_from_django_settings('SUBMITTABLE_LANGUAGES', ['c', 'cpp', 'cxx', 'py']) + "submittable_langs", self._get_from_django_settings("SUBMITTABLE_LANGUAGES", ["c", "cpp", "cxx", "py"]) ) def get_model_solution_regex(self): @@ -193,7 +196,7 @@ def get_model_solution_regex(self): Returns the regex used to determine model solutions. """ extensions = self.get_submittable_extensions() - return rf'^{self.short_name}[0-9]*([bs]?)[0-9]*(_.*)?\.(' + '|'.join(extensions) + ')' + return rf"^{self.short_name}[0-9]*([bs]?)[0-9]*(_.*)?\.(" + "|".join(extensions) + ")" def _get_model_solutions(self) -> list[tuple[ModelSolutionKind, str]]: """ @@ -208,13 +211,17 @@ def _get_model_solutions(self) -> list[tuple[ModelSolutionKind, str]]: return model_solutions - def sort_model_solutions(self, model_solutions: list[tuple[ModelSolutionKind, str]]) -> list[tuple[ModelSolutionKind, str]]: + def sort_model_solutions( + self, model_solutions: list[tuple[ModelSolutionKind, str]] + ) -> list[tuple[ModelSolutionKind, str]]: """ Sorts model solutions by kind. """ + def sort_key(model_solution): kind, name = model_solution return kind.value, naturalsort_key(name[: name.index(".")]) + return list(sorted(model_solutions, key=sort_key)) def _process_prog_files(self): @@ -233,13 +240,17 @@ def _process_prog_files(self): self.additional_files = self.graph_manager.get_prog_files() else: self.additional_files = [] - self.additional_files.extend(self.config.get('extra_compilation_files', [])) - self.additional_files.extend(self.config.get('extra_execution_files', [])) + self.additional_files.extend(self.config.get("extra_compilation_files", [])) + self.additional_files.extend(self.config.get("extra_execution_files", [])) extensions = self.get_submittable_extensions() self.special_files: dict[str, bool] = {} - for file in ('ingen', 'inwer', 'soc', 'chk'): + for file in ("ingen", "inwer", "soc", "chk"): try: - self.additional_files.append(File.get_file_matching_extension(self.get_prog_dir(), self.short_name + file, extensions).filename) + self.additional_files.append( + File.get_file_matching_extension( + self.get_prog_dir(), self.short_name + file, extensions + ).filename + ) self.special_files[file] = True except FileNotFoundError: self.special_files[file] = False @@ -256,12 +267,14 @@ def _process_statements(self): if not os.path.exists(docdir): return - lang_prefs = [''] + ['-' + l[0] for l in self._get_from_django_settings('LANGUAGES', [('en', 'English'), ('pl', 'Polish')])] + lang_prefs = [""] + [ + "-" + l[0] for l in self._get_from_django_settings("LANGUAGES", [("en", "English"), ("pl", "Polish")]) + ] self.lang_statements = {} for lang in lang_prefs: try: - htmlzipfile = self.get_in_doc_dir(self.short_name + 'zad' + lang + '.html.zip') + htmlzipfile = self.get_in_doc_dir(self.short_name + "zad" + lang + ".html.zip") # TODO: what to do with html? # if self._html_disallowed(): # raise ProblemPackageError( @@ -281,8 +294,8 @@ def _process_statements(self): pass try: - pdffile = self.get_in_doc_dir(self.short_name + 'zad' + lang + '.pdf') - if lang == '': + pdffile = self.get_in_doc_dir(self.short_name + "zad" + lang + ".pdf") + if lang == "": self.statement = pdffile else: self.lang_statements[lang[1:]] = pdffile @@ -290,9 +303,7 @@ def _process_statements(self): pass def _process_attachments(self): - """ - - """ + """ """ attachments_dir = self.get_attachments_dir() if not os.path.isdir(attachments_dir): return @@ -305,7 +316,7 @@ def _process_attachments(self): def get_unpack_graph(self) -> GraphOperation | None: try: return GraphOperation( - self.graph_manager.get('unpack'), + self.graph_manager.get("unpack"), True, self._unpack_return_data, ) diff --git a/src/sio3pack/util.py b/src/sio3pack/util.py index 0bc26a3..855f96f 100644 --- a/src/sio3pack/util.py +++ b/src/sio3pack/util.py @@ -1,9 +1,6 @@ import re -import tarfile -import zipfile -from sio3pack.files.file import File def naturalsort_key(key): convert = lambda text: int(text) if text.isdigit() else text - return [convert(c) for c in re.split('([0-9]+)', key)] + return [convert(c) for c in re.split("([0-9]+)", key)] diff --git a/src/sio3pack/utils/archive.py b/src/sio3pack/utils/archive.py index 1a86127..847eb81 100644 --- a/src/sio3pack/utils/archive.py +++ b/src/sio3pack/utils/archive.py @@ -41,7 +41,7 @@ class UnsafeArchive(ArchiveException): """ -def extract(path, to_path='', ext='', **kwargs): +def extract(path, to_path="", ext="", **kwargs): """ Unpack the tar or zip file at the specified path to the directory specified by to_path. @@ -54,7 +54,7 @@ class Archive(object): The external API class that encapsulates an archive implementation. """ - def __init__(self, file, ext=''): + def __init__(self, file, ext=""): """ Arguments: * 'file' can be a string path to a file or a file-like object. @@ -66,7 +66,7 @@ def __init__(self, file, ext=''): self._archive = self._archive_cls(self.filename, ext=ext)(self.filename) @staticmethod - def _archive_cls(file, ext=''): + def _archive_cls(file, ext=""): """ Return the proper Archive implementation class, based on the file type. """ @@ -78,9 +78,7 @@ def _archive_cls(file, ext=''): try: filename = file.name except AttributeError: - raise UnrecognizedArchiveFormat( - "File object not a recognized archive format." - ) + raise UnrecognizedArchiveFormat("File object not a recognized archive format.") lookup_filename = filename + ext base, tail_ext = os.path.splitext(lookup_filename.lower()) cls = extension_map.get(tail_ext) @@ -88,9 +86,7 @@ def _archive_cls(file, ext=''): base, ext = os.path.splitext(base) cls = extension_map.get(ext) if not cls: - raise UnrecognizedArchiveFormat( - "Path not a recognized archive format: %s" % filename - ) + raise UnrecognizedArchiveFormat("Path not a recognized archive format: %s" % filename) return cls @classmethod @@ -152,10 +148,10 @@ def _extract(self, to_path): """ self._archive.extractall(to_path) - def extract(self, to_path='', method='safe'): - if method == 'safe': + def extract(self, to_path="", method="safe"): + if method == "safe": self.check_files(to_path) - elif method == 'insecure': + elif method == "insecure": pass else: raise ValueError("Invalid method option") @@ -175,8 +171,7 @@ def check_files(self, to_path=None): extract_path = os.path.normpath(os.path.realpath(extract_path)) if not extract_path.startswith(target_path): raise UnsafeArchive( - "Archive member destination is outside the target" - " directory. member: %s" % filename + "Archive member destination is outside the target" " directory. member: %s" % filename ) @@ -189,14 +184,10 @@ def __init__(self, file): self._archive = tarfile.open(fileobj=file) def filenames(self): - return [ - tarinfo.name for tarinfo in self._archive.getmembers() if tarinfo.isfile() - ] + return [tarinfo.name for tarinfo in self._archive.getmembers() if tarinfo.isfile()] def dirnames(self): - return [ - tarinfo.name for tarinfo in self._archive.getmembers() if tarinfo.isdir() - ] + return [tarinfo.name for tarinfo in self._archive.getmembers() if tarinfo.isdir()] def extracted_size(self): total = 0 @@ -226,25 +217,17 @@ def extracted_size(self): return total def filenames(self): - return [ - zipinfo.filename - for zipinfo in self._archive.infolist() - if not zipinfo.is_dir() - ] + return [zipinfo.filename for zipinfo in self._archive.infolist() if not zipinfo.is_dir()] def dirnames(self): - return [ - zipinfo.filename - for zipinfo in self._archive.infolist() - if zipinfo.is_dir() - ] + return [zipinfo.filename for zipinfo in self._archive.infolist() if zipinfo.is_dir()] extension_map = { - '.tar': TarArchive, - '.tar.bz2': TarArchive, - '.tar.gz': TarArchive, - '.tgz': TarArchive, - '.tz2': TarArchive, - '.zip': ZipArchive, + ".tar": TarArchive, + ".tar.bz2": TarArchive, + ".tar.gz": TarArchive, + ".tgz": TarArchive, + ".tz2": TarArchive, + ".zip": ZipArchive, } diff --git a/src/sio3pack/utils/classinit.py b/src/sio3pack/utils/classinit.py index 5330916..a1356b4 100644 --- a/src/sio3pack/utils/classinit.py +++ b/src/sio3pack/utils/classinit.py @@ -1,5 +1,6 @@ # From oioioi/base/utils/__init__.py + class ClassInitMeta(type): """Meta class triggering __classinit__ on class intialization.""" @@ -56,28 +57,24 @@ class RegisteredSubclassesBase(ClassInitBase): @classmethod def __classinit__(cls): - this_cls = globals().get('RegisteredSubclassesBase', cls) + this_cls = globals().get("RegisteredSubclassesBase", cls) super(this_cls, cls).__classinit__() if this_cls is cls: # This is RegisteredSubclassesBase class. return - assert 'subclasses' not in cls.__dict__, ( - '%s defines attribute subclasses, but has ' - 'RegisteredSubclassesMeta metaclass' % (cls,) - ) + assert ( + "subclasses" not in cls.__dict__ + ), "%s defines attribute subclasses, but has " "RegisteredSubclassesMeta metaclass" % (cls,) cls.subclasses = [] - cls.abstract = cls.__dict__.get('abstract', False) + cls.abstract = cls.__dict__.get("abstract", False) def find_superclass(cls): superclasses = [c for c in cls.__bases__ if issubclass(c, this_cls)] if not superclasses: return None if len(superclasses) > 1: - raise AssertionError( - '%s derives from more than one ' - 'RegisteredSubclassesBase' % (cls.__name__,) - ) + raise AssertionError("%s derives from more than one " "RegisteredSubclassesBase" % (cls.__name__,)) superclass = superclasses[0] return superclass diff --git a/tests/fixtures.py b/tests/fixtures.py index 5da0f01..7e31c57 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -24,9 +24,9 @@ def _tar_archive(dir, dest, compression=None): Create a tar archive of the specified directory. """ if compression is None: - mode = 'w' + mode = "w" else: - mode = f'w:{compression}' + mode = f"w:{compression}" with tarfile.open(dest, mode) as tar: tar.add(dir, arcname=os.path.basename(dir)) @@ -35,7 +35,7 @@ def _zip_archive(dir, dest): """ Create a zip archive of the specified directory. """ - with zipfile.ZipFile(dest, 'w') as zip: + with zipfile.ZipFile(dest, "w") as zip: for root, dirs, files in os.walk(dir): for file in files: zip.write(os.path.join(root, file), os.path.relpath(os.path.join(root, file), dir)) @@ -70,6 +70,7 @@ def _create_package(package_name, tmpdir, archive=False, extension="zip"): return package_path, type + @pytest.fixture def package(request): """ diff --git a/tests/packages/sinolpack/test_sinolpack.py b/tests/packages/sinolpack/test_sinolpack.py index f4c5351..c591fad 100644 --- a/tests/packages/sinolpack/test_sinolpack.py +++ b/tests/packages/sinolpack/test_sinolpack.py @@ -2,7 +2,7 @@ import pytest -from tests.fixtures import package, package_archived, Compression, all_compressions +from tests.fixtures import Compression, all_compressions, package, package_archived @pytest.mark.parametrize("package", ["simple"], indirect=True) diff --git a/tests/test_packages/simple/__init__.py b/tests/test_packages/simple/__init__.py index b2b61e9..0f21af5 100644 --- a/tests/test_packages/simple/__init__.py +++ b/tests/test_packages/simple/__init__.py @@ -1,2 +1,2 @@ -TASK_ID = 'abc' -TYPE = 'sinolpack' +TASK_ID = "abc" +TYPE = "sinolpack" From 8e92a501955cfa630aac5527460999922f5f8659 Mon Sep 17 00:00:00 2001 From: MasloMaslane Date: Sat, 30 Nov 2024 18:04:50 +0100 Subject: [PATCH 6/9] Fix bugs, refactor --- src/sio3pack/__init__.py | 8 +-- src/sio3pack/files/__init__.py | 1 + src/sio3pack/files/file.py | 30 ++---------- src/sio3pack/files/filetracker_file.py | 2 + src/sio3pack/files/local_file.py | 28 +++++++++++ src/sio3pack/graph/__init__.py | 4 +- src/sio3pack/graph/graph.py | 4 +- src/sio3pack/graph/graph_manager.py | 2 +- src/sio3pack/packages/exceptions.py | 4 +- src/sio3pack/packages/package/model.py | 20 ++++---- src/sio3pack/packages/sinolpack/model.py | 57 ++++++++++++---------- src/sio3pack/test/__init__.py | 2 +- src/sio3pack/test/simple_test.py | 4 +- src/sio3pack/utils/archive.py | 11 ++++- tests/fixtures.py | 46 +++++++++++------ tests/packages/sinolpack/test_sinolpack.py | 29 ++++++----- tests/utils.py | 0 17 files changed, 149 insertions(+), 103 deletions(-) create mode 100644 tests/utils.py diff --git a/src/sio3pack/__init__.py b/src/sio3pack/__init__.py index 51fe899..9761228 100644 --- a/src/sio3pack/__init__.py +++ b/src/sio3pack/__init__.py @@ -1,10 +1,10 @@ __version__ = "0.0.1" -from sio3pack.files.file import File +from sio3pack.files import LocalFile from sio3pack.packages.package import Package -def from_file(file: str | File, django_settings=None) -> Package: +def from_file(file: str | LocalFile, django_settings=None) -> Package: """ Initialize a package object from a file (archive or directory). :param file: The file path or File object. @@ -12,5 +12,5 @@ def from_file(file: str | File, django_settings=None) -> Package: :return: The package object. """ if isinstance(file, str): - file = File(file) - return Package.from_file(file, django_settings) + file = LocalFile(file) + return Package.from_file(file, django_settings=django_settings) diff --git a/src/sio3pack/files/__init__.py b/src/sio3pack/files/__init__.py index bedfbe6..ca063b6 100644 --- a/src/sio3pack/files/__init__.py +++ b/src/sio3pack/files/__init__.py @@ -1,2 +1,3 @@ from sio3pack.files.filetracker_file import FiletrackerFile from sio3pack.files.local_file import LocalFile +from sio3pack.files.file import File diff --git a/src/sio3pack/files/file.py b/src/sio3pack/files/file.py index 779ec3a..1ae0908 100644 --- a/src/sio3pack/files/file.py +++ b/src/sio3pack/files/file.py @@ -1,36 +1,16 @@ -import os.path - - class File: """ Base class for all files in a package. """ - @classmethod - def get_file_matching_extension(cls, dir: str, filename: str, extensions: list[str]) -> "File": - """ - Get the file with the given filename and one of the given extensions. - :param dir: The directory to search in. - :param filename: The filename. - :param extensions: The extensions. - :return: The file object. - """ - for ext in extensions: - path = os.path.join(dir, filename + ext) - if os.path.exists(path): - return cls(path) - raise FileNotFoundError - def __init__(self, path: str): - if not os.path.exists(path): - raise FileNotFoundError self.path = path - self.filename = os.path.basename(path) + + def __str__(self): + return f"<{self.__class__.__name__} {self.path}>" def read(self) -> str: - with open(self.path, "r") as f: - return f.read() + raise NotImplementedError() def write(self, text: str): - with open(self.path, "w") as f: - f.write(text) + raise NotImplementedError() diff --git a/src/sio3pack/files/filetracker_file.py b/src/sio3pack/files/filetracker_file.py index 3698e2d..9ee022e 100644 --- a/src/sio3pack/files/filetracker_file.py +++ b/src/sio3pack/files/filetracker_file.py @@ -8,3 +8,5 @@ class FiletrackerFile(File): def __init__(self, path: str): super().__init__(path) + # TODO: should raise FileNotFoundError if file is not tracked + raise NotImplementedError() diff --git a/src/sio3pack/files/local_file.py b/src/sio3pack/files/local_file.py index 95f4009..43b1c15 100644 --- a/src/sio3pack/files/local_file.py +++ b/src/sio3pack/files/local_file.py @@ -1,3 +1,5 @@ +import os + from sio3pack.files.file import File @@ -6,5 +8,31 @@ class LocalFile(File): Base class for all files in a package that are stored locally. """ + @classmethod + def get_file_matching_extension(cls, dir: str, filename: str, extensions: list[str]) -> "LocalFile": + """ + Get the file with the given filename and one of the given extensions. + :param dir: The directory to search in. + :param filename: The filename. + :param extensions: The extensions. + :return: The file object. + """ + for ext in extensions: + path = os.path.join(dir, filename + ext) + if os.path.exists(path): + return cls(path) + raise FileNotFoundError + def __init__(self, path: str): + if not os.path.exists(path): + raise FileNotFoundError super().__init__(path) + self.filename = os.path.basename(path) + + def read(self) -> str: + with open(self.path, "r") as f: + return f.read() + + def write(self, text: str): + with open(self.path, "w") as f: + f.write(text) diff --git a/src/sio3pack/graph/__init__.py b/src/sio3pack/graph/__init__.py index bf9957e..79efe46 100644 --- a/src/sio3pack/graph/__init__.py +++ b/src/sio3pack/graph/__init__.py @@ -1 +1,3 @@ -from sio3pack.graph import Graph +from sio3pack.graph.graph import Graph +from sio3pack.graph.graph_manager import GraphManager +from sio3pack.graph.graph_op import GraphOperation diff --git a/src/sio3pack/graph/graph.py b/src/sio3pack/graph/graph.py index cdd50b2..370cbb8 100644 --- a/src/sio3pack/graph/graph.py +++ b/src/sio3pack/graph/graph.py @@ -5,7 +5,7 @@ class Graph: @classmethod def from_dict(cls, data: dict): - raise NotImplemented + raise NotImplementedError() def __init__(self, name: str): self.name = name @@ -14,4 +14,4 @@ def get_prog_files(self) -> list[str]: """ Get all program files in the graph. """ - raise NotImplemented + raise NotImplementedError() diff --git a/src/sio3pack/graph/graph_manager.py b/src/sio3pack/graph/graph_manager.py index 585ba8b..4a8e4f8 100644 --- a/src/sio3pack/graph/graph_manager.py +++ b/src/sio3pack/graph/graph_manager.py @@ -1,6 +1,6 @@ import json -from sio3pack import File +from sio3pack.files import File from sio3pack.graph.graph import Graph diff --git a/src/sio3pack/packages/exceptions.py b/src/sio3pack/packages/exceptions.py index ca91db2..307b736 100644 --- a/src/sio3pack/packages/exceptions.py +++ b/src/sio3pack/packages/exceptions.py @@ -1,2 +1,4 @@ class UnknownPackageType(Exception): - pass + def __init__(self, path: str) -> None: + self.path = path + super().__init__(f"Unknown package type for file {path}.") diff --git a/src/sio3pack/packages/package/model.py b/src/sio3pack/packages/package/model.py index 85972e1..0a863ec 100644 --- a/src/sio3pack/packages/package/model.py +++ b/src/sio3pack/packages/package/model.py @@ -1,9 +1,10 @@ from typing import Any -from sio3pack.files.file import File -from sio3pack.graph.graph import Graph +from sio3pack import LocalFile +from sio3pack.files import File +from sio3pack.graph import Graph from sio3pack.packages.exceptions import UnknownPackageType -from sio3pack.test.test import Test +from sio3pack.test import Test from sio3pack.utils.archive import Archive from sio3pack.utils.classinit import RegisteredSubclassesBase @@ -18,17 +19,18 @@ class Package(RegisteredSubclassesBase): def __init__(self, file: File): super().__init__() self.file = file - if Archive.is_archive(file.path): - self.is_archive = True - else: - self.is_archive = False + if isinstance(file, LocalFile): + if Archive.is_archive(file.path): + self.is_archive = True + else: + self.is_archive = False @classmethod - def from_file(cls, file: File, django_settings=None): + def from_file(cls, file: LocalFile, django_settings=None): for subclass in cls.subclasses: if subclass.identify(file): return subclass(file, django_settings) - raise UnknownPackageType + raise UnknownPackageType(file.path) def get_task_id(self) -> str: pass diff --git a/src/sio3pack/packages/sinolpack/model.py b/src/sio3pack/packages/sinolpack/model.py index 65fb3b8..09fbea4 100644 --- a/src/sio3pack/packages/sinolpack/model.py +++ b/src/sio3pack/packages/sinolpack/model.py @@ -4,10 +4,11 @@ import yaml -from sio3pack.files.file import File -from sio3pack.graph.graph import Graph -from sio3pack.graph.graph_manager import GraphManager -from sio3pack.graph.graph_op import GraphOperation +from sio3pack import LocalFile +from sio3pack.files import File +from sio3pack.graph import Graph +from sio3pack.graph import GraphManager +from sio3pack.graph import GraphOperation from sio3pack.packages.package import Package from sio3pack.packages.sinolpack.enums import ModelSolutionKind from sio3pack.util import naturalsort_key @@ -37,7 +38,7 @@ def _find_main_dir(cls, archive: Archive) -> str | None: return None @classmethod - def identify(cls, file: File) -> bool: + def identify(cls, file: LocalFile) -> bool: """ Identifies whether file is a Sinolpack. @@ -57,21 +58,25 @@ def __del__(self): def __init__(self, file: File, django_settings=None): super().__init__(file) - if self.is_archive: - archive = Archive(file.path) - self.short_name = self._find_main_dir(archive) - self.tmpdir = tempfile.TemporaryDirectory() - archive.extract(to_path=self.tmpdir.name) - self.rootdir = os.path.join(self.tmpdir.name, self.short_name) - else: - self.short_name = os.path.basename(file.path) - self.rootdir = file.path - try: - graph_file = self.get_in_root("graph.json") - self.graph_manager = GraphManager.from_file(graph_file) - except FileNotFoundError: - self.has_custom_graph = False + if isinstance(file, LocalFile): + if self.is_archive: + archive = Archive(file.path) + self.short_name = self._find_main_dir(archive) + self.tmpdir = tempfile.TemporaryDirectory() + archive.extract(to_path=self.tmpdir.name) + self.rootdir = os.path.join(self.tmpdir.name, self.short_name) + else: + self.short_name = os.path.basename(file.path) + self.rootdir = file.path + + try: + graph_file = self.get_in_root("graph.json") + self.graph_manager = GraphManager.from_file(graph_file) + except FileNotFoundError: + self.has_custom_graph = False + else: + raise NotImplementedError() self.django_settings = django_settings @@ -98,17 +103,17 @@ def get_doc_dir(self) -> str: """ return os.path.join(self.rootdir, "doc") - def get_in_doc_dir(self, filename: str) -> File: + def get_in_doc_dir(self, filename: str) -> LocalFile: """ Returns the path to the input file in the documents' directory. """ - return File(os.path.join(self.get_doc_dir(), filename)) + return LocalFile(os.path.join(self.get_doc_dir(), filename)) - def get_in_root(self, filename: str) -> File: + def get_in_root(self, filename: str) -> LocalFile: """ Returns the path to the input file in the root directory. """ - return File(os.path.join(self.rootdir, filename)) + return LocalFile(os.path.join(self.rootdir, filename)) def get_prog_dir(self) -> str: """ @@ -116,11 +121,11 @@ def get_prog_dir(self) -> str: """ return os.path.join(self.rootdir, "prog") - def get_in_prog_dir(self, filename: str) -> File: + def get_in_prog_dir(self, filename: str) -> LocalFile: """ Returns the path to the input file in the program directory. """ - return File(os.path.join(self.get_prog_dir(), filename)) + return LocalFile(os.path.join(self.get_prog_dir(), filename)) def get_attachments_dir(self) -> str: """ @@ -247,7 +252,7 @@ def _process_prog_files(self): for file in ("ingen", "inwer", "soc", "chk"): try: self.additional_files.append( - File.get_file_matching_extension( + LocalFile.get_file_matching_extension( self.get_prog_dir(), self.short_name + file, extensions ).filename ) diff --git a/src/sio3pack/test/__init__.py b/src/sio3pack/test/__init__.py index 0478c51..9c36d80 100644 --- a/src/sio3pack/test/__init__.py +++ b/src/sio3pack/test/__init__.py @@ -1 +1 @@ -from sio3pack.test import Test +from sio3pack.test.test import Test diff --git a/src/sio3pack/test/simple_test.py b/src/sio3pack/test/simple_test.py index 57c375c..3532aa0 100644 --- a/src/sio3pack/test/simple_test.py +++ b/src/sio3pack/test/simple_test.py @@ -1,5 +1,5 @@ -from sio3pack.files.file import File -from sio3pack.test.test import Test +from sio3pack.files import File +from sio3pack.test import Test class SimpleTest(Test): diff --git a/src/sio3pack/utils/archive.py b/src/sio3pack/utils/archive.py index 847eb81..533dd1d 100644 --- a/src/sio3pack/utils/archive.py +++ b/src/sio3pack/utils/archive.py @@ -65,6 +65,9 @@ def __init__(self, file, ext=""): self.filename = file self._archive = self._archive_cls(self.filename, ext=ext)(self.filename) + def __str__(self): + return f'' + @staticmethod def _archive_cls(file, ext=""): """ @@ -220,7 +223,13 @@ def filenames(self): return [zipinfo.filename for zipinfo in self._archive.infolist() if not zipinfo.is_dir()] def dirnames(self): - return [zipinfo.filename for zipinfo in self._archive.infolist() if zipinfo.is_dir()] + dirs = set() + for zipinfo in self._archive.infolist(): + if zipinfo.is_dir(): + dirs.add(zipinfo.filename) + else: + dirs.add(os.path.dirname(zipinfo.filename)) + return list(dirs) extension_map = { diff --git a/tests/fixtures.py b/tests/fixtures.py index 7e31c57..1e2a433 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -19,6 +19,17 @@ class Compression(Enum): all_compressions = [c.value for c in Compression if c != Compression.NONE] +class PackageInfo: + def __init__(self, path, type, task_id, compression): + self.path = path + self.type = type + self.task_id = task_id + self.compression = compression + + def is_archive(self): + return self.compression != Compression.NONE + + def _tar_archive(dir, dest, compression=None): """ Create a tar archive of the specified directory. @@ -38,10 +49,12 @@ def _zip_archive(dir, dest): with zipfile.ZipFile(dest, "w") as zip: for root, dirs, files in os.walk(dir): for file in files: - zip.write(os.path.join(root, file), os.path.relpath(os.path.join(root, file), dir)) + file_path = os.path.join(root, file) + arcname = os.path.join(os.path.basename(dir), os.path.relpath(file_path, dir)) + zip.write(file_path, arcname) -def _create_package(package_name, tmpdir, archive=False, extension="zip"): +def _create_package(package_name, tmpdir, archive=False, extension=Compression.ZIP): packages = os.path.join(os.path.dirname(__file__), "test_packages") if not os.path.exists(os.path.join(packages, package_name)): raise FileNotFoundError(f"Package {package_name} does not exist") @@ -58,43 +71,46 @@ def _create_package(package_name, tmpdir, archive=False, extension="zip"): os.unlink(os.path.join(package_path, "__init__.py")) if archive: - if extension == "zip": + if extension == Compression.ZIP: _zip_archive(package_path, os.path.join(tmpdir.name, f"{task_id}.zip")) - elif extension == "tar": - _tar_archive(package_path, os.path.join(tmpdir.name, f"{task_id}.tar")) - elif extension == "tar.gz" or extension == "tgz": - _tar_archive(package_path, os.path.join(tmpdir.name, f"{task_id}.{extension}"), "gz") + elif extension == Compression.TAR_GZ or extension == Compression.TGZ: + _tar_archive(package_path, os.path.join(tmpdir.name, f"{task_id}.{extension.value}"), "gz") else: raise ValueError(f"Unknown extension {extension}") - package_path = os.path.join(tmpdir.name, f"{task_id}.{extension}") + package_path = os.path.join(tmpdir.name, f"{task_id}.{extension.value}") - return package_path, type + return PackageInfo( + path=package_path, + type=type, + task_id=task_id, + compression=extension, + ) @pytest.fixture -def package(request): +def get_package(request): """ Fixture to create a temporary directory with specified package. """ package_name = request.param tmpdir = tempfile.TemporaryDirectory() - package_path, type = _create_package(package_name, tmpdir) + package_info = _create_package(package_name, tmpdir) - yield package_path, type + yield lambda: package_info tmpdir.cleanup() @pytest.fixture -def package_archived(request): +def get_archived_package(request): """ Fixture to create a temporary directory with specified package, but archived. """ package_name, extension = request.param archive = extension != Compression.NONE tmpdir = tempfile.TemporaryDirectory() - package_path, type = _create_package(package_name, tmpdir, archive, extension) + package_info = _create_package(package_name, tmpdir, archive, extension) - yield package_path, type + yield lambda: package_info tmpdir.cleanup() diff --git a/tests/packages/sinolpack/test_sinolpack.py b/tests/packages/sinolpack/test_sinolpack.py index c591fad..d4a6d13 100644 --- a/tests/packages/sinolpack/test_sinolpack.py +++ b/tests/packages/sinolpack/test_sinolpack.py @@ -2,20 +2,19 @@ import pytest -from tests.fixtures import Compression, all_compressions, package, package_archived +import sio3pack +from sio3pack.packages import Sinolpack +from tests.fixtures import Compression, all_compressions, get_package, get_archived_package, PackageInfo -@pytest.mark.parametrize("package", ["simple"], indirect=True) -def test_simple(package): - package_path, type = package - assert type == "sinolpack" - print(os.listdir(package_path)) - assert os.path.isdir(package_path) - - -@pytest.mark.parametrize("package_archived", [("simple", c) for c in all_compressions], indirect=True) -def test_archive(package_archived): - package_path, type = package_archived - assert type == "sinolpack" - print(package_path) - assert os.path.isfile(package_path) +@pytest.mark.parametrize("get_archived_package", [("simple", c) for c in Compression], indirect=True) +def test_from_file(get_archived_package): + package_info: PackageInfo = get_archived_package() + assert package_info.type == "sinolpack" + package = sio3pack.from_file(package_info.path) + assert isinstance(package, Sinolpack) + assert package.short_name == package_info.task_id + if package_info.is_archive(): + assert package.is_archive + else: + assert package.rootdir == package_info.path diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..e69de29 From fffba1a5da128d7c4952fea7430fad68ccc9e89f Mon Sep 17 00:00:00 2001 From: MasloMaslane Date: Sat, 30 Nov 2024 18:05:16 +0100 Subject: [PATCH 7/9] Reformat --- src/sio3pack/files/__init__.py | 2 +- src/sio3pack/packages/sinolpack/model.py | 4 +--- src/sio3pack/utils/archive.py | 2 +- tests/packages/sinolpack/test_sinolpack.py | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/sio3pack/files/__init__.py b/src/sio3pack/files/__init__.py index ca063b6..706ae2e 100644 --- a/src/sio3pack/files/__init__.py +++ b/src/sio3pack/files/__init__.py @@ -1,3 +1,3 @@ +from sio3pack.files.file import File from sio3pack.files.filetracker_file import FiletrackerFile from sio3pack.files.local_file import LocalFile -from sio3pack.files.file import File diff --git a/src/sio3pack/packages/sinolpack/model.py b/src/sio3pack/packages/sinolpack/model.py index 09fbea4..d26c87e 100644 --- a/src/sio3pack/packages/sinolpack/model.py +++ b/src/sio3pack/packages/sinolpack/model.py @@ -6,9 +6,7 @@ from sio3pack import LocalFile from sio3pack.files import File -from sio3pack.graph import Graph -from sio3pack.graph import GraphManager -from sio3pack.graph import GraphOperation +from sio3pack.graph import Graph, GraphManager, GraphOperation from sio3pack.packages.package import Package from sio3pack.packages.sinolpack.enums import ModelSolutionKind from sio3pack.util import naturalsort_key diff --git a/src/sio3pack/utils/archive.py b/src/sio3pack/utils/archive.py index 533dd1d..6a6b8f5 100644 --- a/src/sio3pack/utils/archive.py +++ b/src/sio3pack/utils/archive.py @@ -66,7 +66,7 @@ def __init__(self, file, ext=""): self._archive = self._archive_cls(self.filename, ext=ext)(self.filename) def __str__(self): - return f'' + return f"" @staticmethod def _archive_cls(file, ext=""): diff --git a/tests/packages/sinolpack/test_sinolpack.py b/tests/packages/sinolpack/test_sinolpack.py index d4a6d13..b8860ae 100644 --- a/tests/packages/sinolpack/test_sinolpack.py +++ b/tests/packages/sinolpack/test_sinolpack.py @@ -4,7 +4,7 @@ import sio3pack from sio3pack.packages import Sinolpack -from tests.fixtures import Compression, all_compressions, get_package, get_archived_package, PackageInfo +from tests.fixtures import Compression, PackageInfo, all_compressions, get_archived_package, get_package @pytest.mark.parametrize("get_archived_package", [("simple", c) for c in Compression], indirect=True) From b093aa4b78947f8a71de693b3cc49ee99634bfaa Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Sun, 1 Dec 2024 17:23:08 +0100 Subject: [PATCH 8/9] Refactor --- src/sio3pack/packages/package/model.py | 20 ++++++------ src/sio3pack/packages/sinolpack/model.py | 39 ++++++++++++------------ 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/sio3pack/packages/package/model.py b/src/sio3pack/packages/package/model.py index 0a863ec..ba5e45b 100644 --- a/src/sio3pack/packages/package/model.py +++ b/src/sio3pack/packages/package/model.py @@ -2,7 +2,7 @@ from sio3pack import LocalFile from sio3pack.files import File -from sio3pack.graph import Graph +from sio3pack.graph import Graph, GraphOperation from sio3pack.packages.exceptions import UnknownPackageType from sio3pack.test import Test from sio3pack.utils.archive import Archive @@ -16,14 +16,8 @@ class Package(RegisteredSubclassesBase): abstract = True - def __init__(self, file: File): + def __init__(self): super().__init__() - self.file = file - if isinstance(file, LocalFile): - if Archive.is_archive(file.path): - self.is_archive = True - else: - self.is_archive = False @classmethod def from_file(cls, file: LocalFile, django_settings=None): @@ -32,6 +26,14 @@ def from_file(cls, file: LocalFile, django_settings=None): return subclass(file, django_settings) raise UnknownPackageType(file.path) + def _from_file(self, file: LocalFile): + self.file = file + if isinstance(file, LocalFile): + if Archive.is_archive(file.path): + self.is_archive = True + else: + self.is_archive = False + def get_task_id(self) -> str: pass @@ -56,7 +58,7 @@ def get_tests(self) -> list[Test]: def get_test(self, test_id: str) -> Test: pass - def get_package_graph(self) -> Graph: + def get_unpack_graph(self) -> GraphOperation | None: pass def get_run_graph(self, file: File, tests: list[Test] | None = None) -> Graph: diff --git a/src/sio3pack/packages/sinolpack/model.py b/src/sio3pack/packages/sinolpack/model.py index d26c87e..419248f 100644 --- a/src/sio3pack/packages/sinolpack/model.py +++ b/src/sio3pack/packages/sinolpack/model.py @@ -54,27 +54,26 @@ def __del__(self): if hasattr(self, "tmpdir"): self.tmpdir.cleanup() - def __init__(self, file: File, django_settings=None): - super().__init__(file) - - if isinstance(file, LocalFile): - if self.is_archive: - archive = Archive(file.path) - self.short_name = self._find_main_dir(archive) - self.tmpdir = tempfile.TemporaryDirectory() - archive.extract(to_path=self.tmpdir.name) - self.rootdir = os.path.join(self.tmpdir.name, self.short_name) - else: - self.short_name = os.path.basename(file.path) - self.rootdir = file.path - - try: - graph_file = self.get_in_root("graph.json") - self.graph_manager = GraphManager.from_file(graph_file) - except FileNotFoundError: - self.has_custom_graph = False + def __init__(self): + super().__init__() + + def _from_file(self, file: LocalFile, django_settings=None): + super()._from_file(file) + if self.is_archive: + archive = Archive(file.path) + self.short_name = self._find_main_dir(archive) + self.tmpdir = tempfile.TemporaryDirectory() + archive.extract(to_path=self.tmpdir.name) + self.rootdir = os.path.join(self.tmpdir.name, self.short_name) else: - raise NotImplementedError() + self.short_name = os.path.basename(file.path) + self.rootdir = file.path + + try: + graph_file = self.get_in_root("graph.json") + self.graph_manager = GraphManager.from_file(graph_file) + except FileNotFoundError: + self.has_custom_graph = False self.django_settings = django_settings From f8b49f5612f2813465ffefd94e510de21ba9ed35 Mon Sep 17 00:00:00 2001 From: Mateusz Masiarz Date: Mon, 2 Dec 2024 18:12:17 +0100 Subject: [PATCH 9/9] Django integration (#4) * Add django integration * Remove debug * Add tests * Reformat * Bugfixes, refactor * Fix tests * Fix workflows * Formatter and test setup * Chmod manage.py * Python 3.10 * Fix workflows * Remove arch --------- Co-authored-by: Tomasz Kwiatkowski --- .github/workflows/arch.yml | 28 ---- .github/workflows/check_migrations.yml | 23 ++++ .github/workflows/formatter.yml | 6 +- .github/workflows/macOS.yml | 14 +- .github/workflows/ubuntu.yml | 15 ++- .gitignore | 1 + pyproject.toml | 1 + pytest.ini | 5 + setup.cfg | 4 + src/sio3pack/__init__.py | 17 +++ src/sio3pack/django/__init__.py | 0 src/sio3pack/django/sinolpack/__init__.py | 0 src/sio3pack/django/sinolpack/handler.py | 17 +++ .../sinolpack/migrations/0001_initial.py | 21 +++ .../django/sinolpack/migrations/__init__.py | 0 src/sio3pack/django/sinolpack/models.py | 10 ++ src/sio3pack/packages/exceptions.py | 21 ++- .../packages/package/django/__init__.py | 0 .../packages/package/django/handler.py | 14 ++ src/sio3pack/packages/package/model.py | 71 +++++++++- src/sio3pack/packages/sinolpack/model.py | 59 +++++++-- test.sh | 14 ++ tests/conftest.py | 20 +++ tests/packages/sinolpack/test_sinolpack.py | 12 ++ tests/test_django/__init__.py | 0 tests/test_django/manage.py | 22 +++ tests/test_django/test_django/__init__.py | 0 tests/test_django/test_django/asgi.py | 16 +++ tests/test_django/test_django/settings.py | 125 ++++++++++++++++++ tests/test_django/test_django/urls.py | 23 ++++ tests/test_django/test_django/wsgi.py | 16 +++ tests/test_django/test_sio3pack/__init__.py | 0 .../test_sio3pack/test_sinolpack.py | 22 +++ 33 files changed, 543 insertions(+), 54 deletions(-) delete mode 100644 .github/workflows/arch.yml create mode 100644 .github/workflows/check_migrations.yml create mode 100644 pytest.ini create mode 100644 src/sio3pack/django/__init__.py create mode 100644 src/sio3pack/django/sinolpack/__init__.py create mode 100644 src/sio3pack/django/sinolpack/handler.py create mode 100644 src/sio3pack/django/sinolpack/migrations/0001_initial.py create mode 100644 src/sio3pack/django/sinolpack/migrations/__init__.py create mode 100644 src/sio3pack/django/sinolpack/models.py create mode 100644 src/sio3pack/packages/package/django/__init__.py create mode 100644 src/sio3pack/packages/package/django/handler.py create mode 100644 test.sh create mode 100644 tests/conftest.py create mode 100644 tests/test_django/__init__.py create mode 100755 tests/test_django/manage.py create mode 100644 tests/test_django/test_django/__init__.py create mode 100644 tests/test_django/test_django/asgi.py create mode 100644 tests/test_django/test_django/settings.py create mode 100644 tests/test_django/test_django/urls.py create mode 100644 tests/test_django/test_django/wsgi.py create mode 100644 tests/test_django/test_sio3pack/__init__.py create mode 100644 tests/test_django/test_sio3pack/test_sinolpack.py diff --git a/.github/workflows/arch.yml b/.github/workflows/arch.yml deleted file mode 100644 index 0c4a281..0000000 --- a/.github/workflows/arch.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: pytest-arch -run-name: Run pytest on Arch Linux -on: - push: - branches: 'main' - pull_request: -jobs: - pytest: - runs-on: archlinux:latest - strategy: - matrix: - python-version: ["3.9", "3.13"] - name: pytest-arch-python-${{ matrix.python-version }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - pip3 install .[tests] - - name: Run pytest - env: - PYTEST_ADDOPTS: "--color=yes" - run: | - python3 -m pytest -v -n auto diff --git a/.github/workflows/check_migrations.yml b/.github/workflows/check_migrations.yml new file mode 100644 index 0000000..86e549e --- /dev/null +++ b/.github/workflows/check_migrations.yml @@ -0,0 +1,23 @@ +name: migrations +run-name: Check if all migrations are created +on: + push: + branches: 'main' + pull_request: +jobs: + pytest: + runs-on: ubuntu-latest + name: migrations-ubuntu + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.12 + - name: Install dependencies + run: | + pip3 install .[django] + - name: Check migrations + run: | + python3 tests/test_django/manage.py makemigrations --dry-run --check diff --git a/.github/workflows/formatter.yml b/.github/workflows/formatter.yml index 6b37756..e930f84 100644 --- a/.github/workflows/formatter.yml +++ b/.github/workflows/formatter.yml @@ -1,9 +1,9 @@ name: Python Formatter (isort & black) on: + push: + branches: 'main' pull_request: - branches: [main] - jobs: formatting: runs-on: ubuntu-latest @@ -12,7 +12,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.10' - name: Install isort and black run: | python -m pip install --upgrade pip diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index 0d379c1..11a3188 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -10,7 +10,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python-version: ["3.9", "3.13"] + python-version: ["3.10", "3.13"] name: pytest-macos-python-${{ matrix.python-version }} steps: - name: Checkout @@ -30,10 +30,18 @@ jobs: run: | python3 -m venv .venv source .venv/bin/activate - pip install .[tests] - - name: Run pytest + pip install -e .[tests] + - name: Run pytest without Django env: PYTEST_ADDOPTS: "--color=yes" run: | source .venv/bin/activate pytest -v -n auto + - name: Run pytest with Django + env: + PYTEST_ADDOPTS: "--color=yes" + run: | + source .venv/bin/activate + pip install -e .[tests,django_tests,django] + ./tests/test_django/manage.py migrate + pytest -v -n auto diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 9256238..af94afd 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -6,10 +6,10 @@ on: pull_request: jobs: pytest: - runs-on: ubuntu:latest + runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.13"] + python-version: ["3.10", "3.13"] name: pytest-ubuntu-python-${{ matrix.python-version }} steps: - name: Checkout @@ -20,9 +20,16 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - pip3 install .[tests] - - name: Run pytest + pip3 install -e .[tests] + - name: Run pytest without Django env: PYTEST_ADDOPTS: "--color=yes" run: | python3 -m pytest -v -n auto + - name: Run pytest with Django + env: + PYTEST_ADDOPTS: "--color=yes" + run: | + pip3 install -e .[tests,django_tests,django] + ./tests/test_django/manage.py migrate + python3 -m pytest -v -n auto diff --git a/.gitignore b/.gitignore index 76369bd..471c6cc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ build .vscode .idea __pycache__ +tests/test_django/db.sqlite3 # pytest-cov .coverage* diff --git a/pyproject.toml b/pyproject.toml index d78f6bd..8dcf968 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,3 +9,4 @@ include_trailing_comma = true [tool.black] line_length = 120 +exclude = "migrations/" diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..ddf7218 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +DJANGO_SETTINGS_MODULE = test_django.settings +pythonpath = ./tests/test_django +markers = + no_django: marks tests that should be run without Django diff --git a/setup.cfg b/setup.cfg index da4d6d0..53698c9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,6 +32,10 @@ tests = pytest pytest-cov pytest-xdist +django_tests = + pytest-django +django = + django [tool:pytest] testpaths = diff --git a/src/sio3pack/__init__.py b/src/sio3pack/__init__.py index 9761228..8220ad7 100644 --- a/src/sio3pack/__init__.py +++ b/src/sio3pack/__init__.py @@ -1,6 +1,7 @@ __version__ = "0.0.1" from sio3pack.files import LocalFile +from sio3pack.packages.exceptions import ImproperlyConfigured, PackageAlreadyExists from sio3pack.packages.package import Package @@ -14,3 +15,19 @@ def from_file(file: str | LocalFile, django_settings=None) -> Package: if isinstance(file, str): file = LocalFile(file) return Package.from_file(file, django_settings=django_settings) + + +def from_db(problem_id: int) -> Package: + """ + Initialize a package object from the database. + If sio3pack isn't installed with Django support, it should raise an ImproperlyConfigured exception. + If there is no package with the given problem_id, it should raise an UnknownPackageType exception. + :param problem_id: The problem id. + :return: The package object. + """ + try: + import django + + return Package.from_db(problem_id) + except ImportError: + raise ImproperlyConfigured("sio3pack is not installed with Django support.") diff --git a/src/sio3pack/django/__init__.py b/src/sio3pack/django/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/sio3pack/django/sinolpack/__init__.py b/src/sio3pack/django/sinolpack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/sio3pack/django/sinolpack/handler.py b/src/sio3pack/django/sinolpack/handler.py new file mode 100644 index 0000000..0ae67e7 --- /dev/null +++ b/src/sio3pack/django/sinolpack/handler.py @@ -0,0 +1,17 @@ +from sio3pack.django.sinolpack.models import SinolpackPackage +from sio3pack.packages.exceptions import PackageAlreadyExists +from sio3pack.packages.package.django.handler import DjangoHandler + + +class SinolpackDjangoHandler(DjangoHandler): + def save_to_db(self): + """ + Save the package to the database. + """ + if SinolpackPackage.objects.filter(problem_id=self.problem_id).exists(): + raise PackageAlreadyExists(self.problem_id) + + SinolpackPackage.objects.create( + problem_id=self.problem_id, + short_name=self.package.short_name, + ) diff --git a/src/sio3pack/django/sinolpack/migrations/0001_initial.py b/src/sio3pack/django/sinolpack/migrations/0001_initial.py new file mode 100644 index 0000000..c342290 --- /dev/null +++ b/src/sio3pack/django/sinolpack/migrations/0001_initial.py @@ -0,0 +1,21 @@ +# Generated by Django 5.1.3 on 2024-12-01 17:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="SinolpackPackage", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("problem_id", models.IntegerField()), + ("short_name", models.CharField(max_length=100)), + ], + ), + ] diff --git a/src/sio3pack/django/sinolpack/migrations/__init__.py b/src/sio3pack/django/sinolpack/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/sio3pack/django/sinolpack/models.py b/src/sio3pack/django/sinolpack/models.py new file mode 100644 index 0000000..961494c --- /dev/null +++ b/src/sio3pack/django/sinolpack/models.py @@ -0,0 +1,10 @@ +from django.db import models + + +class SinolpackPackage(models.Model): + """ + A package for the sinolpack package type. + """ + + problem_id = models.IntegerField() + short_name = models.CharField(max_length=100) diff --git a/src/sio3pack/packages/exceptions.py b/src/sio3pack/packages/exceptions.py index 307b736..1f9e0e4 100644 --- a/src/sio3pack/packages/exceptions.py +++ b/src/sio3pack/packages/exceptions.py @@ -1,4 +1,19 @@ class UnknownPackageType(Exception): - def __init__(self, path: str) -> None: - self.path = path - super().__init__(f"Unknown package type for file {path}.") + def __init__(self, arg: str | int) -> None: + if isinstance(arg, str): + self.path = arg + super().__init__(f"Unknown package type for file {arg}.") + else: + self.problem_id = arg + super().__init__(f"Unknown package type for problem with id={arg}.") + + +class ImproperlyConfigured(Exception): + def __init__(self, message: str) -> None: + super().__init__(message) + + +class PackageAlreadyExists(Exception): + def __init__(self, problem_id: int) -> None: + self.problem_id = problem_id + super().__init__(f"A package already exists for problem with id={problem_id}.") diff --git a/src/sio3pack/packages/package/django/__init__.py b/src/sio3pack/packages/package/django/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/sio3pack/packages/package/django/handler.py b/src/sio3pack/packages/package/django/handler.py new file mode 100644 index 0000000..e838fca --- /dev/null +++ b/src/sio3pack/packages/package/django/handler.py @@ -0,0 +1,14 @@ +from typing import Type + +from sio3pack.packages.exceptions import ImproperlyConfigured + + +class DjangoHandler: + def __init__(self, package: Type["Package"], problem_id: int): + self.package = package + self.problem_id = problem_id + + +class NoDjangoHandler: + def __call__(self, *args, **kwargs): + raise ImproperlyConfigured("sio3pack is not installed with Django support.") diff --git a/src/sio3pack/packages/package/model.py b/src/sio3pack/packages/package/model.py index ba5e45b..9438957 100644 --- a/src/sio3pack/packages/package/model.py +++ b/src/sio3pack/packages/package/model.py @@ -1,9 +1,11 @@ +import importlib from typing import Any from sio3pack import LocalFile from sio3pack.files import File from sio3pack.graph import Graph, GraphOperation from sio3pack.packages.exceptions import UnknownPackageType +from sio3pack.packages.package.django.handler import NoDjangoHandler from sio3pack.test import Test from sio3pack.utils.archive import Archive from sio3pack.utils.classinit import RegisteredSubclassesBase @@ -19,11 +21,23 @@ class Package(RegisteredSubclassesBase): def __init__(self): super().__init__() + @classmethod + def identify(cls, file: LocalFile): + """ + Identify if the package is of this type. + """ + raise NotImplementedError() + @classmethod def from_file(cls, file: LocalFile, django_settings=None): + """ + Create a package from a file. + """ for subclass in cls.subclasses: if subclass.identify(file): - return subclass(file, django_settings) + package = subclass() + package._from_file(file, django_settings) + return package raise UnknownPackageType(file.path) def _from_file(self, file: LocalFile): @@ -34,6 +48,50 @@ def _from_file(self, file: LocalFile): else: self.is_archive = False + @classmethod + def identify_db(cls, problem_id: int): + """ + Identify if the package is of this type. Should check if there + is a package of this type in the database with the given problem_id. + """ + raise NotImplementedError() + + @classmethod + def from_db(cls, problem_id: int): + """ + Create a package from the database. If sio3pack isn't installed with Django + support, it should raise an ImproperlyConfigured exception. If there is no + package with the given problem_id, it should raise an UnknownPackageType + exception. + """ + for subclass in cls.subclasses: + if subclass.identify_db(problem_id): + package = subclass() + package._from_db(problem_id) + return package + raise UnknownPackageType(problem_id) + + def _from_db(self, problem_id: int): + """ + Internal method to setup the package from the database. If sio3pack + isn't installed with Django support, it should raise an ImproperlyConfigured + exception. + """ + self.problem_id = problem_id + + def _setup_django_handler(self, problem_id: int): + try: + import django + + self.django_enabled = True + module_path, class_name = self.django_handler.rsplit(".", 1) + module = importlib.import_module(module_path) + handler = getattr(module, class_name) + self.django = handler(package=self, problem_id=problem_id) + except ImportError: + self.django_enabled = False + self.django = NoDjangoHandler() + def get_task_id(self) -> str: pass @@ -61,8 +119,15 @@ def get_test(self, test_id: str) -> Test: def get_unpack_graph(self) -> GraphOperation | None: pass - def get_run_graph(self, file: File, tests: list[Test] | None = None) -> Graph: + def get_run_graph(self, file: File, tests: list[Test] | None = None) -> GraphOperation | None: + pass + + def get_save_outs_graph(self, tests: list[Test] | None = None) -> GraphOperation | None: pass - def get_save_outs_graph(self, tests: list[Test] | None = None) -> Graph: + def save_to_db(self, problem_id: int): + """ + Save the package to the database. If sio3pack isn't installed with Django + support, it should raise an ImproperlyConfigured exception. + """ pass diff --git a/src/sio3pack/packages/sinolpack/model.py b/src/sio3pack/packages/sinolpack/model.py index 419248f..9d31864 100644 --- a/src/sio3pack/packages/sinolpack/model.py +++ b/src/sio3pack/packages/sinolpack/model.py @@ -5,8 +5,8 @@ import yaml from sio3pack import LocalFile -from sio3pack.files import File from sio3pack.graph import Graph, GraphManager, GraphOperation +from sio3pack.packages.exceptions import ImproperlyConfigured from sio3pack.packages.package import Package from sio3pack.packages.sinolpack.enums import ModelSolutionKind from sio3pack.util import naturalsort_key @@ -18,6 +18,8 @@ class Sinolpack(Package): Represents a OIOIOI's standard problem package. """ + django_handler = "sio3pack.django.sinolpack.handler.SinolpackDjangoHandler" + @classmethod def _find_main_dir(cls, archive: Archive) -> str | None: dirs = list(map(os.path.normcase, archive.dirnames())) @@ -50,6 +52,18 @@ def identify(cls, file: LocalFile) -> bool: except UnrecognizedArchiveFormat: return os.path.exists(os.path.join(path, "in")) and os.path.exists(os.path.join(path, "out")) + @classmethod + def identify_from_db(cls, problem_id: int) -> bool: + """ + Identifies whether problem is a Sinolpack. + + :param problem_id: ID of the problem. + :return: True when problem is a Sinolpack, otherwise False. + """ + from sio3pack.django.sinolpack.models import SinolpackPackage + + return SinolpackPackage.objects.filter(problem_id=problem_id).exists() + def __del__(self): if hasattr(self, "tmpdir"): self.tmpdir.cleanup() @@ -66,6 +80,7 @@ def _from_file(self, file: LocalFile, django_settings=None): archive.extract(to_path=self.tmpdir.name) self.rootdir = os.path.join(self.tmpdir.name, self.short_name) else: + # FIXME: Won't work in sinol-make. self.short_name = os.path.basename(file.path) self.rootdir = file.path @@ -77,6 +92,14 @@ def _from_file(self, file: LocalFile, django_settings=None): self.django_settings = django_settings + self._process_package() + + def _from_db(self, problem_id: int): + super()._from_db(problem_id) + super()._setup_django_handler(problem_id) + if not self.django_enabled: + raise ImproperlyConfigured("sio3pack is not installed with Django support.") + def _default_graph_manager(self) -> GraphManager: return GraphManager( { @@ -140,7 +163,9 @@ def _process_package(self): if not self.has_custom_graph: # Create the graph with processed files. - self.graph_manager = self._default_graph_manager() + # TODO: Uncomment this line when Graph will work. + # self.graph_manager = self._default_graph_manager() + pass def _process_config_yml(self): """ @@ -180,8 +205,8 @@ def _detect_full_name_translations(self): two-letter language code defined in ``settings.py``), if any such key is given. """ self.lang_titles = {} - for lang_code, lang in self._get_from_django_settings("LANGUAGES", [("en", "English")]): - key = "title_%s" % lang_code + for lang_code, _ in self._get_from_django_settings("LANGUAGES", [("en", "English")]): + key = f"title_{lang_code}" if key in self.config: self.lang_titles[lang_code] = self.config[key] @@ -190,7 +215,8 @@ def get_submittable_extensions(self): Returns a list of extensions that are submittable. """ return self.config.get( - "submittable_langs", self._get_from_django_settings("SUBMITTABLE_LANGUAGES", ["c", "cpp", "cxx", "py"]) + "submittable_langs", + self._get_from_django_settings("SUBMITTABLE_LANGUAGES", ["c", "cpp", "cc", "cxx", "py"]), ) def get_model_solution_regex(self): @@ -204,11 +230,14 @@ def _get_model_solutions(self) -> list[tuple[ModelSolutionKind, str]]: """ Returns a list of model solutions, where each element is a tuple of model solution kind and filename. """ + if not os.path.exists(self.get_prog_dir()): + return [] + regex = self.get_model_solution_regex() model_solutions = [] for file in os.listdir(self.get_prog_dir()): match = re.match(regex, file) - if re.match(regex, file) and os.path.isfile(os.path.join(self.get_prog_dir(), file)): + if match and os.path.isfile(os.path.join(self.get_prog_dir(), file)): model_solutions.append((ModelSolutionKind.from_regex(match.group(1)), file)) return model_solutions @@ -270,13 +299,13 @@ def _process_statements(self): return lang_prefs = [""] + [ - "-" + l[0] for l in self._get_from_django_settings("LANGUAGES", [("en", "English"), ("pl", "Polish")]) + f"-{lang}" for lang, _ in self._get_from_django_settings("LANGUAGES", [("en", "English"), ("pl", "Polish")]) ] self.lang_statements = {} for lang in lang_prefs: try: - htmlzipfile = self.get_in_doc_dir(self.short_name + "zad" + lang + ".html.zip") + htmlzipfile = self.get_in_doc_dir(f"{self.short_name}zad{lang}.html.zip") # TODO: what to do with html? # if self._html_disallowed(): # raise ProblemPackageError( @@ -296,12 +325,12 @@ def _process_statements(self): pass try: - pdffile = self.get_in_doc_dir(self.short_name + "zad" + lang + ".pdf") + pdffile = self.get_in_doc_dir(f"{self.short_name}zad{lang}.pdf") if lang == "": self.statement = pdffile else: self.lang_statements[lang[1:]] = pdffile - except: + except FileNotFoundError: pass def _process_attachments(self): @@ -331,3 +360,13 @@ def _unpack_return_data(self, data: dict): """ # TODO: implement. The unpack will probably return tests, so we need to process them. pass + + def save_to_db(self, problem_id: int): + """ + Save the package to the database. If sio3pack isn't installed with Django + support, it should raise an ImproperlyConfigured exception. + """ + self._setup_django_handler(problem_id) + if not self.django_enabled: + raise ImproperlyConfigured("sio3pack is not installed with Django support.") + self.django.save_to_db() diff --git a/test.sh b/test.sh new file mode 100644 index 0000000..29564c3 --- /dev/null +++ b/test.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +cd "`dirname $0`" + +cd tests/test_django +./manage.py makemigrations +./manage.py migrate +cd ../.. + +if [ -z "$1" ]; then + pytest -v tests/ +else + pytest -v tests/ $1 +fi diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..890beac --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,20 @@ +import pytest + +try: + import django + + __django_installed = True +except ImportError: + __django_installed = False + + +def pytest_collection_modifyitems(config, items): + for item in items: + if "no_django" in item.keywords: + if __django_installed: + item.add_marker(pytest.mark.skip(reason="Django is installed, skipping no_django tests.")) + + +def pytest_ignore_collect(collection_path, config): + if not __django_installed: + return "test_django" in str(collection_path) diff --git a/tests/packages/sinolpack/test_sinolpack.py b/tests/packages/sinolpack/test_sinolpack.py index b8860ae..d798c4a 100644 --- a/tests/packages/sinolpack/test_sinolpack.py +++ b/tests/packages/sinolpack/test_sinolpack.py @@ -18,3 +18,15 @@ def test_from_file(get_archived_package): assert package.is_archive else: assert package.rootdir == package_info.path + + +@pytest.mark.no_django +@pytest.mark.parametrize("get_package", ["simple"], indirect=True) +def test_no_django(get_package): + package_info: PackageInfo = get_package() + with pytest.raises(sio3pack.ImproperlyConfigured): + sio3pack.from_db(1) + + package = sio3pack.from_file(package_info.path) + with pytest.raises(sio3pack.ImproperlyConfigured): + package.save_to_db(1) diff --git a/tests/test_django/__init__.py b/tests/test_django/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_django/manage.py b/tests/test_django/manage.py new file mode 100755 index 0000000..8b47138 --- /dev/null +++ b/tests/test_django/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "test_django.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/tests/test_django/test_django/__init__.py b/tests/test_django/test_django/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_django/test_django/asgi.py b/tests/test_django/test_django/asgi.py new file mode 100644 index 0000000..b83d195 --- /dev/null +++ b/tests/test_django/test_django/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for test_django project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/5.1/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "test_django.settings") + +application = get_asgi_application() diff --git a/tests/test_django/test_django/settings.py b/tests/test_django/test_django/settings.py new file mode 100644 index 0000000..68bec3e --- /dev/null +++ b/tests/test_django/test_django/settings.py @@ -0,0 +1,125 @@ +""" +Django settings for test_django project. + +Generated by 'django-admin startproject' using Django 5.1.3. + +For more information on this file, see +https://docs.djangoproject.com/en/5.1/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/5.1/ref/settings/ +""" + +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "django-insecure-f$oojxszb)@)-w0&-=4+24&oa$brdv6ltj34n@25=rq1n-kq6&" + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + # Here add all sio3pack apps + "sio3pack.django.sinolpack", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "test_django.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "test_django.wsgi.application" + + +# Database +# https://docs.djangoproject.com/en/5.1/ref/settings/#databases + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": BASE_DIR / "db.sqlite3", + } +} + + +# Password validation +# https://docs.djangoproject.com/en/5.1/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/5.1/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/5.1/howto/static-files/ + +STATIC_URL = "static/" + +# Default primary key field type +# https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" diff --git a/tests/test_django/test_django/urls.py b/tests/test_django/test_django/urls.py new file mode 100644 index 0000000..7a1b55e --- /dev/null +++ b/tests/test_django/test_django/urls.py @@ -0,0 +1,23 @@ +""" +URL configuration for test_django project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/5.1/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" + +from django.contrib import admin +from django.urls import path + +urlpatterns = [ + path("admin/", admin.site.urls), +] diff --git a/tests/test_django/test_django/wsgi.py b/tests/test_django/test_django/wsgi.py new file mode 100644 index 0000000..a3bfd6b --- /dev/null +++ b/tests/test_django/test_django/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for test_django project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/5.1/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "test_django.settings") + +application = get_wsgi_application() diff --git a/tests/test_django/test_sio3pack/__init__.py b/tests/test_django/test_sio3pack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_django/test_sio3pack/test_sinolpack.py b/tests/test_django/test_sio3pack/test_sinolpack.py new file mode 100644 index 0000000..6516081 --- /dev/null +++ b/tests/test_django/test_sio3pack/test_sinolpack.py @@ -0,0 +1,22 @@ +import pytest + +import sio3pack +from sio3pack.django.sinolpack.models import SinolpackPackage +from sio3pack.packages import Sinolpack +from tests.fixtures import Compression, PackageInfo, get_archived_package + + +@pytest.mark.django_db +@pytest.mark.parametrize("get_archived_package", [("simple", c) for c in Compression], indirect=True) +def test_simple(get_archived_package): + package_info: PackageInfo = get_archived_package() + assert package_info.type == "sinolpack" + package = sio3pack.from_file(package_info.path) + assert isinstance(package, Sinolpack) + package.save_to_db(1) + assert SinolpackPackage.objects.filter(problem_id=1).exists() + db_package = SinolpackPackage.objects.get(problem_id=1) + assert db_package.short_name == package.short_name + + with pytest.raises(sio3pack.PackageAlreadyExists): + package.save_to_db(1)