From e2a1702529e66a7efd89c5b5d800b0469cefa000 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 24 Jan 2025 17:35:38 -0800 Subject: [PATCH 1/3] Allow setting env on the launched docker containers --- .../web/management/commands/evaluator.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/frx_challenges/web/management/commands/evaluator.py b/frx_challenges/web/management/commands/evaluator.py index cc4caa0..fcc846c 100644 --- a/frx_challenges/web/management/commands/evaluator.py +++ b/frx_challenges/web/management/commands/evaluator.py @@ -1,4 +1,5 @@ import asyncio +import collections.abc import json import logging import os @@ -80,13 +81,22 @@ async def start_evaluation(self, input_uri): ) for c in settings.EVALUATOR_DOCKER_CMD ] - container = await self.docker.containers.create( - config={ - "Image": self.image, - "Cmd": cmd, - "HostConfig": host_config, - } - ) + + config={ + "Image": self.image, + "Cmd": cmd, + "HostConfig": host_config, + } + + if settings.EVALUATOR_DOCKER_CONTAINER_ENV: + if isinstance(settings.EVALUATOR_DOCKER_CONTAINER_ENV, collections.abc.Mapping): + config["Env"] = [f"{k}={v}" for k, v in settings.EVALUATOR_DOCKER_CONTAINER_ENV.items()] + elif isinstance(settings.EVALUATOR_DOCKER_CONTAINER_ENV, list): + config["Env"] = settings.EVALUATOR_DOCKER_CONTAINER_ENV + else: + raise RuntimeError(f"settings.EVALUATOR_DOCKER_CONTAINER_ENV should be a dict or list, found {type( settings.EVALUATOR_DOCKER_CONTAINER_ENV)}") + + container = await self.docker.containers.create(config) logger.debug(f"Container created with ID: {container.id}") try: From 28df9e9f3e7aba903fa21d55bc72330bd1925b14 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 27 Jan 2025 15:17:56 -0800 Subject: [PATCH 2/3] Allow updating arbitrary container host config Allows us to tweak things like `ulimit` without having to specifically add extra support for them --- .../web/management/commands/evaluator.py | 4 ++++ frx_challenges/web/utils.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 frx_challenges/web/utils.py diff --git a/frx_challenges/web/management/commands/evaluator.py b/frx_challenges/web/management/commands/evaluator.py index fcc846c..da48b9b 100644 --- a/frx_challenges/web/management/commands/evaluator.py +++ b/frx_challenges/web/management/commands/evaluator.py @@ -14,6 +14,7 @@ from django.core.management.base import BaseCommand from ...models import Evaluation +from web.utils import recursive_update logger = logging.getLogger() @@ -82,6 +83,9 @@ async def start_evaluation(self, input_uri): for c in settings.EVALUATOR_DOCKER_CMD ] + if settings.EVALUATOR_DOCKER_EXTRA_HOST_CONFIG: + recursive_update(host_config, settings.EVALUATOR_DOCKER_EXTRA_HOST_CONFIG) + config={ "Image": self.image, "Cmd": cmd, diff --git a/frx_challenges/web/utils.py b/frx_challenges/web/utils.py new file mode 100644 index 0000000..cc2107e --- /dev/null +++ b/frx_challenges/web/utils.py @@ -0,0 +1,18 @@ +# From https://github.com/jupyter-server/jupyter_server/blob/fc0ac3236fdd92778ea765db6e8982212c8389ee/jupyter_server/config_manager.py#L14 +def recursive_update(target: dict, new: dict) -> None: + """ + Recursively update one dictionary in-place using another. + + None values will delete their keys. + """ + for k, v in new.items(): + if isinstance(v, dict): + if k not in target: + target[k] = {} + recursive_update(target[k], v) + + elif v is None: + target.pop(k, None) + + else: + target[k] = v \ No newline at end of file From bd68510e55693e97d7a5189f497057a648f43a52 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 23:41:23 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../web/management/commands/evaluator.py | 17 ++++++++++++----- frx_challenges/web/utils.py | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/frx_challenges/web/management/commands/evaluator.py b/frx_challenges/web/management/commands/evaluator.py index da48b9b..c7f6598 100644 --- a/frx_challenges/web/management/commands/evaluator.py +++ b/frx_challenges/web/management/commands/evaluator.py @@ -12,9 +12,9 @@ import fsspec from django.conf import settings from django.core.management.base import BaseCommand +from web.utils import recursive_update from ...models import Evaluation -from web.utils import recursive_update logger = logging.getLogger() @@ -86,19 +86,26 @@ async def start_evaluation(self, input_uri): if settings.EVALUATOR_DOCKER_EXTRA_HOST_CONFIG: recursive_update(host_config, settings.EVALUATOR_DOCKER_EXTRA_HOST_CONFIG) - config={ + config = { "Image": self.image, "Cmd": cmd, "HostConfig": host_config, } if settings.EVALUATOR_DOCKER_CONTAINER_ENV: - if isinstance(settings.EVALUATOR_DOCKER_CONTAINER_ENV, collections.abc.Mapping): - config["Env"] = [f"{k}={v}" for k, v in settings.EVALUATOR_DOCKER_CONTAINER_ENV.items()] + if isinstance( + settings.EVALUATOR_DOCKER_CONTAINER_ENV, collections.abc.Mapping + ): + config["Env"] = [ + f"{k}={v}" + for k, v in settings.EVALUATOR_DOCKER_CONTAINER_ENV.items() + ] elif isinstance(settings.EVALUATOR_DOCKER_CONTAINER_ENV, list): config["Env"] = settings.EVALUATOR_DOCKER_CONTAINER_ENV else: - raise RuntimeError(f"settings.EVALUATOR_DOCKER_CONTAINER_ENV should be a dict or list, found {type( settings.EVALUATOR_DOCKER_CONTAINER_ENV)}") + raise RuntimeError( + f"settings.EVALUATOR_DOCKER_CONTAINER_ENV should be a dict or list, found {type( settings.EVALUATOR_DOCKER_CONTAINER_ENV)}" + ) container = await self.docker.containers.create(config) diff --git a/frx_challenges/web/utils.py b/frx_challenges/web/utils.py index cc2107e..f678473 100644 --- a/frx_challenges/web/utils.py +++ b/frx_challenges/web/utils.py @@ -15,4 +15,4 @@ def recursive_update(target: dict, new: dict) -> None: target.pop(k, None) else: - target[k] = v \ No newline at end of file + target[k] = v