wip

src/discord-cluster-manager/cogs/github_cog.py

@@ -53,6 +53,11 @@ async def _run_submission(
         await status.push("Downloading artifacts...")
 
         artifacts = await run.download_artifacts()
+        if "run_results" not in artifacts:
+            logger.error("Could not find `run_results` among artifacts: %s", artifacts.keys())
+            await status.push("Downloading artifacts...  failed")
+            return FullResult(success=False, error="Could not download artifacts", compile=None, run=None)
+
         logs = artifacts["run-result"]["result.json"].decode("utf-8")
 
         await status.update("Downloading artifacts... done")

src/discord-cluster-manager/cogs/verify_run_cog.py

@@ -8,6 +8,8 @@
 from cogs.modal_cog import ModalCog
 from discord import app_commands
 from discord.ext import commands
+
+from consts import SubmissionMode
 from task import make_task
 from utils import send_discord_message, setup_logging
 
@@ -56,7 +58,7 @@ async def verify_github_run(
             )
             task = make_task("examples/identity_cuda")
 
-        github_thread, _ = await github_command(interaction, sub_code, choice, task=task)
+        github_thread, _ = await github_command(interaction, sub_code, choice, task=task, mode=SubmissionMode.TEST)
 
         message_contents = [msg.content async for msg in github_thread.history(limit=None)]
 
@@ -105,7 +107,7 @@ async def verify_modal_run(
             )
             task = make_task("examples/identity_cuda")
 
-        modal_thread, _ = await modal_command(interaction, sub_code, t4, task=task)
+        modal_thread, _ = await modal_command(interaction, sub_code, t4, task=task, mode=SubmissionMode.TEST)
 
         message_contents = [msg.content async for msg in modal_thread.history(limit=None)]
 

src/discord-cluster-manager/consts.py

@@ -42,6 +42,10 @@ class ExitCode(IntEnum):
     PIPE_FAILED = 111
     # didn't crash, but tests failed
     VALIDATE_FAIL = 112
+    # problem parsing test/benchmark
+    TEST_SPEC = 113
+    # process was shut down because it timed out
+    TIMEOUT_EXPIRED = 114
 
 
 class SubmissionMode(Enum):

src/discord-cluster-manager/run_eval.py

@@ -186,7 +186,7 @@ def compile_cuda_script(  # # noqa: C901
     )
 
 
-def run_program(args: list[str], seed: int) -> RunResult:
+def run_program(args: list[str], seed: int, timeout: int = 30) -> RunResult:
     print("[Running]")
     # set up a pipe so the tester can communicate its verdict with us
     env = os.environ.copy()
@@ -195,15 +195,27 @@ def run_program(args: list[str], seed: int) -> RunResult:
     env["POPCORN_SEED"] = str(seed)
 
     execution_start_time = time.perf_counter()
-    run_process = subprocess.run(
-        args,
-        capture_output=True,
-        text=True,
-        check=False,
-        env=env,
-        pass_fds=[pipe_write],
-        timeout=10
-    )
+    try:
+        run_process = subprocess.run(
+            args,
+            capture_output=True,
+            text=True,
+            check=False,
+            env=env,
+            pass_fds=[pipe_write],
+            timeout=timeout
+        )
+    except subprocess.TimeoutExpired as e:
+        return RunResult(
+            success=False,
+            passed=False,
+            command=_make_cmd(e.cmd),
+            stdout=_limit_length(e.stdout),
+            stderr=_limit_length(e.stderr),
+            exit_code=ExitCode.TIMEOUT_EXPIRED,
+            duration=timeout,
+            result=None,
+        )
     execution_end_time = time.perf_counter()
 
     # terminate output writing