🔀 Merge branch 'main' into rename-frx-challenges

.github/workflows/deploy-docs.yml

@@ -0,0 +1,47 @@
+# This file was created automatically with `myst init --gh-pages` 🪄 💚
+
+name: MyST GitHub Pages Deploy
+on:
+  push:
+    # Runs on pushes targeting the default branch
+    branches: [main]
+env:
+  # `BASE_URL` determines the website is served from, including CSS & JS assets
+  # You may need to change this to `BASE_URL: ''`
+  BASE_URL: /${{ github.event.repository.name }}
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+jobs:
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18.x
+      - name: Install MyST Markdown
+        run: npm install -g mystmd
+      - name: Build HTML Assets
+        working-directory: ./docs
+        run: myst build --html
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: "./docs/_build/html"
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/publish-chart.yaml

@@ -76,6 +76,23 @@ jobs:
         run: |
           docker login -u "${{ secrets.QUAY_USERNAME }}" -p "${{ secrets.QUAY_PASSWORD }}" quay.io
 
+      - name: Set up push rights to frx-challenges-helm-chart
+        # This was setup by...
+        # 1. Generating a private/public key pair:
+        #    ssh-keygen -t rsa -b 4096 -C "frx-challenges-helm-chart" -f gh-pages -N ""
+        # 2. Registering the private key (gh-pages) as a secret for this
+        #    repo:
+        #    https://github.com/2i2c-org/frx-challenges/settings/secrets/actions
+        # 3. Registering the public key (gh-pages.pub) as a deploy key
+        #    with push rights for the 2i2c-org/frx-challenges-helm-chart repo:
+        #    https://github.com/2i2c-org/frx-challenges-helm-chart/settings/keys
+        if: steps.publishing.outputs.publishing
+        run: |
+          mkdir -p ~/.ssh
+          ssh-keyscan github.com >> ~/.ssh/known_hosts
+          echo "${{ secrets.ACTIONS_DEPLOY_KEY }}" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+
       - name: Install dependencies
         run: |
           pip install -r dev-requirements.txt
@@ -84,8 +101,8 @@ jobs:
 
       - name: Configure a git user
         run: |
-          git config --global user.email "github-actions@example.local"
-          git config --global user.name "GitHub Actions user"
+          git config --global user.email "github-actions@github.com"
+          git config --global user.name "github-actions"
 
       - name: Build image, push if necessary
         env:
@@ -102,8 +119,6 @@ jobs:
 
       - name: Publish chart with chartpress
         if: steps.publishing.outputs.publishing
-        env:
-          GITHUB_TOKEN: "${{ github.token }}"
         run: |
           set -eux
 

.gitignore

@@ -160,3 +160,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# MyST
+_build

.readthedocs.yaml

@@ -0,0 +1,20 @@
+# Configuration on how ReadTheDocs (RTD) builds our documentation
+# ref: https://readthedocs.org/projects/frx-challenges/
+# ref: https://docs.readthedocs.io/en/stable/config-file/v2.html
+#
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    nodejs: "18"
+  commands:
+    - mkdir $READTHEDOCS_OUTPUT
+    - cd docs
+    - pip install -r requirements.txt
+    - myst build --html --ci --check-links
+    - cp --recursive _build/* $READTHEDOCS_OUTPUT/
+
+# Explicitly opt out of trying to build additional formats such as PDF and ePub
+formats: []

README.md

@@ -1 +1,8 @@
-# frx-challenges
+# FRX Challenges
+
+## The Helm chart
+
+The Helm chart lets a user create a reproducible and maintainable
+deployment of FRX Challenges on a Kubernetes cluster in a cloud environment. The
+released charts are made available in our [Helm chart
+repository](https://2i2c.org/frx-challenges-helm-chart).

chartpress.yaml

@@ -27,8 +27,8 @@ charts:
     baseVersion: patch
 
     repo:
-      git: 2i2c-org/frx-challenges
-      published: https://2i2c.org/frx-challenges/
+      git: 2i2c-org/frx-challenges-helm-chart
+      published: https://2i2c.org/frx-challenges-helm-chart/
 
     images:
       frx-challenges:

comptest/comptest/settings.py

@@ -224,6 +224,7 @@
 EVALUATOR_DOCKER_DISABLE_NETWORK = True
 EVALUATOR_DOCKER_EXTRA_BINDS = []
 EVALUATOR_DOCKER_AUTH = {}
+MAX_RUNNING_EVALUATIONS = 3
 # If you want to use a GCP Artifact Registry service key, use the following config:
 # for EVALUATOR_DOCKER_AUTH
 # {

comptest/web/management/commands/evaluator.py

@@ -8,6 +8,7 @@
 import aiodocker
 import aiodocker.containers
 import fsspec
+from asgiref.sync import sync_to_async
 from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.db.models import Exists, OuterRef, Q
@@ -108,10 +109,19 @@ async def is_still_running(self, state: dict) -> bool:
             if e.status == 404:
                 return False
             raise
+
         return container["State"].get("Status") == "running"
 
     async def get_result(self, state: dict) -> dict:
-        container = await self.docker.containers.get(container_id=state["container_id"])
+        try:
+            container = await self.docker.containers.get(
+                container_id=state["container_id"]
+            )
+        except aiodocker.DockerError as e:
+            if e.status == 404:
+                return None
+            raise
+
         logger.debug(f"Container state: {container['State']}")
 
         success = (
@@ -167,19 +177,31 @@ async def ahandle(self):
                 e = Evaluation(version=v)
                 await e.asave()
 
-            # Start Evaluations when they have not been started yet
+            # Get the Evaluations that have not been started yet
             unstarted_evaluations = Evaluation.objects.select_related("version").filter(
                 status=Evaluation.Status.NOT_STARTED
             )
 
-            async for e in unstarted_evaluations:
-                await self.start_evaluation(evaluator, e)
-
-            # Check running evaluations
+            # Get the running Evaluation objects
             running_evaluations = Evaluation.objects.select_related("version").filter(
                 status=Evaluation.Status.EVALUATING
             )
 
+            # Get the number of running evaluations
+            # This is a synchronous operation so it's wrapped in sync_to_async
+            num = await sync_to_async(running_evaluations.count)()
+            async for e in unstarted_evaluations:
+                # Only start a new evaluation if the number of running evaluations is less than the maximum
+                if num < settings.MAX_RUNNING_EVALUATIONS:
+                    await self.start_evaluation(evaluator, e)
+                    # Instead of checking the database again, we can just increment the counter
+                    # And the database will be checked in the next iteration
+                    num += 1
+                else:
+                    logger.warning(
+                        "Maximum number of running evaluations reached, waiting for some to finish"
+                    )
+
             async for e in running_evaluations:
                 await self.process_running_evaluation(evaluator, e)
 

docs/.gitignore

@@ -0,0 +1,2 @@
+# MyST build outputs
+/_build/

docs/CONTRIBUTING.md

@@ -0,0 +1,34 @@
+---
+title: Contribute to the documentation
+short_title: Write Documentation
+---
+
+## Build the documentation locally
+
+To build the documentation:
+
+1. Clone this repository:
+
+   ```bash
+   git clone https://github.com/2i2c-org/unnamed-thingity-thing
+   ```
+
+1. Navigate to the docs folder:
+
+   ```bash
+   cd docs/
+   ```
+
+1. Install the software packages for documentation:
+
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+1. Start a MyST server to preview the documentation:
+
+   ```bash
+   myst start
+   ```
+
+This will build the documentation locally so that you can preview what your changes will look like.

docs/README.md

@@ -0,0 +1,5 @@
+# Docs
+
+This folder contains the source files for the documentation.
+
+See the [contributing guide](CONTRIBUTING.md) for how to build and contribute to this project's documentation.

docs/index.md

@@ -0,0 +1,18 @@
+# FRX Challenges
+
+Welcome to the FRX Challenges project! This is an open source repository that provides key software components for running competitive data science challenges.
+
+:::{card} Getting Started
+:link: quickstart.md
+See our quickstart guide to deploying this project.
+:::
+
+## Features
+
+- submissions!
+- evaluators!
+- teams!
+
+## Goals
+
+FRX stands for **F**rictionless **R**eproducibility e**X**change. Inspired by [Donoho, 2023](https://arxiv.org/abs/2310.00865v1).

docs/myst.yml

@@ -0,0 +1,26 @@
+# See docs at: https://mystmd.org/guide/frontmatter
+version: 1
+project:
+  id: 73942dbd-1ba2-4a30-a345-526e897b7fb4
+  title: FRX Challenges
+  # description:
+  # keywords: []
+  # authors: []
+  github: https://github.com/2i2c-org/unnamed-thingity-thing
+  license:
+    code: BSD-3-Clause
+    content: CC-BY-4.0
+  toc:
+    # Auto-generated by `myst init --write-toc`
+    - file: index.md
+    - file: quickstart.md
+    - title: Contribute
+      children:
+        - file: CONTRIBUTING.md
+
+site:
+  template: book-theme
+  options:
+    hide_outline: true
+    logo: images/logo.png
+  #   favicon: favicon.ico

docs/quickstart.md

@@ -0,0 +1,7 @@
+---
+title: Get Started
+subject: Quickstart guide
+description: Get up and running deploying an FRX Challenge.
+---
+
+This is the quickstart guide to get you started with your first deployment of an FRX Challenge website.

docs/requirements.txt

@@ -0,0 +1 @@
+mystmd

helm-chart/templates/_helpers.tpl

@@ -65,14 +65,3 @@ app.kubernetes.io/instance: {{ .Release.Name }}
   {{- end }}
 {{- end }}
 {{- end }}
-
-{{/*
-Adds quotes to each element of a list and returns them as string "a b c ..."
-*/}}
-{{- define "quoteList" -}}
-{{- $out := list -}}
-{{- range $index, $element := . -}}
-{{- $out = append $out $element -}}
-{{- end -}}
-{{- $out | join " " -}}
-{{- end -}}

helm-chart/templates/deployment.yaml

@@ -63,10 +63,17 @@ spec:
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           workingDir: /opt/frx-challenges/comptest
           command:
+          {{- if .Values.adminUsers }}
             - python
             - manage.py
             - promote
-            - '{{ include "quoteList" .Values.adminUsers }}'
+            {{- range .Values.adminUsers }}
+            - {{ . }}
+            {{- end }}
+          {{- else }}
+            - echo
+            - "No admin users to add"
+          {{- end }}
           securityContext:
             runAsUser: 0
           volumeMounts: