Introduce Calculate Elo API for external use.

This should make it possible for external use to calculate the ELo based on Either (Ptnml or WDL) and elo0, elo1 bounds (elo_model is optional and defaults to "normalized"). The format is: https://tests.stockfishchess.org/api/calc_elo?LL=93&LD=18711&DDWL=58877&WD=18739&WW=100&elo0=-1.75&elo1=0.25 needs testing. Introduce Calculate Elo API for external use. This should make it possible for external use to calculate the ELo based on Either (Ptnml or WDL) and elo0, elo1 bounds (elo_model is optional and defaults to "normalized"). The format is: https://tests.stockfishchess.org/api/calc_elo?LL=93&LD=18711&DDWL=58877&WD=18739&WW=100&elo0=-1.75&elo1=0.25 needs testing. Introduce Calculate Elo API for external use. This should make it possible for external use to calculate the ELo based on Either (Ptnml or WDL) and elo0, elo1 bounds (elo_model is optional and defaults to "normalized"). The format is: https://tests.stockfishchess.org/api/calc_elo?LL=93&LD=18711&DDWL=58877&WD=18739&WW=100&elo0=-1.75&elo1=0.25 needs testing.
official-stockfish · Jul 9, 2023 · db543fb · db543fb
1 parent 1c3f5b3
commit db543fb
Show file tree

Hide file tree

Showing 10 changed files with 136 additions and 48 deletions.
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -51,7 +51,7 @@ jobs:
         id: prettier
         continue-on-error: true
         run: |
-          prettier --check "server/fishtest/static/{css/*.css,html/*.html,js/*.js}"
+          prettier --check "server/fishtest/static/{css/*.css,html/*.html,js/*.js}" --trailing-comma none
 
       - name: Check linters and formatters status
         run: |
@@ -67,4 +67,4 @@ jobs:
           echo "Run the following commands to format the code:"
           echo "black --exclude='env|packages' ."
           echo "isort --profile black --skip env --skip venv --skip worker/packages ."
-          echo "npx prettier --write 'server/fishtest/static/{css/*.css,html/*.html,js/*.js}'"
+          echo "npx prettier --write 'server/fishtest/static/{css/*.css,html/*.html,js/*.js}' --trailing-comma none"
diff --git a/server/fishtest/__init__.py b/server/fishtest/__init__.py
@@ -118,6 +118,7 @@ def group_finder(username, request):
     config.add_route("api_download_nn", "/api/nn/{id}")
     config.add_route("api_get_elo", "/api/get_elo/{id}")
     config.add_route("api_actions", "/api/actions")
+    config.add_route("api_calc_elo", "/api/calc_elo")
 
     config.scan()
     return config.make_wsgi_app()
diff --git a/server/fishtest/api.py b/server/fishtest/api.py
@@ -347,6 +347,91 @@ def get_elo(self):
         run["elo"] = a
         return run
 
+    @view_config(route_name="api_calc_elo")
+    def calc_elo(self):
+        W = self.request.params.get("W")
+        D = self.request.params.get("D")
+        L = self.request.params.get("L")
+        LL = self.request.params.get("LL")
+        LD = self.request.params.get("LD")
+        DDWL = self.request.params.get("DDWL")
+        WD = self.request.params.get("WD")
+        WW = self.request.params.get("WW")
+        elo0 = self.request.params.get("elo0")
+        elo1 = self.request.params.get("elo1")
+        elo_model = self.request.params.get("elo_model", "normalized")
+
+        if elo_model not in ["BayesElo", "logistic", "normalized"]:
+            self.handle_error(
+                "Valid Elo models are: BayesElo, logistic, and normalized."
+            )
+
+        isPtnml = all(
+            value is not None
+            and (
+                (
+                    isinstance(value, (int, float))
+                    or isinstance(value, str)
+                    and value.replace(".", "").replace("-", "").isdigit()
+                )
+                and float(value) >= 0
+            )
+            for value in [LL, LD, DDWL, WD, WW, elo0, elo1]
+        )
+
+        isWdl = not isPtnml and all(
+            value is not None
+            and (
+                (
+                    isinstance(value, (int, float))
+                    or isinstance(value, str)
+                    and value.replace(".", "").replace("-", "").isdigit()
+                )
+                and float(value) >= 0
+            )
+            for value in [W, D, L, elo0, elo1]
+        )
+
+        if not isPtnml and not isWdl:
+            self.handle_error(
+                "Invalid or missing parameters. Please provide all values as valid numbers."
+            )
+
+        if isPtnml:
+            LL = int(LL)
+            LD = int(LD)
+            DDWL = int(DDWL)
+            WD = int(WD)
+            WW = int(WW)
+            if (LL + LD + DDWL + WD + WW) * 2 > 2**32:
+                self.handle_error("Number of games exceeds the limit.")
+            if LL + LD + DDWL + WD + WW == 0:
+                self.handle_error("No games to calculate Elo.")
+            results = {
+                "pentanomial": [LL, LD, DDWL, WD, WW],
+            }
+        if isWdl:
+            W = int(W)
+            D = int(D)
+            L = int(L)
+            if W + D + L > 2**32:
+                self.handle_error("Number of games exceeds the limit.")
+            if W + D + L == 0:
+                self.handle_error("No games to calculate Elo.")
+            results = {
+                "wins": W,
+                "draws": D,
+                "losses": L,
+            }
+        elo0 = float(elo0)
+        elo1 = float(elo1)
+        alpha = 0.05
+        beta = 0.05
+        a = SPRT_elo(
+            results, alpha=alpha, beta=beta, elo0=elo0, elo1=elo1, elo_model=elo_model
+        )
+        return a
+
     @view_config(route_name="api_request_task")
     def request_task(self):
         self.validate_request("/api/request_task")

diff --git a/server/fishtest/static/html/maintenance.html b/server/fishtest/static/html/maintenance.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!doctype html>
 <html lang="en">
   <head>
     <title>Site Maintenance | Stockfish Testing Framework</title>
@@ -9,7 +9,9 @@
       body {
         text-align: center;
         padding: 20px;
-        font: 20px Helvetica, sans-serif;
+        font:
+          20px Helvetica,
+          sans-serif;
         color: #b1ada7;
         background-color: #111;
       }

diff --git a/server/fishtest/static/js/application.js b/server/fishtest/static/js/application.js
@@ -127,8 +127,8 @@ document.addEventListener("DOMContentLoaded", () => {
     await fetch("/logout", {
       method: "POST",
       headers: {
-        "X-CSRF-Token": csrfToken,
-      },
+        "X-CSRF-Token": csrfToken
+      }
     });
     window.location = "/";
   });
@@ -166,7 +166,7 @@ function notify(title, body, link, fallback) {
     const notification = new Notification(title, {
       body: body,
       requireInteraction: true,
-      icon: "https://tests.stockfishchess.org/img/stockfish.png",
+      icon: "https://tests.stockfishchess.org/img/stockfish.png"
     });
     notification.onclick = () => {
       window.open(link, "_blank");
@@ -196,9 +196,9 @@ async function fetch_post(url, payload) {
     mode: "cors",
     cache: "no-cache",
     headers: {
-      "Content-Type": "application/json",
+      "Content-Type": "application/json"
     },
-    body: JSON.stringify(payload),
+    body: JSON.stringify(payload)
   };
   return fetch_json(url, options);
 }

diff --git a/server/fishtest/static/js/calc.js b/server/fishtest/static/js/calc.js
@@ -5,7 +5,7 @@ const defaultParameters = {
   "elo-0": "0.0",
   "elo-1": "2.0",
   "draw-ratio": "0.49",
-  "rms-bias": "191",
+  "rms-bias": "191"
 };
 
 google.charts.load("current", { packages: ["corechart"] });
@@ -100,7 +100,7 @@ function draw_charts() {
   expected_chart.loaded = false;
   const data_pass = [["Elo", { role: "annotation" }, "Pass Probability"]];
   const data_expected = [
-    ["Elo", { role: "annotation" }, "Expected Number of Games"],
+    ["Elo", { role: "annotation" }, "Expected Number of Games"]
   ];
   const d = elo1 - elo0;
   const elo_start = Math.floor(elo0 - d / 3);
@@ -121,7 +121,7 @@ function draw_charts() {
     data_expected.push([
       elo,
       null,
-      { v: c[1], f: (c[1] / 1000).toFixed(1) + "K" },
+      { v: c[1], f: (c[1] / 1000).toFixed(1) + "K" }
     ]);
     for (const elo_ of specials) {
       if (elo < elo_ && elo_next >= elo_) {
@@ -130,12 +130,12 @@ function draw_charts() {
         data_pass.push([
           elo_,
           elo_,
-          { v: c_[0], f: (c_[0] * 100).toFixed(1) + "%" },
+          { v: c_[0], f: (c_[0] * 100).toFixed(1) + "%" }
         ]);
         data_expected.push([
           elo_,
           elo_,
-          { v: c_[1], f: (c_[1] / 1000).toFixed(1) + "K" },
+          { v: c_[1], f: (c_[1] / 1000).toFixed(1) + "K" }
         ]);
       }
     }
@@ -148,12 +148,12 @@ function draw_charts() {
     color: "#999",
     fontSize: 16,
     bold: true,
-    italic: false,
+    italic: false
   };
 
   const options = {
     legend: {
-      position: "none",
+      position: "none"
     },
     curveType: "function",
     hAxis: {
@@ -162,35 +162,35 @@ function draw_charts() {
       textStyle: chart_text_style,
       gridlines: {
         count: elo_end - elo_start,
-        color: "#666",
+        color: "#666"
       },
-      minorGridlines: minor_gridlines_style,
+      minorGridlines: minor_gridlines_style
     },
     vAxis: {
       title: "Pass Probability",
       titleTextStyle: title_text_style,
       textStyle: chart_text_style,
       gridlines: gridlines_style,
       minorGridlines: minor_gridlines_style,
-      format: "percent",
+      format: "percent"
     },
     tooltip: {
-      trigger: "selection",
+      trigger: "selection"
     },
     backgroundColor: {
-      fill: "transparent",
+      fill: "transparent"
     },
     chartArea: {
       left: "15%",
       top: "5%",
       width: "80%",
-      height: "80%",
+      height: "80%"
     },
     annotations: {
       style: "line",
       stem: { color: "orange" },
-      textStyle: title_text_style,
-    },
+      textStyle: title_text_style
+    }
   };
   let data_table = google.visualization.arrayToDataTable(data_pass);
   pass_chart.draw(data_table, options);
@@ -200,7 +200,7 @@ function draw_charts() {
     textStyle: chart_text_style,
     gridlines: gridlines_style,
     minorGridlines: minor_gridlines_style,
-    format: "short",
+    format: "short"
   };
   data_table = google.visualization.arrayToDataTable(data_expected);
   expected_chart.draw(data_table, options);

diff --git a/server/fishtest/static/js/live_elo.js b/server/fishtest/static/js/live_elo.js
@@ -3,7 +3,7 @@
 async function follow_live(test_id) {
   await Promise.all([
     DOM_loaded(),
-    google.charts.load("current", { packages: ["gauge"] }),
+    google.charts.load("current", { packages: ["gauge"] })
   ]);
 
   let LOS_chart = null;
@@ -46,7 +46,7 @@ async function follow_live(test_id) {
     }
     const LOS_chart_data = google.visualization.arrayToDataTable([
       ["Label", "Value"],
-      ["LOS", Math.round(1000 * LOS) / 10],
+      ["LOS", Math.round(1000 * LOS) / 10]
     ]);
     const LOS_chart_options = {
       width: 500,
@@ -57,13 +57,13 @@ async function follow_live(test_id) {
       yellowTo: 95,
       redFrom: 0,
       redTo: 5,
-      minorTicks: 5,
+      minorTicks: 5
     };
     LOS_chart.draw(LOS_chart_data, LOS_chart_options);
 
     const LLR_chart_data = google.visualization.arrayToDataTable([
       ["Label", "Value"],
-      ["LLR", Math.round(100 * LLR) / 100],
+      ["LLR", Math.round(100 * LLR) / 100]
     ]);
     a = Math.round(100 * a) / 100;
     b = Math.round(100 * b) / 100;
@@ -78,20 +78,20 @@ async function follow_live(test_id) {
       yellowTo: b,
       max: b,
       min: a,
-      minorTicks: 3,
+      minorTicks: 3
     };
     LLR_chart.draw(LLR_chart_data, LLR_chart_options);
 
     const ELO_chart_data = google.visualization.arrayToDataTable([
       ["Label", "Value"],
-      ["Elo", set_gauges.last_elo],
+      ["Elo", set_gauges.last_elo]
     ]);
     const ELO_chart_options = {
       width: 500,
       height: 150,
       max: 4,
       min: -4,
-      minorTicks: 4,
+      minorTicks: 4
     };
     if (ci_lower < 0 && ci_upper > 0) {
       ELO_chart_options.redFrom = ci_lower;

diff --git a/server/fishtest/static/js/notifications.js b/server/fishtest/static/js/notifications.js
@@ -77,7 +77,7 @@ function LRU(capacity, content) {
         this.content.splice(idx, 1);
         return true;
       }
-    },
+    }
   };
 }
 
@@ -206,10 +206,10 @@ async function purge_notifications(last_fetch_time) {
     try {
       json1 = await fetch_post("/api/actions", {
         action: {
-          $in: ["finished_run", "stop_run", "delete_run"],
+          $in: ["finished_run", "stop_run", "delete_run"]
         },
         run_id: run_id,
-        time: { $gte: last_fetch_time / 1000 - 60 },
+        time: { $gte: last_fetch_time / 1000 - 60 }
       });
     } catch (e) {
       console_log(e, true);
@@ -275,7 +275,7 @@ async function main_follow_loop() {
       if (notifications.count()) {
         json = await fetch_post("/api/actions", {
           action: { $in: ["finished_run", "stop_run", "delete_run"] },
-          run_id: { $in: notifications.toArray() },
+          run_id: { $in: notifications.toArray() }
         });
       }
     } catch (e) {
@@ -302,7 +302,7 @@ async function main_follow_loop() {
       try {
         json = await fetch_post("/api/actions", {
           action: "new_run",
-          run_id: run_id,
+          run_id: run_id
         });
         notify_elo(json[0], entry);
       } catch (e) {