Update evaluation scripts

Makefile.toml

@@ -65,17 +65,17 @@ parallel = true
 # Evaluation Crap #
 # ############### #
 
-[tasks.evaluate-inertia]
+[tasks.eval]
 command = "node"
 args = ["ide/packages/evaluation/dist/evaluation.cjs", "-h", "${@}"]
 dependencies = ["build"]
 
-[tasks.evaluation-init]
+[tasks.eval-init]
 command = "node"
 args = ["ide/packages/evaluation/dist/evaluation.cjs", "-s", "./data", "${@}"]
 dependencies = ["build"]
 
-[tasks.evaluation-serve]
+[tasks.eval-serve]
 script = """
 python3 -m webbrowser http://localhost:8080/eval
 ./scripts/evaluation.scm

ide/packages/common/src/TreeInfo.ts

@@ -6,13 +6,15 @@ import type {
   EvaluationResult,
   GoalIdx,
   GoalKind,
+  Heuristic,
   Implementors,
   ProofNodeIdx,
   ResultIdx,
   SerializedTree,
   SetHeuristic,
   TreeTopology
 } from "./bindings";
+import type { SortStrategy } from "./lib";
 
 export type TreeViewWithRoot = TreeView & { root: ProofNodeIdx };
 
@@ -280,7 +282,26 @@ export class TreeInfo {
     return this.tree.root;
   }
 
-  public failedSets() {
+  public numFailedSets() {
+    return this.failedSets().length;
+  }
+
+  public failedSetsSorted(sortAs: SortStrategy = "inertia"): SetHeuristic[] {
+    const sets = this.failedSets();
+
+    switch (sortAs) {
+      case "inertia":
+        return _.sortBy(sets, TreeInfo.setInertia);
+      case "depth":
+        return _.sortBy(sets, s => this.setDepth(s));
+      case "vars":
+        return _.sortBy(sets, s => this.setInferVars(s));
+      default:
+        throw new Error("Unknown sort strategy");
+    }
+  }
+
+  private failedSets(): SetHeuristic[] {
     if (this.showHidden) return this.tree.analysis.problematicSets;
 
     const setHasBadUnification = (s: SetHeuristic) =>
@@ -360,13 +381,6 @@ export class TreeInfo {
 
   public children(n: ProofNodeIdx): ProofNodeIdx[] {
     const nodesToUnifyFailures = this.nodesInUnificationFailurePath();
-
-    // if (_.includes(nodesToUnifyFailures, 6222)) {
-    //   throw new Error("NODE NOT THERE");
-    // } else {
-    //   console.debug("Nodes to unify failures includes 6222");
-    // }
-
     const children = this.view.topology.children[n] ?? [];
     return _.difference(children, nodesToUnifyFailures);
   }
@@ -444,6 +458,16 @@ export class TreeInfo {
     return set.momentum;
   };
 
+  public setDepth(set: SetHeuristic) {
+    const heuristicDepth = (h: Heuristic) => this.depth(h.idx);
+    return _.sum(_.map(set.goals, heuristicDepth));
+  }
+
+  public setInferVars(set: SetHeuristic) {
+    const heuristicVars = (h: Heuristic) => this.inferVars(h.idx);
+    return _.sum(_.map(set.goals, heuristicVars));
+  }
+
   public minInertiaOnPath(n: ProofNodeIdx): number {
     const hs = _.filter(this.failedSets(), h =>
       _.some(h.goals, g => _.includes(this.pathToRoot(g.idx).pathInclusive, n))

ide/packages/common/src/context.ts

@@ -3,12 +3,7 @@ import { createContext } from "react";
 import type BodyInfo from "./BodyInfo";
 import type TreeInfo from "./TreeInfo";
 import type { MessageSystem, TreeRenderParams } from "./communication";
-import type {
-  EvaluationMode,
-  Filename,
-  PanoptesConfig,
-  SystemSpec
-} from "./lib";
+import type { Filename, PanoptesConfig, SystemSpec } from "./lib";
 
 export const settingsToggles = ["show-hidden-obligations"] as const;
 
@@ -18,9 +13,9 @@ export type Settings = {
 
 export const AppContext = {
   MessageSystemContext: createContext<MessageSystem | undefined>(undefined),
-  ConfigurationContext: createContext<
-    (PanoptesConfig & { evalMode: EvaluationMode }) | undefined
-  >(undefined),
+  ConfigurationContext: createContext<Required<PanoptesConfig> | undefined>(
+    undefined
+  ),
   SystemSpecContext: createContext<SystemSpec | undefined>(undefined),
   SettingsContext: createContext<Settings>({ "show-hidden-obligations": false })
 };

ide/packages/common/src/lib.ts

@@ -31,10 +31,13 @@ export const ConfigConsts = {
 export type PanoptesOptionalData = {
   target?: ErrorJumpTargetInfo;
   evalMode?: EvaluationMode;
+  rankMode?: SortStrategy;
 };
 
 export type SystemSpec = Omit<IssueOptions, "logText">;
-export type EvaluationMode = "release" | "rank" | "random";
+export type EvaluationMode = "release" | "evaluate";
+export type SortStrategy = "inertia" | "depth" | "vars";
+
 export interface FileInfo {
   fn: Filename;
   data: ObligationsInBody[];

ide/packages/evaluation/src/basic.ts

@@ -32,7 +32,7 @@ async function createWorkspaceRunner() {
         console.debug(`MISSING: cause ${workspace}/${filename}`);
         return;
       }
-      const page = await openPage(context, filename, bundles, "rank");
+      const page = await openPage(context, filename, bundles);
 
       await sleep(5000);
       await expandBottomUpView(page);
@@ -63,8 +63,6 @@ async function createWorkspaceRunner() {
       return {
         workspace,
         filename,
-        cause: cause.message,
-        numberTreeNodes,
         rank
       };
     });

ide/packages/evaluation/src/main.ts

@@ -1,9 +1,7 @@
 import _ from "lodash";
 
 import fs from "node:fs";
-
 import { run as runBasic } from "./basic";
-import { run as runRandom } from "./random";
 import { rootCauses as TEST_CAUSES } from "./rootCauses";
 import { PORT, withServerOnPort } from "./serve";
 import { run as runVisual } from "./visual";
@@ -12,6 +10,7 @@ declare global {
   var debugging: boolean;
   var testMatcher: string;
   var outputFile: string;
+  var rankBy: string;
 }
 
 async function runForJson<T>(func: () => Promise<T>) {
@@ -21,28 +20,23 @@ async function runForJson<T>(func: () => Promise<T>) {
 async function main() {
   const hasTestMatcher = _.find(process.argv, arg => arg.startsWith("--test="));
   const hasOutputFile = _.find(process.argv, arg => arg.startsWith("--ofile="));
+  let hasRankBy = _.find(process.argv, arg => arg.startsWith("--rankBy="));
 
   global.debugging = _.includes(process.argv, "--debug");
-  global.outputFile = hasOutputFile
-    ? hasOutputFile?.split("=")[1]
-    : "heuristic-precision.csv";
   global.testMatcher = hasTestMatcher
     ? hasTestMatcher.split("=")[1]
     : "[\\s\\S]*";
+  global.rankBy = hasRankBy ? hasRankBy.split("=")[1] : "inertia";
+  global.outputFile = hasOutputFile
+    ? hasOutputFile?.split("=")[1]
+    : `heuristic-precision[${global.rankBy}].csv`;
 
   const argv = _.filter(
     process.argv,
     arg => arg !== "--debug" && !arg.startsWith("--test")
   );
 
   switch (argv[2]) {
-    case "-r": {
-      const N = Number(argv[3] ?? "10");
-      await withServerOnPort(PORT, () =>
-        runForJson(() => runRandom(N, TEST_CAUSES))
-      ).then(console.log);
-      break;
-    }
     case "-h": {
       await withServerOnPort(PORT, () => runBasic(TEST_CAUSES)).then(
         writeToCSV(global.outputFile)

ide/packages/evaluation/src/page.ts

@@ -6,6 +6,7 @@ import {
   type EvaluationMode,
   type Filename,
   type PanoptesConfig,
+  type SortStrategy,
   configToString
 } from "@argus/common/lib";
 import _ from "lodash";
@@ -741,13 +742,15 @@ export function webHtml(
   title: string,
   filename: Filename,
   bundles: BodyBundle[],
-  evalMode: EvaluationMode = "rank"
+  rankMode: SortStrategy,
+  evalMode: EvaluationMode = "evaluate"
 ) {
   const config: PanoptesConfig = {
     type: "WEB_BUNDLE",
     target: findErrorTargetInBundles(bundles),
     closedSystem: bundles,
-    evalMode
+    evalMode,
+    rankMode
   };
 
   const panoptesDir = path.resolve(__dirname, "..", "..", "panoptes");

ide/packages/evaluation/src/utils.ts

@@ -1,6 +1,6 @@
 import fs from "node:fs";
 import type { BodyBundle } from "@argus/common/bindings";
-import type { EvaluationMode, Filename, Result } from "@argus/common/lib";
+import type { Filename, Result, SortStrategy } from "@argus/common/lib";
 import { type ExecNotifyOpts, execNotify as _execNotify } from "@argus/system";
 import _ from "lodash";
 import type { BrowserContext, Page } from "playwright";
@@ -90,14 +90,21 @@ export async function forFileInBundle<T>(
   );
 }
 
+function sanitizeSort(s: string | undefined): s is SortStrategy {
+  return s !== undefined && (s === "inertia" || s === "depth" || s === "vars");
+}
+
 export async function openPage(
   context: BrowserContext,
   filename: string,
-  bundles: BodyBundle[],
-  evalMode: EvaluationMode
+  bundles: BodyBundle[]
 ) {
+  if (!sanitizeSort(global.rankBy)) {
+    throw new Error(`Invalid rank mode: ${global.rankBy}`);
+  }
+
   const tmpobj = tmp.fileSync({ postfix: ".html" });
-  const html = webHtml("EVAL", filename, bundles, evalMode);
+  const html = webHtml("EVAL", filename, bundles, global.rankBy, "evaluate");
   fs.writeSync(tmpobj.fd, html);
   const page = await context.newPage();
   await page.goto(`file://${tmpobj.name}`, {

ide/packages/evaluation/src/visual.ts

@@ -38,7 +38,7 @@ async function argusScreenshots(
     filename: Filename,
     bundles: BodyBundle[]
   ) => {
-    const page = await openPage(context, filename, bundles, "rank");
+    const page = await openPage(context, filename, bundles);
     await expandBottomUpView(page);
     await page.screenshot({ path: out, fullPage: true });
   };

ide/packages/panoptes/src/App.tsx

@@ -9,7 +9,6 @@ import {
   settingsToggles
 } from "@argus/common/context";
 import {
-  type EvaluationMode,
   type FileInfo,
   type PanoptesConfig,
   type SystemSpec,
@@ -245,8 +244,9 @@ const App = observer(({ config }: { config: PanoptesConfig }) => {
     config.type === "VSCODE_BACKING" ? config.spec : webSysSpec;
 
   config.evalMode = config.evalMode ?? "release";
-  const configNoUndef: PanoptesConfig & { evalMode: EvaluationMode } =
-    config as any;
+  config.rankMode = config.rankMode ?? "inertia";
+
+  const configNoUndef: Required<PanoptesConfig> = config as any;
 
   useEffect(() => {
     const listen = (e: MessageEvent) =>