feat: ui temperature stability

Signed-off-by: Nick Mitchell <[email protected]>
IBM · Mar 6, 2025 · 82daf10 · 82daf10
1 parent f0ad647
commit 82daf10
Show file tree

Hide file tree

Showing 7 changed files with 209 additions and 143 deletions.
diff --git a/pdl-live-react/src/view/masonry/MasonryTile.tsx b/pdl-live-react/src/view/masonry/MasonryTile.tsx
@@ -51,10 +51,7 @@ export default function MasonryTile({
   idx,
   footer1Key,
   footer1Value,
-  footer2Key,
-  footer2Value,
-  footer2DetailHeader,
-  footer2DetailBody,
+  stability,
   actions: tileActions = [],
   block,
   run,
@@ -73,8 +70,8 @@ export default function MasonryTile({
           )}
           {tileActions.map((action) =>
             action === "run" ? (
-              <Suspense fallback={<div />}>
-                <RunMenu key="run" run={run} block={block} />
+              <Suspense key="run" fallback={<div />}>
+                <RunMenu run={run} block={block} />
               </Suspense>
             ) : (
               <></>
@@ -117,9 +114,9 @@ export default function MasonryTile({
     </CardHeader>
   )
 
-  const hasFooter = (footer1Key && footer1Value) || (footer2Key && footer2Value)
+  const hasFooter = (footer1Key && footer1Value) || stability
   const footer = hasFooter && (
-    <DescriptionList isCompact isHorizontal isFluid>
+    <DescriptionList isAutoColumnWidths columnModifier={{ default: "3Col" }}>
       {footer1Key && (
         <DescriptionListGroup>
           <DescriptionListTerm>{footer1Key}</DescriptionListTerm>
@@ -128,14 +125,20 @@ export default function MasonryTile({
           </DescriptionListDescription>
         </DescriptionListGroup>
       )}
-      {footer2Key && (
-        <DescriptionListGroup className="pdl-masonry-tile-footer2-dg">
-          <DescriptionListTerm>{footer2Key}</DescriptionListTerm>
-          <DescriptionListDescription>
-            {renderValue(footer2Value, footer2DetailHeader, footer2DetailBody)}
-          </DescriptionListDescription>
-        </DescriptionListGroup>
-      )}
+      {stability &&
+        stability.map((m) => (
+          <DescriptionListGroup
+            key={m.temperature}
+            className="pdl-masonry-tile-footer2-dg"
+          >
+            <DescriptionListTerm>
+              T={m.temperature} Stability
+            </DescriptionListTerm>
+            <DescriptionListDescription>
+              {renderValue(m)}
+            </DescriptionListDescription>
+          </DescriptionListGroup>
+        ))}
     </DescriptionList>
   )
 
@@ -163,11 +166,15 @@ export default function MasonryTile({
   )
 }
 
-function renderValue(
-  value: Props["footer2Value"],
-  detailHeader?: Props["footer2DetailHeader"],
-  detailBody?: Props["footer2DetailBody"],
-) {
+function renderValue({
+  temperature,
+  matrix,
+  results,
+}: import("./stability").BlockWithStabilityMetrics["pdl__stability"][number]) {
+  const value = matrix
+  const detailHeader = `Stability across calls with the same input (temperature=${temperature})`
+  const detailBody = results
+
   if (Array.isArray(value) && detailBody) {
     const lookup: { i: number; j: number }[] = []
     let k = 0

diff --git a/pdl-live-react/src/view/masonry/RunMenu.tsx b/pdl-live-react/src/view/masonry/RunMenu.tsx
@@ -10,7 +10,6 @@ import {
 
 import RunIcon from "@patternfly/react-icons/dist/esm/icons/redo-icon"
 import ModelIcon from "@patternfly/react-icons/dist/esm/icons/theater-masks-icon"
-import IdempotencyIcon from "@patternfly/react-icons/dist/esm/icons/equals-icon"
 import TemperatureIcon from "@patternfly/react-icons/dist/esm/icons/thermometer-half-icon"
 
 const preventOverflow = {
@@ -32,10 +31,10 @@ export default function RunMenu({ block, run }: Props) {
     }
   }, [block, run])
 
-  const runIdempotency = useCallback(async () => {
+  const runTemperature = useCallback(async () => {
     if (block && run) {
-      const { runIdempotencyCheck } = await import("./similarity")
-      await runIdempotencyCheck(block, run)
+      const { runNTimes } = await import("./stability")
+      await runNTimes(block, run, 4, [0, 0.25, 0.5, 0.75, 1, 2])
     }
   }, [block, run])
 
@@ -83,33 +82,24 @@ export default function RunMenu({ block, run }: Props) {
       <DropdownList>
         <DropdownItem
           icon={<RunIcon />}
-          description="Re-run with the same inputs and update to reflect the new response"
+          description="Re-run and update the UI to reflect the new response"
           onClick={runOnce}
         >
           Run Once
         </DropdownItem>
 
-        <DropdownItem
-          icon={<IdempotencyIcon />}
-          description="Run several times to evaluate model stability for a fixed input"
-          onClick={runIdempotency}
-          isDisabled={!window.__TAURI_INTERNALS__}
-        >
-          Analyze Idempotency
-        </DropdownItem>
-
         <DropdownItem
           icon={<TemperatureIcon />}
-          description="Run several times to evaluate model stability across varying temperatures"
-          onClick={runOnce}
-          isDisabled
+          description="Evaluate model stability across a range of temperatures"
+          onClick={runTemperature}
+          isDisabled={!window.__TAURI_INTERNALS__}
         >
-          Analyze Temperature Variability
+          Analyze Stability
         </DropdownItem>
 
         <DropdownItem
           icon={<ModelIcon />}
-          description="Run several times to evaluate model stability across varying models"
+          description="Evaluate model stability across varying models"
           onClick={runOnce}
           isDisabled
         >

diff --git a/pdl-live-react/src/view/masonry/Stability.css b/pdl-live-react/src/view/masonry/Stability.css
@@ -8,27 +8,35 @@
   width: 1rem;
   height: 1rem;
   margin: 0;
-  border: 3px solid #70707050;
+
   border-radius: 1px / 2px;
 
   &:hover {
-    filter: brightness(1.2);
+    filter: brightness(0.875);
   }
 
+  /* https://colorbrewer2.org/#type=diverging&scheme=Spectral&n=5 */
   &[data-quartile="4"] {
-    background-color: #1a9641;
+    background-color: #2b83ba;
+    &:hover {
+      filter: brightness(1.3);
+    }
   }
   &[data-quartile="3"] {
-    background-color: #f9e0a2;
+    background-color: #abdda4;
+    height: 0.75rem;
   }
   &[data-quartile="2"] {
-    background-color: #f0ab00;
+    background-color: #ffffbf;
+    height: 0.5rem;
   }
   &[data-quartile="1"] {
-    background-color: #f4b678;
+    background-color: #fdae61;
+    height: 0.25rem;
   }
   &[data-quartile="0"] {
-    background-color: #c9190b;
+    background-color: #d7191c;
+    height: 3px;
   }
 }
 

diff --git a/pdl-live-react/src/view/masonry/Tile.ts b/pdl-live-react/src/view/masonry/Tile.ts
@@ -13,10 +13,7 @@ type Tile = {
 
   footer1Key?: string
   footer1Value?: string | number | boolean
-  footer2Key?: string
-  footer2Value?: string | number | boolean | number[]
-  footer2DetailHeader?: string
-  footer2DetailBody?: string[]
+  stability?: import("./stability").StabilityMetric[]
 
   block?: import("../../helpers").NonScalarPdlBlock
   actions?: "run"[]

diff --git a/pdl-live-react/src/view/masonry/model.ts b/pdl-live-react/src/view/masonry/model.ts
@@ -14,7 +14,7 @@ import {
 } from "../../helpers"
 
 import type Tile from "./Tile"
-import { hasSimilarityMetrics } from "./similarity"
+import { hasStabilityMetrics, type StabilityMetric } from "./stability"
 
 /** The final result of the block */
 /* function result(block: import("../../pdl_ast").PdlBlock) {
@@ -65,6 +65,9 @@ export default function computeModel(block: import("../../pdl_ast").PdlBlock) {
                     : undefined,
             }
 
+        const stability = hasStabilityMetrics(block)
+          ? block.pdl__stability
+          : ([] satisfies StabilityMetric[])
         return withDefs(block, [
           {
             id,
@@ -80,16 +83,13 @@ export default function computeModel(block: import("../../pdl_ast").PdlBlock) {
               ? capitalizeAndUnSnakeCase(String(meta[0][0]))
               : undefined,
             footer1Value: meta?.[0]?.[1] ? String(meta[0][1]) : undefined,
-            footer2Key: hasSimilarityMetrics(block) ? "Idempotency" : undefined,
-            footer2Value: hasSimilarityMetrics(block)
-              ? block.pdl__similarity
-              : undefined,
-            footer2DetailHeader: hasSimilarityMetrics(block)
-              ? "Stability across calls with the same input (Idempotency)"
-              : undefined,
-            footer2DetailBody: hasSimilarityMetrics(block)
-              ? block.pdl__results
-              : undefined,
+
+            stability,
+            //footer2Key: stability.map((m) => `T=${m.temperature} Stability`),
+            //footer2Value: stability.map((m) => m.matrix),
+            //footer2DetailHeader: stability.map((m) => `Stability across calls with the same input (temperature=${m.temperature})`),
+            //footer2DetailBody: stability.map((m) => m.results),
+
             block,
             actions: isLLMBlock(block) ? ["run"] : [],
             content: resultForDisplay,

diff --git a/pdl-live-react/src/view/masonry/similarity.ts b/pdl-live-react/src/view/masonry/similarity.ts