feat: add replay panel for visualization (#96)

* feat: add replay panel

* feat: add subtitle to the replay widget

* fix: fit the mobile page

* feat: only show player in Planning

* feat: reorder the panel

* chore: optimize test list switch

---------

Co-authored-by: zhouxiao.shaw <[email protected]>

packages/midscene/src/ai-model/common.ts

@@ -60,5 +60,7 @@ export async function callAiFn<T>(options: {
     return parseResult;
   }
 
-  throw Error('Does not contain coze and openai environment variables');
+  throw Error(
+    'Cannot find Coze or OpenAI config. You should set at least one of them.',
+  );
 }

packages/visualizer/scripts/build-html.ts

@@ -22,9 +22,11 @@ const htmlPath = join(__dirname, '../html/tpl.html');
 const cssPath = join(__dirname, '../dist/report/index.css');
 const jsPath = join(__dirname, '../dist/report/index.js');
 const demoPath = join(__dirname, './fixture/demo-dump.json');
+const demoMobilePath = join(__dirname, './fixture/demo-mobile-dump.json');
 const multiEntrySegment = join(__dirname, './fixture/multi-entries.html');
 const outputHTML = join(__dirname, '../dist/report/index.html');
 const outputDemoHTML = join(__dirname, '../dist/report/demo.html');
+const outputDemoMobileHTML = join(__dirname, '../dist/report/demo-mobile.html');
 const outputMultiEntriesHTML = join(__dirname, '../dist/report/multi.html');
 const outputEmptyDumpHTML = join(__dirname, '../dist/report/empty-error.html');
 
@@ -75,6 +77,15 @@ function build() {
   writeFileSync(outputDemoHTML, resultWithDemo);
   console.log(`HTML file generated successfully: ${outputDemoHTML}`);
 
+  const demoMobileData = readFileSync(demoMobilePath, 'utf-8');
+  const resultWithDemoMobile = tplReplacer(html, {
+    css: `<style>\n${css}\n</style>\n`,
+    js: `<script>\n${js}\n</script>`,
+    dump: `<script type="midscene_web_dump" type="application/json">${demoMobileData}</script>`,
+  });
+  writeFileSync(outputDemoMobileHTML, resultWithDemoMobile);
+  console.log(`HTML file generated successfully: ${outputDemoMobileHTML}`);
+
   const multiEntriesData = readFileSync(multiEntrySegment, 'utf-8');
   const resultWithMultiEntries = tplReplacer(html, {
     css: `<style>\n${css}\n</style>\n`,

packages/visualizer/src/component/blackboard.tsx

@@ -8,21 +8,53 @@ import { colorForName, highlightColorForType } from './color';
 import './blackboard.less';
 import { useBlackboardPreference, useInsightDump } from './store';
 
-const itemFillAlpha = 0.3;
+const itemFillAlpha = 0.4;
+const highlightAlpha = 0.7;
 const bgOnAlpha = 0.8;
 const bgOffAlpha = 0.3;
 const noop = () => {
   // noop
 };
 
+export const rectMarkForItem = (
+  rect: Rect,
+  name: string,
+  ifHighlight: boolean,
+  onPointOver?: () => void,
+  onPointerOut?: () => void,
+) => {
+  const { left, top, width, height } = rect;
+  const themeColor = ifHighlight
+    ? highlightColorForType('element')
+    : colorForName(name);
+  const alpha = ifHighlight ? highlightAlpha : itemFillAlpha;
+  const graphics = new PIXI.Graphics();
+  graphics.beginFill(themeColor, alpha);
+  graphics.lineStyle(1, themeColor, 1);
+  graphics.drawRect(left, top, width, height);
+  graphics.endFill();
+  if (onPointOver && onPointerOut) {
+    graphics.interactive = true;
+    graphics.on('pointerover', onPointOver);
+    graphics.on('pointerout', onPointerOut);
+  }
+
+  const nameFontSize = 18;
+  const texts = new PIXI.Text(name, {
+    fontSize: nameFontSize,
+    fill: 0x0,
+  });
+  texts.x = left;
+  texts.y = Math.max(top - (nameFontSize + 4), 0);
+  return [graphics, texts];
+};
+
 const BlackBoard = (): JSX.Element => {
   const dump = useInsightDump((store) => store.data);
   const setHighlightElements = useInsightDump(
     (store) => store.setHighlightElements,
   );
-  const highlightSectionNames = useInsightDump(
-    (store) => store.highlightSectionNames,
-  );
+
   const highlightElements = useInsightDump((store) => store.highlightElements);
   const highlightIds = highlightElements.map((e) => e.id);
 
@@ -107,34 +139,6 @@ const BlackBoard = (): JSX.Element => {
     };
   }, [app.stage, appInitialed]);
 
-  const rectMarkForItem = (
-    rect: Rect,
-    name: string,
-    themeColor: string,
-    alpha: number,
-    onPointOver: () => void,
-    onPointerOut: () => void,
-  ) => {
-    const { left, top, width, height } = rect;
-    const graphics = new PIXI.Graphics();
-    graphics.beginFill(themeColor, alpha);
-    graphics.lineStyle(1, themeColor, 1);
-    graphics.drawRect(left, top, width, height);
-    graphics.endFill();
-    graphics.interactive = true;
-    graphics.on('pointerover', onPointOver);
-    graphics.on('pointerout', onPointerOut);
-
-    const nameFontSize = 18;
-    const texts = new PIXI.Text(name, {
-      fontSize: nameFontSize,
-      fill: 0x0,
-    });
-    texts.x = left;
-    texts.y = Math.max(top - (nameFontSize + 4), 0);
-    return [graphics, texts];
-  };
-
   const { highlightElementRects } = useMemo(() => {
     const highlightElementRects: Rect[] = [];
 
@@ -149,10 +153,7 @@ const BlackBoard = (): JSX.Element => {
         const [graphics] = rectMarkForItem(
           rect,
           content,
-          ifHighlight
-            ? highlightColorForType('element')
-            : colorForName('element', content),
-          ifHighlight ? 1 : itemFillAlpha,
+          ifHighlight,
           noop,
           noop,
         );
@@ -162,10 +163,7 @@ const BlackBoard = (): JSX.Element => {
       const [graphics] = rectMarkForItem(
         rect,
         content,
-        ifHighlight
-          ? highlightColorForType('element')
-          : colorForName('element', content),
-        ifHighlight ? 1 : itemFillAlpha,
+        ifHighlight,
         () => {
           setHighlightElements([element]);
         },

packages/visualizer/src/component/color.tsx

@@ -1,12 +1,13 @@
 // https://coolors.co/palettes/popular/#01204e
 const sectionColor = ['#028391'];
-const elementColor = ['#fb6107'];
+// const elementColor = ['#fb6107'];
+const elementColor = ['#01204E'];
 const highlightColorForSection = '#01204E';
 const highlightColorForElement = '#F56824'; // @main-orange
 
 function djb2Hash(str?: string): number {
   if (!str) {
-    console.warn('djb2Hash: empty string');
+    // console.warn('djb2Hash: empty string');
     str = 'unnamed';
   }
   let hash = 5381;
@@ -16,20 +17,12 @@ function djb2Hash(str?: string): number {
   return hash >>> 0; // Convert to unsigned 32
 }
 
-export function colorForName(
-  type: 'section' | 'element',
-  name: string,
-): string {
+export function colorForName(name: string): string {
   const hashNumber = djb2Hash(name);
-  if (type === 'section') {
-    return sectionColor[hashNumber % sectionColor.length];
-  }
   return elementColor[hashNumber % elementColor.length];
 }
 
 export function highlightColorForType(type: 'section' | 'element'): string {
-  if (type === 'section') {
-    return highlightColorForSection;
-  }
+  // return highlightColorForSection;
   return highlightColorForElement;
 }

packages/visualizer/src/component/detail-panel.less

@@ -6,9 +6,15 @@
   display: flex;
   flex-direction: column;
   height: 100%;
+  box-sizing: border-box;
   padding: @layout-space;
   background: #FFF;
 
+  .scrollable {
+    height: 100%;
+    overflow: auto;
+  }
+
   .ant-segmented{
     padding: 0;
   }
@@ -24,6 +30,8 @@
     flex-direction: column;
     display: flex;
     flex-grow: 1;
+    height: 100%;
+    overflow: hidden;
   }
 
   .blackboard {
@@ -48,6 +56,8 @@
     img {
       border: 1px solid @heavy-border-color;
       max-width: 100%;
+      max-height: 720px;
+      box-sizing: border-box;
     }
   }