feat(web): playwright integration (#5)

* feat: use a resizable layout

* feat: update visualizer

* fix: resize bug

* feat: add .query for playwright integration

* fix: ts error

packages/midscene/src/action/executor.ts

@@ -7,9 +7,7 @@ import {
   ExecutionTaskReturn,
   ExecutorContext,
 } from '@/types';
-import { actionDumpFileExt, getPkgInfo, writeDumpFile } from '@/utils';
-
-const logFileExt = actionDumpFileExt;
+import { getPkgInfo } from '@/utils';
 
 export class Executor {
   name: string;
@@ -95,7 +93,10 @@ export class Executor {
           element: previousFindOutput?.element,
         };
         if (task.type === 'Insight') {
-          assert(task.subType === 'find', `unsupported insight subType: ${task.subType}`);
+          assert(
+            task.subType === 'find' || task.subType === 'query',
+            `unsupported insight subType: ${task.subType}`,
+          );
           returnValue = await task.executor(param, executorContext);
           previousFindOutput = (returnValue as ExecutionTaskReturn<ExecutionTaskInsightFindOutput>)?.output;
         } else if (task.type === 'Action' || task.type === 'Planning') {
@@ -136,17 +137,14 @@ export class Executor {
     }
   }
 
-  dump() {
+  dump(): ExecutionDump {
     const dumpData: ExecutionDump = {
       sdkVersion: getPkgInfo().version,
       logTime: Date.now(),
       name: this.name,
       description: this.description,
       tasks: this.tasks,
     };
-    const fileContent = JSON.stringify(dumpData, null, 2);
-    this.dumpFileName = this.dumpFileName || `pid-${process.pid}-${Date.now()}`;
-    const dumpPath = writeDumpFile(this.dumpFileName, logFileExt, fileContent);
-    return dumpPath;
+    return dumpData;
   }
 }

packages/midscene/src/automation/planning.ts

@@ -33,7 +33,9 @@ export function systemPromptToTaskPlanning(query: string) {
   * Input: 'Weather in Shanghai'
   * KeyboardPress: 'Enter'
   
-  Remember: The actions you composed MUST be based on the page context information you get. Instead of making up actions that are not related to the page context.
+  Remember: 
+  1. The actions you composed MUST be based on the page context information you get. Instead of making up actions that are not related to the page context.
+  2. In most cases, you should Find one element first, then do other actions on it. For example, alway Find one element, then hover on it. But if you think it's necessary to do other actions first (like global scroll, global key press), you can do that.
 
   If any error occurs during the task planning (like the page content and task are irrelevant, or the element mentioned does not exist at all), please return the error message with explanation in the errors field. The thoughts、prompts、error messages should all in the same language as the user query.
   

packages/midscene/src/types.ts

@@ -70,22 +70,12 @@ export interface AISectionParseResponse<DataShape> {
  * context
  */
 
-// export type ContextDescriberFn = () => Promise<{
-//   description: string;
-//   elementById: (id: string) => BaseElement;
-// }>;
-
 export abstract class UIContext<ElementType extends BaseElement = BaseElement> {
   abstract screenshotBase64: string;
 
   abstract content: ElementType[];
 
   abstract size: Size;
-
-  // abstract describer: () => Promise<{
-  //   description: string;
-  //   elementById: (id: string) => ElementType;
-  // }>;
 }
 
 /**
@@ -215,7 +205,7 @@ export interface ExecutionRecorderItem {
   timing?: string;
 }
 
-export type ExecutionTaskType = 'Planning' | 'Insight' | 'Action';
+export type ExecutionTaskType = 'Planning' | 'Insight' | 'Action' | 'Assertion';
 
 export interface ExecutorContext {
   task: ExecutionTask;
@@ -291,22 +281,19 @@ export type ExecutionTaskInsightFind = ExecutionTask<ExecutionTaskInsightFindApp
 /*
 task - insight-extract
 */
-// export interface ExecutionTaskInsightExtractParam {
-//   dataDemand: InsightExtractParam;
-// }
+export interface ExecutionTaskInsightQueryParam {
+  dataDemand: InsightExtractParam;
+}
 
-// export interface ExecutionTaskInsightExtractOutput {
-//   data: any;
-// }
+export interface ExecutionTaskInsightQueryOutput {
+  data: any;
+}
 
-// export type ExecutionTaskInsightExtractApply = ExecutionTaskApply<
-//   'insight-extract', // TODO: remove task-extract ?
-//   ExecutionTaskInsightExtractParam
-// >;
+export type ExecutionTaskInsightQueryApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightQueryParam>;
 
-// export type ExecutionTaskInsightExtract = ExecutionTask<ExecutionTaskInsightExtractApply>;
+export type ExecutionTaskInsightQuery = ExecutionTask<ExecutionTaskInsightQueryApply>;
 
-export type ExecutionTaskInsight = ExecutionTaskInsightFind; // | ExecutionTaskInsightExtract;
+// export type ExecutionTaskInsight = ExecutionTaskInsightFind; // | ExecutionTaskInsightExtract;
 
 /*
 task - action (i.e. interact) 

packages/midscene/src/utils.ts

@@ -38,7 +38,6 @@ export function getPkgInfo(): PkgInfo {
 let logDir = join(process.cwd(), './midscene_run/');
 let logEnvReady = false;
 export const insightDumpFileExt = 'insight-dump.json';
-export const actionDumpFileExt = 'action-dump.json';
 export const groupedActionDumpFileExt = 'all-logs.json';
 
 export function getDumpDir() {

packages/midscene/tests/executor/index.test.ts

@@ -100,9 +100,8 @@ describe('executor', () => {
     expect(tapperFn.mock.calls[0][1].element).toBe(element);
     expect(tapperFn.mock.calls[0][1].task).toBeTruthy();
 
-    executor.dump();
-    const latestFile = join(getDumpDir(), 'latest.action-dump.json');
-    expect(existsSync(latestFile)).toBeTruthy();
+    const dump = executor.dump();
+    expect(dump.logTime).toBeTruthy();
 
   }, {
     timeout: 999 * 1000,
@@ -134,11 +133,8 @@ describe('executor', () => {
     expect(tapperFn).toBeCalledTimes(0);
 
 
-    const dumpPath = initExecutor.dump();
-
-    const dumpJsonContent1 = JSON.parse(readFileSync(dumpPath, 'utf-8'));
-    expect(dumpJsonContent1.tasks.length).toBe(2);
-
+    const dumpContent1 = initExecutor.dump();
+    expect(dumpContent1.tasks.length).toBe(2);
 
     // append while running
     await Promise.all([
@@ -161,9 +157,8 @@ describe('executor', () => {
     expect(initExecutor.status).toBe('pending');
 
     // same dumpPath to append
-    initExecutor.dump();
-    const dumpJsonContent2 = JSON.parse(readFileSync(dumpPath, 'utf-8'));
-    expect(dumpJsonContent2.tasks.length).toBe(4);
+    const dumpContent2 = initExecutor.dump();
+    expect(dumpContent2.tasks.length).toBe(4);
   });
 
   // it('insight - run with error', async () => {

packages/midscene/tests/image/__snapshots__/index.test.ts.snap

@@ -27,6 +27,13 @@ exports[`image utils > imageInfo 1`] = `
 }
 `;
 
+exports[`image utils > jpeg + base64 + imageInfo 1`] = `
+{
+  "height": 905,
+  "width": 400,
+}
+`;
+
 exports[`image utils > trim image 1`] = `
 {
   "height": 70,

packages/midscene/tests/image/index.test.ts

@@ -24,6 +24,13 @@ describe('image utils', () => {
     expect(info).toMatchSnapshot();
   });
 
+  it('jpeg + base64 + imageInfo', async () => {
+    const image = getFixture('heytea.jpeg');
+    const base64 = base64Encoded(image);
+    const info = await imageInfoOfBase64(base64);
+    expect(info).toMatchSnapshot();
+  });
+
   it('trim image', async () => {
     const file = getFixture('long-text.png');
     const info = await trimImage(file);

packages/visualizer/package.json

@@ -15,25 +15,25 @@
   },
   "dependencies": {
     "@ant-design/icons": "5.3.7",
+    "@midscene/core": "workspace:*",
     "@modern-js/runtime": "^2.54.2",
     "antd": "5.17.3",
     "dayjs": "1.11.11",
     "pixi.js": "8.1.1",
     "react": "~18.2.0",
     "react-dom": "~18.2.0",
-    "@midscene/core": "workspace:*",
+    "react-resizable-panels": "2.0.22",
     "zustand": "4.5.2"
   },
-  "peerDependencies": {},
   "devDependencies": {
     "@modern-js/module-tools": "^2.54.2",
     "@modern-js/plugin-module-doc": "^2.33.1",
-    "react": "~18.2.0",
-    "react-dom": "~18.2.0",
     "@types/react": "~18.2.22",
     "@types/react-dom": "~18.2.7",
-    "typescript": "~5.0.4",
-    "rimraf": "~3.0.2"
+    "react": "~18.2.0",
+    "react-dom": "~18.2.0",
+    "rimraf": "~3.0.2",
+    "typescript": "~5.0.4"
   },
   "sideEffects": [
     "**/*.css",

packages/visualizer/src/component/common.less

@@ -18,4 +18,4 @@
 @weak-bg: #F3F3F3;
 
 @side-horizontal-padding: 10px;
-@side-vertical-spacing: 8px;
+@side-vertical-spacing: 10px;

packages/visualizer/src/component/detail-side.less

@@ -15,7 +15,7 @@
   }
 
   .meta-kv {
-    padding: @side-vertical-spacing @side-horizontal-padding;
+    padding: @side-vertical-spacing @side-horizontal-padding calc(@side-vertical-spacing + 4px);
     .meta {
       box-sizing: border-box;
       padding: 2px 0;

packages/visualizer/src/component/detail-side.tsx

@@ -7,6 +7,7 @@ import {
   BaseElement,
   ExecutionTaskAction,
   ExecutionTaskInsightFind,
+  ExecutionTaskInsightQuery,
   ExecutionTaskPlanning,
   UISection,
 } from '@midscene/core';
@@ -152,6 +153,10 @@ const DetailSide = (): JSX.Element => {
       </span>
     );
 
+    if (Array.isArray(data) || typeof data !== 'object') {
+      return <pre className="description-content">{JSON.stringify(data, undefined, 2)}</pre>;
+    }
+
     return Object.keys(data).map((key) => {
       const value = data[key];
       let content;
@@ -201,14 +206,20 @@ const DetailSide = (): JSX.Element => {
     taskParam = MetaKV({
       data: [
         { key: 'type', content: (task && typeStr(task)) || '' },
-        { key: 'userPrompt', content: (task as ExecutionTaskPlanning)?.param?.userPrompt },
+        { key: 'param', content: (task as ExecutionTaskPlanning)?.param?.userPrompt },
       ],
     });
   } else if (task?.type === 'Insight') {
     taskParam = MetaKV({
       data: [
         { key: 'type', content: (task && typeStr(task)) || '' },
-        { key: 'query', content: (task as ExecutionTaskInsightFind)?.param?.query },
+        {
+          key: 'param',
+          content: JSON.stringify(
+            (task as ExecutionTaskInsightFind)?.param?.query ||
+              (task as ExecutionTaskInsightQuery)?.param?.dataDemand,
+          ),
+        },
       ],
     });
   } else if (task?.type === 'Action') {
@@ -302,20 +313,14 @@ const DetailSide = (): JSX.Element => {
   ) : null;
 
   const dataCard = dump?.data ? (
-    <Card
-      liteMode={true}
-      title="Data Extracted"
-      onMouseEnter={noop}
-      onMouseLeave={noop}
-      content={<pre>{kv(dump.data)}</pre>}
-    ></Card>
+    <Card liteMode={true} onMouseEnter={noop} onMouseLeave={noop} content={<pre>{kv(dump.data)}</pre>}></Card>
   ) : null;
 
   const plans = (task as ExecutionTaskPlanning)?.output?.plans;
   let timelineData: TimelineItemProps[] = [];
   if (plans) {
     timelineData = timelineData.concat(
-      plans.map((item, index) => {
+      plans.map((item) => {
         return {
           color: '#06B1AB',
           children: (
@@ -338,10 +343,10 @@ const DetailSide = (): JSX.Element => {
       <PanelTitle title="Task Meta" />
       {metaKVElement}
       {/* Param  */}
-      <PanelTitle title="Task Param" />
+      <PanelTitle title="Param" />
       {taskParam}
       {/* Response */}
-      <PanelTitle title="Task Output" />
+      <PanelTitle title="Output" />
       <div className="item-list item-list-space-up">
         {errorSection}
         {dataCard}

packages/visualizer/src/component/global-hover-preview.less

@@ -0,0 +1,23 @@
+@import './common.less';
+
+@max-size: 400px;
+.global-hover-preview {
+  position: fixed;
+  display: block;
+  max-width: @max-size;
+  max-height: @max-size;
+  overflow: hidden;
+  z-index: 10;
+  text-align: center;
+  border: 1px solid @border-color;
+  box-sizing: border-box;
+  background: @side-bg;
+  box-shadow: 1px 1px 5px 0 rgba(0, 0, 0, 0.2);
+
+  img {
+    max-width: @max-size;
+    max-height: @max-size;
+    width: auto;
+    height: auto;
+  }
+}

packages/visualizer/src/component/global-hover-preview.tsx

@@ -0,0 +1,50 @@
+import { useRef, useState } from 'react';
+import { useExecutionDump } from './store';
+import './global-hover-preview.less';
+
+const size = 400; // @max-size
+const GlobalHoverPreview = () => {
+  const wrapperRef = useRef<HTMLDivElement>(null);
+  const hoverTask = useExecutionDump((store) => store.hoverTask);
+  const hoverPreviewConfig = useExecutionDump((store) => store.hoverPreviewConfig);
+  const [imageW, setImageW] = useState(size);
+  const [imageH, setImageH] = useState(size);
+
+  const images = hoverTask?.recorder
+    ?.filter((item) => {
+      return item.screenshot;
+    })
+    .map((item) => item.screenshot);
+
+  const { x, y } = hoverPreviewConfig || {};
+  let left = 0;
+  let top = 0;
+
+  const shouldShow = images?.length && typeof x !== 'undefined' && typeof y !== 'undefined';
+  if (shouldShow) {
+    const { clientWidth, clientHeight } = document.body;
+    const widthInPractice = imageW >= imageH ? size : size * (imageW / imageH);
+    const heightInPractice = imageW >= imageH ? size * (imageH / imageW) : size;
+    left = x + widthInPractice > clientWidth ? clientWidth - widthInPractice : x;
+    top = y + heightInPractice > clientHeight ? clientHeight - heightInPractice : y;
+  }
+  // if x + size exceed the screen width, use (screenWidth - size) instead
+
+  return shouldShow ? (
+    <div className="global-hover-preview" style={{ left, top }} ref={wrapperRef}>
+      {images?.length ? (
+        <img
+          src={images[0]}
+          onLoad={(img) => {
+            const imgElement = img.target as HTMLImageElement;
+            const width = imgElement.naturalWidth;
+            const height = imgElement.naturalHeight;
+            setImageW(width);
+            setImageH(height);
+          }}
+        />
+      ) : null}
+    </div>
+  ) : null;
+};
+export default GlobalHoverPreview;