feat: puppeteer integration

apps/site/docs/visualization/index.mdx

@@ -3,4 +3,4 @@ pageType: custom
 ---
 import Visualizer from '@midscene/visualizer';
 
-<Visualizer showLogo={false} />
+<Visualizer hideLogo />

packages/midscene/src/utils.ts

@@ -87,8 +87,8 @@ export function getTmpDir() {
   return path;
 }
 
-export function getTmpFile(fileExt: string) {
-  const filename = `${randomUUID()}.${fileExt}`;
+export function getTmpFile(fileExtWithoutDot: string) {
+  const filename = `${randomUUID()}.${fileExtWithoutDot}`;
   return join(getTmpDir(), filename);
 }
 

packages/midscene/tests/utils.ts

@@ -23,35 +23,6 @@ export function sleep(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
-// export async function launch(url: string, opt?: {
-//   viewport?: Viewport,
-// }) {
-//   const browser = await puppeteer.launch();
-
-//   const page = (await browser.pages())[0];
-//   const viewportConfig = {
-//     width: opt?.viewport?.pixelWidth || 1920,
-//     height: opt?.viewport?.pixelHeight || 1080,
-//     deviceScaleFactor: opt?.viewport?.dpr || 1,
-//   }
-//   await page.setViewport(viewportConfig);
-//   await Promise.all([
-//     page.waitForNavigation({
-//       timeout: 20 * 1000,
-//       waitUntil: 'networkidle0',
-//     }),
-//     (async () => {
-//       const response = await page.goto(url);
-//       if (response?.status) {
-//         assert(response.status() <= 399, `Page load failed: ${response.status()}`);
-//       }
-//     })(),
-//   ]);
-//   await sleep(2 * 1000);
-
-//   return browser;
-// }
-
 export function fakeInsight(content: string) {
   const screenshot = getFixture('baidu.png');
   const basicContext = {

packages/visualizer/docs/index.tsx

@@ -2,5 +2,6 @@ import React from 'react';
 import Tool from '@/index';
 
 export default () => {
+  // return <Tool hideLogo />;
   return <Tool />;
 };

packages/visualizer/src/component/common.less

@@ -3,7 +3,7 @@
 
 @main-orange: #F9483E;
 
-@side-bg: #ECECEC;
+@side-bg: #f7f7f7;
 @title-bg: #DDDDDD;
 @border-color: #CCCCCC;
 @heavy-border-color: #888;

packages/visualizer/src/component/detail-panel.tsx

@@ -18,15 +18,15 @@ const ScreenshotItem = (props: { time: string; img: string }) => {
   );
 };
 
+const VIEW_TYPE_BLACKBOARD = 'blackboard';
 const VIEW_TYPE_SCREENSHOT = 'screenshot';
 const VIEW_TYPE_JSON = 'json';
-const VIEW_TYPE_BLACKBOARD = 'blackboard';
 
 const DetailPanel = (): JSX.Element => {
   const dumpId = useInsightDump((store) => store._loadId);
   const blackboardViewAvailable = Boolean(dumpId);
   const activeTask = useExecutionDump((store) => store.activeTask);
-  const [preferredViewType, setViewType] = useState(dumpId ? VIEW_TYPE_BLACKBOARD : VIEW_TYPE_SCREENSHOT);
+  const [preferredViewType, setViewType] = useState(VIEW_TYPE_BLACKBOARD);
 
   const viewType =
     preferredViewType === VIEW_TYPE_BLACKBOARD && !dumpId ? VIEW_TYPE_SCREENSHOT : preferredViewType;

packages/visualizer/src/component/detail-side.less

@@ -119,10 +119,8 @@
     }
   }
 
-  .context {
-    pre {
-      text-wrap: balance;
-    }
+  pre {
+    text-wrap: balance;
   }
 
   .item-list-space-up {

packages/visualizer/src/component/detail-side.tsx

@@ -313,7 +313,12 @@ const DetailSide = (): JSX.Element => {
   ) : null;
 
   const dataCard = dump?.data ? (
-    <Card liteMode={true} onMouseEnter={noop} onMouseLeave={noop} content={<pre>{kv(dump.data)}</pre>}></Card>
+    <Card
+      liteMode={true}
+      onMouseEnter={noop}
+      onMouseLeave={noop}
+      content={<pre>{JSON.stringify(dump.data, undefined, 2)}</pre>}
+    ></Card>
   ) : null;
 
   const plans = (task as ExecutionTaskPlanning)?.output?.plans;

packages/visualizer/src/component/sidebar.tsx

@@ -72,7 +72,7 @@ const SideItem = (props: {
   );
 };
 
-const Sidebar = (): JSX.Element => {
+const Sidebar = (props: { hideLogo?: boolean }): JSX.Element => {
   const groupedDumps = useExecutionDump((store) => store.dump);
   const setActiveTask = useExecutionDump((store) => store.setActiveTask);
   const activeTask = useExecutionDump((store) => store.activeTask);
@@ -177,7 +177,7 @@ const Sidebar = (): JSX.Element => {
   return (
     <div className="side-bar">
       <div className="top-controls">
-        <div className="brand" onClick={reset}>
+        <div className="brand" onClick={reset} style={{ display: props?.hideLogo ? 'none' : 'block' }}>
           <img
             src={logo}
             alt="Logo"

packages/visualizer/src/component/timeline.tsx

@@ -67,7 +67,7 @@ const TimelineWidget = (props: {
   const sizeRatio = 2;
 
   const titleBg = 0xdddddd; // @title-bg
-  const sideBg = 0xececec;
+  const sideBg = 0xf7f7f7; // @side-bg
   const gridTextColor = 0;
   const shotBorderColor = 0x777777;
   const gridLineColor = 0xcccccc; // @border-color

packages/visualizer/src/index.less

@@ -88,7 +88,7 @@ footer.mt-8{
   .main-canvas-container {
     flex-grow: 1;
     height: 100%;
-    background: #F5F5F5;
+    background: #ffffff;
     overflow-x: hidden;
     overflow-y: scroll;
     border-left: 1px solid @border-color;

packages/visualizer/src/index.tsx

@@ -13,7 +13,7 @@ import DetailSide from '@/component/detail-side';
 import Sidebar from '@/component/sidebar';
 
 const { Dragger } = Upload;
-const Index = (): JSX.Element => {
+const Index = (props: { hideLogo?: boolean }): JSX.Element => {
   const executionDump = useExecutionDump((store) => store.dump);
   const setGroupedDump = useExecutionDump((store) => store.setGroupedDump);
   const reset = useExecutionDump((store) => store.reset);
@@ -149,7 +149,7 @@ const Index = (): JSX.Element => {
         }}
       >
         <Panel maxSize={95}>
-          <Sidebar />
+          <Sidebar hideLogo={props?.hideLogo} />
         </Panel>
         <PanelResizeHandle
           onDragging={(isChanging) => {

packages/web-integration/modern.inspect.config.ts

@@ -8,7 +8,7 @@ export default defineConfig({
     buildType: 'bundle',
     format: 'iife',
     input: {
-      htmlElement:'src/html-element/index.ts',
+      htmlElement:'src/extractor/index.ts',
     },
     outDir: 'dist/script',
     esbuildOptions: options => {

packages/web-integration/src/common/agent.ts

@@ -0,0 +1,80 @@
+import { ExecutionDump, GroupedActionDump } from '@midscene/core';
+import { groupedActionDumpFileExt, writeDumpFile } from '@midscene/core/utils';
+import { PageTaskExecutor } from '../common/tasks';
+import { WebPage } from '@/common/page';
+
+export class PageAgent {
+  page: WebPage;
+
+  dumps: GroupedActionDump[];
+
+  constructor(page: WebPage) {
+    this.page = page;
+    this.dumps = [];
+  }
+
+  appendDump(groupName: string, execution: ExecutionDump) {
+    let currentDump = this.dumps.find((dump) => dump.groupName === groupName);
+    if (!currentDump) {
+      currentDump = {
+        groupName,
+        executions: [],
+      };
+      this.dumps.push(currentDump);
+    }
+    currentDump.executions.push(execution);
+  }
+
+  writeOutActionDumps() {
+    writeDumpFile(`playwright-${process.pid}`, groupedActionDumpFileExt, JSON.stringify(this.dumps));
+  }
+
+  async aiAction(taskPrompt: string, dumpGroupName = 'MidScene / Web', dumpCaseName = 'AI Action') {
+    const actionAgent = new PageTaskExecutor(this.page, { taskName: dumpCaseName });
+    let error: Error | undefined;
+    try {
+      await actionAgent.action(taskPrompt);
+    } catch (e: any) {
+      error = e;
+    }
+    if (actionAgent.executionDump) {
+      this.appendDump(dumpGroupName, actionAgent.executionDump);
+      this.writeOutActionDumps();
+    }
+    if (error) {
+      // playwright cli won't print error cause, so we print it here
+      console.error(error);
+      throw new Error(error.message, { cause: error });
+    }
+  }
+
+  async aiQuery(demand: any, dumpGroupName = 'MidScene / Web', dumpCaseName = 'AI Query') {
+    const actionAgent = new PageTaskExecutor(this.page, { taskName: dumpCaseName });
+    let error: Error | undefined;
+    let result: any;
+    try {
+      result = await actionAgent.query(demand);
+    } catch (e: any) {
+      error = e;
+    }
+    if (actionAgent.executionDump) {
+      this.appendDump(dumpGroupName, actionAgent.executionDump);
+      this.writeOutActionDumps();
+    }
+    if (error) {
+      // playwright cli won't print error cause, so we print it here
+      console.error(error);
+      throw new Error(error.message, { cause: error });
+    }
+    return result;
+  }
+
+  async ai(taskPrompt: string, type = 'action', dumpGroupName = 'MidScene / Web', dumpCaseName = 'AI') {
+    if (type === 'action') {
+      return this.aiAction(taskPrompt, dumpGroupName, dumpCaseName);
+    } else if (type === 'query') {
+      return this.aiQuery(taskPrompt, dumpGroupName, dumpCaseName);
+    }
+    throw new Error(`Unknown or Unsupported task type: ${type}, only support 'action' or 'query'`);
+  }
+}

packages/web-integration/src/common/page.d.ts

@@ -0,0 +1,5 @@
+import type { Page as PlaywrightPage } from 'playwright';
+import type { Page as PuppeteerPage, KeyInput } from 'puppeteer';
+
+export type WebPage = PlaywrightPage | PuppeteerPage;
+export type WebKeyInput = KeyInput;