feat(playwright): optimize the visualization and extraction of playwright ai process data

.github/workflows/ai.yml

@@ -1,4 +1,4 @@
-name: CI
+name: AI test
 
 on:
   push:
@@ -38,9 +38,13 @@ jobs:
 
     - name: Install dependencies
       run: pnpm install --frozen-lockfile
-
+    
     - name: Build project
       run: pnpm run build:pkg
 
+    - name: Run e2e tests
+      run: pnpm run e2e
+
     - name: Run tests
-      run: pnpm run test
+      run: pnpm run test:ai
+

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: AI test
+name: CI
 
 on:
   push:

packages/midscene/src/ai-model/automation/index.ts

@@ -72,11 +72,11 @@ export async function plan(
     throw new Error(planFromAI.error);
   }
 
-  actions.forEach((task) => {
-    if (task.type === 'Error') {
-      throw new Error(task.thought);
-    }
-  });
+  // actions.forEach((task) => {
+  //   if (task.type === 'Error') {
+  //     throw new Error(task.thought);
+  //   }
+  // });
 
   return { plans: actions };
 }

packages/web-integration/install-deps.bash

@@ -0,0 +1,11 @@
+if [ "$SKIP_DEVTOOLS_POSTINSTALL" = "true" ]; then
+    echo "Skipping devtools postinstall script."
+    exit 0
+fi
+
+if [ "$GITHUB_ACTIONS" = "true" ]; then
+    echo "Running in GitHub Actions environment."
+    npx playwright install-deps && npx playwright install
+else
+    echo "Not running in GitHub Actions environment."
+fi

packages/web-integration/package.json

@@ -51,7 +51,8 @@
     "e2e": "playwright test --config=playwright.config.ts",
     "e2e:cache": "MIDSCENE_CACHE=true playwright test --config=playwright.config.ts",
     "e2e:ui": "playwright test --config=playwright.config.ts --ui",
-    "e2e:ui-cache": "MIDSCENE_CACHE=true playwright test --config=playwright.config.ts  --ui"
+    "e2e:ui:cache": "MIDSCENE_CACHE=true playwright test --config=playwright.config.ts  --ui",
+    "postinstall": "bash install-deps.bash"
   },
   "files": ["dist", "README.md"],
   "dependencies": {

packages/web-integration/src/common/utils.ts

@@ -71,13 +71,12 @@ async function alignElements(
   });
   for (const item of validElements) {
     const { rect, id, content, attributes, locator } = item;
-
-    const aligned = await alignCoordByTrim(screenshotBuffer, rect);
-    item.rect = aligned;
-    item.center = [
-      Math.round(aligned.left + aligned.width / 2),
-      Math.round(aligned.top + aligned.height / 2),
-    ];
+    // const aligned = await alignCoordByTrim(screenshotBuffer, rect);
+    // item.rect = aligned;
+    // item.center = [
+    //   Math.round(aligned.left + aligned.width / 2),
+    //   Math.round(aligned.top + aligned.height / 2),
+    // ];
     textsAligned.push(
       new WebElementInfo({
         rect,

packages/web-integration/tests/ai/e2e/tool.ts → packages/web-integration/src/debug/index.ts

@@ -5,10 +5,9 @@ import { processImageElementInfo } from '@/img/img';
 import { getElementInfos } from '@/img/util';
 import { resizeImg, saveBase64Image } from '@midscene/core/image';
 
-export async function generateTestData(
+export async function generateExtractData(
   page: WebPage,
   targetDir: string,
-  inputImgBase64: string,
   saveImgType?: {
     disableInputImage: boolean;
     disableOutputImage: boolean;
@@ -17,6 +16,11 @@ export async function generateTestData(
     disableSnapshot: boolean;
   },
 ) {
+  const buffer = await page.screenshot({
+    encoding: 'base64',
+  });
+  const inputImgBase64 = buffer.toString('base64');
+
   const {
     elementsPositionInfo,
     captureElementSnapshot,

packages/web-integration/src/extractor/extractor.ts

@@ -84,7 +84,7 @@ export function extractTextWithPosition(
     logger('collectElementInfo', node, node.nodeName, rect);
     if (!rect) {
       logger('Element is not visible', node);
-      return;
+      return true;
     }
 
     if (isInputElement(node)) {

packages/web-integration/src/extractor/util.ts

@@ -92,12 +92,12 @@ export function visibleRect(
   el: HTMLElement | Node | null,
 ): { left: number; top: number; width: number; height: number } | false {
   if (!el) {
-    logger('Element is not in the DOM hierarchy');
+    logger(el, 'Element is not in the DOM hierarchy');
     return false;
   }
 
   if (!(el instanceof HTMLElement) && el.nodeType !== Node.TEXT_NODE) {
-    logger('Element is not in the DOM hierarchy');
+    logger(el, 'Element is not in the DOM hierarchy');
     return false;
   }
 
@@ -108,15 +108,15 @@ export function visibleRect(
       style.visibility === 'hidden' ||
       (style.opacity === '0' && el.tagName !== 'INPUT')
     ) {
-      logger('Element is hidden');
+      logger(el, 'Element is hidden');
       return false;
     }
   }
 
   const rect = getRect(el);
 
   if (rect.width === 0 && rect.height === 0) {
-    logger('Element has no size');
+    logger(el, 'Element has no size');
     return false;
   }
 
@@ -134,8 +134,13 @@ export function visibleRect(
     rect.top < viewportHeight;
 
   if (!isPartiallyInViewport) {
-    logger('Element is completely outside the viewport');
-    logger(rect, viewportHeight, viewportWidth, scrollTop, scrollLeft);
+    logger(el, 'Element is completely outside the viewport', {
+      rect,
+      viewportHeight,
+      viewportWidth,
+      scrollTop,
+      scrollLeft,
+    });
     return false;
   }
 

packages/web-integration/src/index.ts

@@ -2,3 +2,5 @@ export { PlaywrightAiFixture } from './playwright';
 export type { PlayWrightAiFixtureType } from './playwright';
 
 export { PuppeteerAgent } from './puppeteer';
+
+export { generateExtractData } from './debug';

packages/web-integration/src/playwright/index.ts

@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { PageAgent } from '@/common/agent';
 import type { WebPage } from '@/common/page';
 import type { AgentWaitForOpt } from '@midscene/core/.';
-import type { TestInfo, TestType } from '@playwright/test';
+import { type TestInfo, type TestType, test } from '@playwright/test';
 import type { Page as PlaywrightPage } from 'playwright';
 import type { PageTaskExecutor } from '../common/tasks';
 import { readTestCache, writeTestCache } from './cache';
@@ -80,10 +80,14 @@ export const PlaywrightAiFixture = () => {
       const agent = agentForPage(page, testInfo);
       await use(
         async (taskPrompt: string, opts?: { type?: 'action' | 'query' }) => {
-          await page.waitForLoadState('networkidle');
-          const actionType = opts?.type || 'action';
-          const result = await agent.ai(taskPrompt, actionType);
-          return result;
+          return new Promise((resolve, reject) => {
+            test.step(`ai - ${taskPrompt}`, async () => {
+              await page.waitForLoadState('networkidle');
+              const actionType = opts?.type || 'action';
+              const result = await agent.ai(taskPrompt, actionType);
+              resolve(result);
+            });
+          });
         },
       );
       const taskCacheJson = agent.taskExecutor.taskCache.generateTaskCache();
@@ -98,8 +102,10 @@ export const PlaywrightAiFixture = () => {
       const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
       const agent = agentForPage(page, testInfo);
       await use(async (taskPrompt: string) => {
-        await page.waitForLoadState('networkidle');
-        await agent.aiAction(taskPrompt);
+        test.step(`aiAction - ${taskPrompt}`, async () => {
+          await page.waitForLoadState('networkidle');
+          await agent.aiAction(taskPrompt);
+        });
       });
       // Why there's no cache here ?
       updateDumpAnnotation(testInfo, agent.dumpDataString());
@@ -111,9 +117,13 @@ export const PlaywrightAiFixture = () => {
     ) => {
       const agent = agentForPage(page, testInfo);
       await use(async (demand: any) => {
-        await page.waitForLoadState('networkidle');
-        const result = await agent.aiQuery(demand);
-        return result;
+        return new Promise((resolve, reject) => {
+          test.step(`aiQuery - ${JSON.stringify(demand)}`, async () => {
+            await page.waitForLoadState('networkidle');
+            const result = await agent.aiQuery(demand);
+            resolve(result);
+          });
+        });
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
     },
@@ -124,8 +134,13 @@ export const PlaywrightAiFixture = () => {
     ) => {
       const agent = agentForPage(page, testInfo);
       await use(async (assertion: string, errorMsg?: string) => {
-        await page.waitForLoadState('networkidle');
-        await agent.aiAssert(assertion, errorMsg);
+        return new Promise((resolve, reject) => {
+          test.step(`aiAssert - ${assertion}`, async () => {
+            await page.waitForLoadState('networkidle');
+            await agent.aiAssert(assertion, errorMsg);
+            resolve(null);
+          });
+        });
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
     },
@@ -136,7 +151,12 @@ export const PlaywrightAiFixture = () => {
     ) => {
       const agent = agentForPage(page, testInfo);
       await use(async (assertion: string, opt?: AgentWaitForOpt) => {
-        await agent.aiWaitFor(assertion, opt);
+        return new Promise((resolve, reject) => {
+          test.step(`aiWaitFor - ${assertion}`, async () => {
+            await agent.aiWaitFor(assertion, opt);
+            resolve(null);
+          });
+        });
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
     },

packages/web-integration/tests/ai/e2e/ai-auto-todo.spec.ts

@@ -1,10 +1,38 @@
+import path from 'node:path';
+import { generateExtractData } from '@/debug';
 import { expect } from 'playwright/test';
 import { test } from './fixture';
+import { getLastModifiedHTMLFile } from './util';
 
 test.beforeEach(async ({ page }) => {
   await page.goto('https://todomvc.com/examples/react/dist/');
 });
 
+// test.afterEach(async ({ page, ai, aiAssert }, testInfo) => {
+//   if (testInfo.title.indexOf('ai todo') !== -1) {
+//     await new Promise((resolve) => setTimeout(resolve, 3000));
+//     const htmlFile = getLastModifiedHTMLFile(
+//       path.join(__dirname, '../../../midscene_run/report'),
+//     );
+//     console.log('report html path:', htmlFile);
+//     await page.goto(`file:${htmlFile}`);
+//     await ai(
+//       'Move your mouse over the top task list (next to the logo) and click Select ai todo from the drop-down list',
+//     );
+//     const actionsList = await ai(
+//       'Sidebar task list Array<{title: string, actions: Array<string>}>',
+//       {
+//         type: 'query',
+//       },
+//     );
+//     const parseList = JSON.stringify(actionsList, null, 4);
+//     expect(parseList).toMatchSnapshot();
+//     await aiAssert(
+//       'On the left taskbar, check whether the specific execution content of the right task is normal',
+//     );
+//   }
+// });
+
 test('ai todo', async ({ ai, aiQuery, aiWaitFor }) => {
   await ai(
     'Enter "Learn JS today" in the task box, then press Enter to create',

packages/web-integration/tests/ai/e2e/ai-auto-todo.spec.ts-snapshots/ai-todo-1-chromium-darwin.txt

@@ -0,0 +1,31 @@
+{
+    "Sidebar task list": [
+        {
+            "title": "Enter \"Learn JS today\" in the task box, then press Enter to create",
+            "actions": [
+                "Planning",
+                "Insight / Locate",
+                "Action / Input",
+                "Action / KeyboardPress"
+            ]
+        },
+        {
+            "title": "Enter \"Learn Rust tomorrow\" in the task box, then press Enter to create",
+            "actions": [
+                "Planning",
+                "Insight / Locate",
+                "Action / Input",
+                "Action / KeyboardPress"
+            ]
+        },
+        {
+            "title": "Enter \"Learning AI the day after tomorrow\" in the task box, then press Enter to create",
+            "actions": [
+                "Planning",
+                "Insight / Locate",
+                "Action / Input",
+                "Action / KeyboardPress"
+            ]
+        }
+    ]
+}

packages/web-integration/tests/ai/e2e/ai-online-order.spec.ts

@@ -1,13 +1,17 @@
+import path from 'node:path';
+import { generateExtractData } from '@/debug';
 import { expect } from 'playwright/test';
 import { test } from './fixture';
 
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
 test.beforeEach(async ({ page }) => {
   page.setViewportSize({ width: 400, height: 905 });
   await page.goto('https://heyteavivocity.meuu.online/home');
   await page.waitForLoadState('networkidle');
 });
 
-test('ai online order', async ({ ai, aiQuery }) => {
+test('ai online order', async ({ ai, page, aiQuery }) => {
   await ai('点击左上角语言切换按钮(英文、中文)，在弹出的下拉列表中点击中文');
   await ai('向下滚动一屏');
   await ai('直接点击多肉葡萄的规格按钮');