feat(web): input supports additions, deletions, and changes (#78)

* feat(web): input supports additions, deletions, and changes * chore: optimize unit test content
web-infra-dev · Aug 28, 2024 · 84ee713 · 84ee713
1 parent f8fdf60
commit 84ee713
Show file tree

Hide file tree

Showing 39 changed files with 1,034 additions and 399 deletions.
diff --git a/packages/midscene/src/ai-model/automation/planning.ts b/packages/midscene/src/ai-model/automation/planning.ts
@@ -13,7 +13,7 @@ export function systemPromptToTaskPlanning() {
   * type: 'Hover', hover the previous element found
     * param: null
   * type: 'Input', 'KeyboardPress', input something or press a key
-    * param: { value: string }, the value to input or the key to press. Use （Enter, Shift, Control, Alt, Meta, ShiftLeft, ControlOrMeta, ControlOrMeta） to represent the key.
+    * param: { value: string }, Based on the value of the existing input, give the required value of the final input (may add or subtract content from the original),Typing in the input box usually requires clicking input first. the value to input or the key to press. Use （Enter, Shift, Control, Alt, Meta, ShiftLeft, ControlOrMeta, ControlOrMeta） to represent the key.
   * type: 'Scroll'
     * param: { scrollType: 'ScrollUntilBottom', 'ScrollUntilTop', 'ScrollDown', 'ScrollUp' }
   * type: 'Error'

diff --git a/packages/midscene/tests/ai/automation/__snapshots__/planning-input.test.ts.snap b/packages/midscene/tests/ai/automation/__snapshots__/planning-input.test.ts.snap
@@ -0,0 +1,118 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`automation - planning input - openAI > input value 1`] = `
+[
+  {
+    "param": {
+      "prompt": "",
+    },
+    "thought": undefined,
+    "type": "Locate",
+  },
+  {
+    "param": null,
+    "thought": undefined,
+    "type": "Tap",
+  },
+  {
+    "param": {
+      "value": "learning english",
+    },
+    "thought": undefined,
+    "type": "Input",
+  },
+]
+`;
+
+exports[`automation - planning input - openAI > input value 2`] = `
+[
+  {
+    "param": {
+      "prompt": "",
+    },
+    "thought": undefined,
+    "type": "Locate",
+  },
+  {
+    "param": {
+      "value": "learning english",
+    },
+    "thought": undefined,
+    "type": "Input",
+  },
+  {
+    "param": {
+      "value": "Enter",
+    },
+    "thought": undefined,
+    "type": "KeyboardPress",
+  },
+]
+`;
+
+exports[`automation - planning input - openAI > input value Add, delete, correct and check 1`] = `
+[
+  {
+    "param": {
+      "prompt": "",
+    },
+    "thought": undefined,
+    "type": "Locate",
+  },
+  {
+    "param": null,
+    "thought": undefined,
+    "type": "Tap",
+  },
+  {
+    "param": {
+      "value": "Learn English tomorrow",
+    },
+    "thought": undefined,
+    "type": "Input",
+  },
+]
+`;
+
+exports[`automation - planning input - openAI > input value Add, delete, correct and check 2`] = `
+[
+  {
+    "param": {
+      "prompt": "",
+    },
+    "thought": undefined,
+    "type": "Locate",
+  },
+  {
+    "param": {
+      "value": "Learn skiing",
+    },
+    "thought": undefined,
+    "type": "Input",
+  },
+]
+`;
+
+exports[`automation - planning input - openAI > input value Add, delete, correct and check 3`] = `
+[
+  {
+    "param": {
+      "prompt": "",
+    },
+    "thought": undefined,
+    "type": "Locate",
+  },
+  {
+    "param": null,
+    "thought": undefined,
+    "type": "Tap",
+  },
+  {
+    "param": {
+      "value": "Learn ",
+    },
+    "thought": undefined,
+    "type": "Input",
+  },
+]
+`;
diff --git a/packages/midscene/tests/ai/automation/planning-input.test.ts b/packages/midscene/tests/ai/automation/planning-input.test.ts
@@ -0,0 +1,44 @@
+import { plan } from '@/ai-model';
+import type { PlanningAction } from '@/types';
+import { getPageDataOfTestName } from 'tests/ai/inspector/util';
+/* eslint-disable max-lines-per-function */
+import { describe, expect, it, vi } from 'vitest';
+import { makePlanResultStable, modelList } from '../util';
+
+vi.setConfig({
+  testTimeout: 180 * 1000,
+  hookTimeout: 30 * 1000,
+});
+
+modelList.forEach((model) => {
+  describe(`automation - planning input - ${model}`, () => {
+    it('input value', async () => {
+      const { context } = await getPageDataOfTestName('todo');
+      const instructions = [
+        'In the taskbar, type learning english',
+        'In the taskbar, type learning english and hit Enter',
+      ];
+
+      for (const instruction of instructions) {
+        const { plans } = await plan(instruction, { context }, model);
+        const res = makePlanResultStable(plans);
+        expect(res).toMatchSnapshot();
+      }
+    });
+
+    it('input value Add, delete, correct and check', async () => {
+      const { context } = await getPageDataOfTestName('todo-input-with-value');
+      const instructions = [
+        'Append "tomorrow" to the existing content in the task input box',
+        'Replace "English" with "skiing" in the existing content of the task input box',
+        'Delete "English" from the existing content in the task input box',
+      ];
+
+      for (const instruction of instructions) {
+        const { plans } = await plan(instruction, { context }, model);
+        const res = makePlanResultStable(plans);
+        expect(res).toMatchSnapshot();
+      }
+    });
+  });
+});
diff --git a/packages/midscene/tests/ai/automation/planning.test.ts b/packages/midscene/tests/ai/automation/planning.test.ts
@@ -1,20 +1,14 @@
 import { plan } from '@/ai-model';
-import { useCozeModel } from '@/ai-model/coze';
 import { getPageDataOfTestName } from 'tests/ai/inspector/util';
 /* eslint-disable max-lines-per-function */
 import { describe, expect, it, vi } from 'vitest';
+import { modelList } from '../util';
 
 vi.setConfig({
   testTimeout: 180 * 1000,
   hookTimeout: 30 * 1000,
 });
 
-const modelList: Array<'openAI' | 'coze'> = ['openAI'];
-
-if (useCozeModel('coze')) {
-  modelList.push('coze');
-}
-
 modelList.forEach((model) => {
   describe(`automation - planning ${model}`, () => {
     it('basic run', async () => {

diff --git a/packages/midscene/tests/ai/inspector/__snapshots__/online_order_inspector.test.ts.snap b/packages/midscene/tests/ai/inspector/__snapshots__/online_order_inspector.test.ts.snap
@@ -33,11 +33,11 @@
     "elements": [
       {
         "id": "14103376fb",
-        "indexId": "22",
+        "indexId": "25",
       },
       {
         "id": "0250e12e67",
-        "indexId": "28",
+        "indexId": "31",
       },
     ],
     "error": [],
@@ -47,11 +47,11 @@
     "elements": [
       {
         "id": "fadfa10182",
-        "indexId": "23",
+        "indexId": "26",
       },
       {
         "id": "001617dbf8",
-        "indexId": "29",
+        "indexId": "32",
       },
     ],
     "error": [],
@@ -61,7 +61,7 @@
     "elements": [
       {
         "id": "cad3004a2d",
-        "indexId": "30",
+        "indexId": "33",
       },
     ],
     "error": [],

diff --git a/packages/midscene/tests/ai/inspector/__snapshots__/todo_inspector.test.ts.snap b/packages/midscene/tests/ai/inspector/__snapshots__/todo_inspector.test.ts.snap
@@ -12,8 +12,8 @@
   {
     "elements": [
       {
-        "id": "e477ad490a",
-        "indexId": "8",
+        "id": "7b0f4c3c94",
+        "indexId": "11",
       },
     ],
     "error": [],
@@ -23,7 +23,7 @@
     "elements": [
       {
         "id": "7ccd467339",
-        "indexId": "9",
+        "indexId": "13",
       },
     ],
     "error": [],
@@ -33,7 +33,7 @@
     "elements": [
       {
         "id": "eb987bf616",
-        "indexId": "10",
+        "indexId": "14",
       },
     ],
     "error": [],
@@ -43,7 +43,7 @@
     "elements": [
       {
         "id": "284ba7f2b3",
-        "indexId": "15",
+        "indexId": "20",
       },
     ],
     "error": [],