From 43bf165ddc675552adcba9c57d13ac71edd6a636 Mon Sep 17 00:00:00 2001 From: Leyang Date: Mon, 23 Sep 2024 10:57:19 +0800 Subject: [PATCH] define appium export in package.json & encapsulated screenshot picture path generation (#95) * feat: define appium export in package.json * feat: encapsulated screenshot picture path generation --- packages/midscene/src/utils.ts | 2 -- packages/web-integration/README.md | 2 +- packages/web-integration/package.json | 5 ++++ packages/web-integration/src/appium/page.ts | 17 ++++++------- packages/web-integration/src/common/tasks.ts | 10 +++----- packages/web-integration/src/common/utils.ts | 4 +-- packages/web-integration/src/debug/index.ts | 4 +-- packages/web-integration/src/index.ts | 2 +- packages/web-integration/src/page.ts | 11 +------- .../src/puppeteer/base-page.ts | 25 +++++-------------- .../tests/ai/native/appium/dongchedi.test.ts | 14 ++++++++--- .../tests/ai/web/playwright/tool.ts | 8 +++--- .../tests/unit-test/web-extractor.test.ts | 7 ++---- 13 files changed, 41 insertions(+), 70 deletions(-) diff --git a/packages/midscene/src/utils.ts b/packages/midscene/src/utils.ts index d1b037df..ddc5b773 100644 --- a/packages/midscene/src/utils.ts +++ b/packages/midscene/src/utils.ts @@ -145,8 +145,6 @@ export async function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } -export const commonScreenshotParam = { type: 'jpeg', quality: 75 } as any; - export function replacerForPageObject(key: string, value: any) { if (value && value.constructor?.name === 'Page') { return '[Page object]'; diff --git a/packages/web-integration/README.md b/packages/web-integration/README.md index 789d067d..47637d67 100644 --- a/packages/web-integration/README.md +++ b/packages/web-integration/README.md @@ -17,7 +17,7 @@ then execute the command to start appium server: appium --use-plugins=universal-xml ``` -now you can use run tests for the iOS/Android device: +now you can use run tests for iOS/Android devices: ```bash npm run test:ai -- appium diff --git a/packages/web-integration/package.json b/packages/web-integration/package.json index 3e8337f6..4544ab5c 100644 --- a/packages/web-integration/package.json +++ b/packages/web-integration/package.json @@ -27,6 +27,11 @@ "import": "./dist/es/playwright-report.js", "require": "./dist/lib/playwright-report.js" }, + "./appium": { + "types": "./dist/types/appium.d.ts", + "import": "./dist/es/appium.js", + "require": "./dist/lib/appium.js" + }, "./debug": { "types": "./dist/types/debug.d.ts", "import": "./dist/es/debug.js", diff --git a/packages/web-integration/src/appium/page.ts b/packages/web-integration/src/appium/page.ts index d74714da..21dcde32 100644 --- a/packages/web-integration/src/appium/page.ts +++ b/packages/web-integration/src/appium/page.ts @@ -1,10 +1,11 @@ import fs from 'node:fs'; +import { getTmpFile } from '@midscene/core/utils'; import { resizeImg } from '@midscene/shared/img'; import { DOMParser } from '@xmldom/xmldom'; import type { KeyInput as PuppeteerKeyInput } from 'puppeteer'; import type { Browser } from 'webdriverio'; import { type ElementInfo, clientExtractTextWithPosition } from '../extractor'; -import type { AbstractPage, MouseButton, screenshotOptions } from '../page'; +import type { AbstractPage, MouseButton } from '../page'; type WebKeyInput = PuppeteerKeyInput; @@ -37,21 +38,17 @@ export class Page implements AbstractPage { return infos; } - async screenshot(options: screenshotOptions): Promise { - if (!options.path) { - throw new Error('path is required for screenshot'); - } - + async screenshot(): Promise { const { width, height } = await this.browser.getWindowSize(); - const screenshotBuffer = await this.browser.saveScreenshot(options.path); + const path = getTmpFile('png'); + const screenshotBuffer = await this.browser.saveScreenshot(path); const resizedScreenshotBuffer = await resizeImg(screenshotBuffer, { width, height, }); + fs.writeFileSync(path, resizedScreenshotBuffer); - if (options.path) { - fs.writeFileSync(options.path, resizedScreenshotBuffer); - } + return path; } get mouse() { diff --git a/packages/web-integration/src/common/tasks.ts b/packages/web-integration/src/common/tasks.ts index 43a940c5..06e13dd3 100644 --- a/packages/web-integration/src/common/tasks.ts +++ b/packages/web-integration/src/common/tasks.ts @@ -24,7 +24,7 @@ import Insight, { type PlanningActionParamWaitFor, type PlanningActionParamError, } from '@midscene/core'; -import { commonScreenshotParam, getTmpFile, sleep } from '@midscene/core/utils'; +import { sleep } from '@midscene/core/utils'; import { base64Encoded } from '@midscene/shared/img'; import type { KeyInput } from 'puppeteer'; import type { ElementInfo } from '../extractor'; @@ -56,15 +56,11 @@ export class PageTaskExecutor { } private async recordScreenshot(timing: ExecutionRecorderItem['timing']) { - const file = getTmpFile('png'); - await this.page.screenshot({ - ...commonScreenshotParam, - path: file, - }); + const file = await this.page.screenshot(); const item: ExecutionRecorderItem = { type: 'screenshot', ts: Date.now(), - screenshot: base64Encoded(file), + screenshot: base64Encoded(file as string), timing, }; return item; diff --git a/packages/web-integration/src/common/utils.ts b/packages/web-integration/src/common/utils.ts index 8069db82..8e94e702 100644 --- a/packages/web-integration/src/common/utils.ts +++ b/packages/web-integration/src/common/utils.ts @@ -23,9 +23,7 @@ export async function parseContextFromWebPage( assert(page, 'page is required'); const url = page.url(); - const file = getTmpFile('jpeg'); - await page.screenshot({ path: file }); - + const file = await page.screenshot(); const screenshotBuffer = readFileSync(file); const screenshotBase64 = base64Encoded(file); const captureElementSnapshot = await page.getElementInfos(); diff --git a/packages/web-integration/src/debug/index.ts b/packages/web-integration/src/debug/index.ts index 07b195ab..2e159e92 100644 --- a/packages/web-integration/src/debug/index.ts +++ b/packages/web-integration/src/debug/index.ts @@ -3,7 +3,6 @@ import path from 'node:path'; import type { WebPage } from '@/common/page'; import type { ElementInfo } from '@/extractor'; import { NodeType } from '@/extractor/constants'; -import { getTmpFile } from '@midscene/core/utils'; import { processImageElementInfo, resizeImg, @@ -21,8 +20,7 @@ export async function generateExtractData( disableSnapshot: boolean; }, ) { - const file = getTmpFile('png'); - await page.screenshot({ path: file }); + const file = await page.screenshot(); const screenshotBuffer = readFileSync(file); const inputImgBase64 = screenshotBuffer.toString('base64'); diff --git a/packages/web-integration/src/index.ts b/packages/web-integration/src/index.ts index 3eff165f..13d0e1ea 100644 --- a/packages/web-integration/src/index.ts +++ b/packages/web-integration/src/index.ts @@ -3,6 +3,6 @@ export type { PlayWrightAiFixtureType } from './playwright'; export { PuppeteerAgent } from './puppeteer'; export { PlaywrightAgent } from './playwright'; -export { AppiumAgent } from './appium'; +export { AppiumAgent, AppiumPage } from './appium'; export { generateExtractData } from './debug'; diff --git a/packages/web-integration/src/page.ts b/packages/web-integration/src/page.ts index e226fca6..24e8d0da 100644 --- a/packages/web-integration/src/page.ts +++ b/packages/web-integration/src/page.ts @@ -1,20 +1,11 @@ import type { WebKeyInput } from './common/page'; import type { ElementInfo } from './extractor'; -type imageType = 'jpeg' | 'png'; -type encodingType = 'base64' | 'binary'; - -export type screenshotOptions = { - path?: string; - encoding?: encodingType; - type?: imageType; - quality?: number; -}; export type MouseButton = 'left' | 'right' | 'middle'; export abstract class AbstractPage { abstract pageType: string; - abstract screenshot(options?: screenshotOptions): Promise; + abstract screenshot(): Promise; abstract getElementInfos(): Promise; abstract url(): string; diff --git a/packages/web-integration/src/puppeteer/base-page.ts b/packages/web-integration/src/puppeteer/base-page.ts index fba9232e..cdb4554c 100644 --- a/packages/web-integration/src/puppeteer/base-page.ts +++ b/packages/web-integration/src/puppeteer/base-page.ts @@ -1,11 +1,12 @@ import { readFileSync, writeFileSync } from 'node:fs'; +import { getTmpFile } from '@midscene/core/utils'; import { resizeImg } from '@midscene/shared/img'; import type { Page as PlaywrightPage } from 'playwright'; import type { Page as PuppeteerPage } from 'puppeteer'; import type { WebKeyInput } from '../common/page'; import { getExtraReturnLogic } from '../common/utils'; import type { ElementInfo } from '../extractor'; -import type { AbstractPage, screenshotOptions } from '../page'; +import type { AbstractPage } from '../page'; import type { MouseButton } from '../page'; export class Page< @@ -37,12 +38,7 @@ export class Page< return captureElementSnapshot as ElementInfo[]; } - async screenshot(options: screenshotOptions): Promise { - const { path } = options; - if (!path) { - throw new Error('path is required for screenshot'); - } - + async screenshot(): Promise { // get viewport size from page const viewportSize: { width: number; @@ -56,6 +52,8 @@ export class Page< }; }); + const path = getTmpFile('jpeg'); + await this.page.screenshot({ path, type: 'jpeg', @@ -71,18 +69,7 @@ export class Page< writeFileSync(path, buf); } - // return await this.page.screenshot({ - // path, - // type: 'jpeg', - // quality: 75, - // clip: { - // x: 0, - // y: 0, - // width: viewportSize.width, - // height: viewportSize.height, - // scale: 1 / viewportSize.deviceScaleFactor, - // }, - // }); + return path; } url(): string { diff --git a/packages/web-integration/tests/ai/native/appium/dongchedi.test.ts b/packages/web-integration/tests/ai/native/appium/dongchedi.test.ts index 842b38ef..e6f7cb78 100644 --- a/packages/web-integration/tests/ai/native/appium/dongchedi.test.ts +++ b/packages/web-integration/tests/ai/native/appium/dongchedi.test.ts @@ -25,6 +25,9 @@ const IOS_OPTIONS = { const ANDROID_OPTIONS = { port: 4723, + waitforTimeout: 10000, + connectionRetryTimeout: 120000, + connectionRetryCount: 3, capabilities: { platformName: 'Android', 'appium:automationName': 'UiAutomator2', @@ -39,13 +42,16 @@ describe( 'appium integration', () => { it('懂车帝查找小米 SU7', async () => { - const page = await launchPage(IOS_OPTIONS); + const page = await launchPage(ANDROID_OPTIONS); const mid = new AppiumAgent(page); await mid.aiAction('点击同意按钮'); + await sleep(3000); + await mid.aiAction('点击允许获取应用位置信息'); + await sleep(3000); await mid.aiAction('点击顶部输入框'); - // await generateExtractData(page, './tmp'); - await mid.aiAction('在输入框里输入"小米SU7"，并点击搜索'); - // await sleep(3000); + await sleep(3000); + await mid.aiAction('在输入框里输入"SU7"，并点击搜索'); + await sleep(3000); const items = await mid.aiQuery( '"{carName: string, price: number }[], return item name, price', ); diff --git a/packages/web-integration/tests/ai/web/playwright/tool.ts b/packages/web-integration/tests/ai/web/playwright/tool.ts index 61022dfe..3635f763 100644 --- a/packages/web-integration/tests/ai/web/playwright/tool.ts +++ b/packages/web-integration/tests/ai/web/playwright/tool.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import type { WebPage } from '@/common/page'; import { getElementInfos } from '@/debug'; @@ -16,10 +16,8 @@ export async function generateExtractData( disableSnapshot: boolean; }, ) { - const buffer = await page.screenshot({ - encoding: 'base64', - }); - const inputImgBase64 = buffer.toString('base64'); + const filePath = await page.screenshot(); + const inputImgBase64 = readFileSync(filePath).toString('base64'); const { elementsPositionInfo, diff --git a/packages/web-integration/tests/unit-test/web-extractor.test.ts b/packages/web-integration/tests/unit-test/web-extractor.test.ts index e6b50e0e..092ea6a6 100644 --- a/packages/web-integration/tests/unit-test/web-extractor.test.ts +++ b/packages/web-integration/tests/unit-test/web-extractor.test.ts @@ -1,7 +1,6 @@ import path, { join } from 'node:path'; import { parseContextFromWebPage } from '@/common/utils'; import { generateExtractData } from '@/debug'; -import { getTmpFile } from '@midscene/core/utils'; import { imageInfo } from '@midscene/shared/img'; import { describe, expect, it } from 'vitest'; import { launchPage } from '../ai/web/puppeteer/utils'; @@ -51,8 +50,7 @@ describe( }, }); - const shotpath = getTmpFile('jpeg'); - await page.screenshot({ path: shotpath }); + const shotpath = await page.screenshot(); const info = await imageInfo(shotpath); expect(info.width).toBe(1080); @@ -69,8 +67,7 @@ describe( }, }); - const shotpath = getTmpFile('jpeg'); - await page.screenshot({ path: shotpath }); + const shotpath = await page.screenshot(); const info = await imageInfo(shotpath); expect(info.width).toBe(1080); // always 1x for screenshot