From c951c54a9f7c425b5da108e7ced5d862688a7ca3 Mon Sep 17 00:00:00 2001 From: Rrrrrrray <28804884+Rrrrrrray@users.noreply.github.com> Date: Tue, 31 Dec 2024 10:28:07 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E7=AD=89=E5=BE=85=E5=85=83=E7=B4=A0?= =?UTF-8?q?=E8=B6=85=E6=97=B610=E7=A7=92=EF=BC=8C=E4=BB=A5=E9=98=B2?= =?UTF-8?q?=E5=85=83=E7=B4=A0=E4=B8=8D=E5=AD=98=E5=9C=A8=E7=AD=89=E5=BE=85?= =?UTF-8?q?=E8=BF=87=E9=95=BF=20(#28)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: 等待元素超时10秒，以防元素不存在等待过长 * refactor: 移除旧的eslint配置，添加新的eslint配置文件；更新package.json中的时间字段 --------- Co-authored-by: 时瑾 <74231782+sj817@users.noreply.github.com> --- eslint.config.js => eslint.config.mjs | 0 packages/puppeteer-core/package.json | 1 + packages/puppeteer-core/src/puppeteer/core.ts | 177 +++++++++++++----- .../puppeteer-core/src/puppeteer/index.ts | 2 - packages/puppeteer/package.json | 1 + 5 files changed, 128 insertions(+), 53 deletions(-) rename eslint.config.js => eslint.config.mjs (100%) diff --git a/eslint.config.js b/eslint.config.mjs similarity index 100% rename from eslint.config.js rename to eslint.config.mjs diff --git a/packages/puppeteer-core/package.json b/packages/puppeteer-core/package.json index 3501217..4ecb9db 100644 --- a/packages/puppeteer-core/package.json +++ b/packages/puppeteer-core/package.json @@ -25,6 +25,7 @@ "types": "./lib/puppeteer.d.ts" } }, + "time": "2024-13-31", "license": "MIT", "author": "shijin", "type": "module", diff --git a/packages/puppeteer-core/src/puppeteer/core.ts b/packages/puppeteer-core/src/puppeteer/core.ts index 1280489..2cd6777 100644 --- a/packages/puppeteer-core/src/puppeteer/core.ts +++ b/packages/puppeteer-core/src/puppeteer/core.ts @@ -9,18 +9,18 @@ import type { Page, ScreenshotOptions, ElementHandle, - BoundingBox + BoundingBox, } from 'puppeteer-core' export interface screenshot extends ScreenshotOptions { /** http地址、本地文件路径、html字符串 */ file: string /** - * 选择的元素截图 - * fullPage为false时生效 - * 如果未找到指定元素则使用body - * @default 'body' - */ + * 选择的元素截图 + * fullPage为false时生效 + * 如果未找到指定元素则使用body + * @default 'body' + */ selector?: string /** 截图类型 默认'jpeg' */ type?: 'png' | 'jpeg' | 'webp' @@ -77,7 +77,7 @@ export interface screenshot extends ScreenshotOptions { /** 分页截图 传递数字则视为视窗高度 返回数组 */ multiPage?: number | boolean /** 页面goto时的参数 */ - pageGotoParams?: WaitForOptions, + pageGotoParams?: WaitForOptions /** 等待指定元素加载完成 */ waitForSelector?: string | string[] /** 等待特定函数完成 */ @@ -91,11 +91,11 @@ export interface screenshot extends ScreenshotOptions { } /** 截图返回 */ -export type RenderEncoding = T['encoding'] extends 'base64' ? string : Buffer +export type RenderEncoding = + T['encoding'] extends 'base64' ? string : Buffer /** 单页或多页截图返回 */ -export type RenderResult = T['multiPage'] extends true | number - ? RenderEncoding extends string ? string[] : Buffer[] - : RenderEncoding +export type RenderResult = T['multiPage'] extends | true | number + ? RenderEncoding extends string ? string[] : Buffer[] : RenderEncoding export class Render { /** 浏览器id */ @@ -147,7 +147,10 @@ export class Render { * @param data 截图参数 * @returns 截图结果 */ - async render (echo: string, data: T): Promise> { + async render ( + echo: string, + data: T + ): Promise> { let page: Page | undefined try { this.list.set(echo, true) @@ -177,7 +180,6 @@ export class Render { return data.multiPage ? await this.handleMultiPageScreenshot(body!, data, box, timeout) : await this.handleSingleElementScreenshot(body!, options, timeout) - } finally { this.cleanupPage(page, echo) } @@ -196,13 +198,22 @@ export class Render { ): ReturnType { return new Promise((resolve, reject) => { const timer = setTimeout(() => { - reject(new Error(JSON.stringify({ - message: `TimeoutError: Navigation Timeout Exceeded: ${timeout}ms exceeded`, - options - }, null, 2))) + reject( + new Error( + JSON.stringify( + { + message: `TimeoutError: Navigation Timeout Exceeded: ${timeout}ms exceeded`, + options, + }, + null, + 2 + ) + ) + ) }, timeout) - page.screenshot(options) + page + .screenshot(options) .then((data) => { clearTimeout(timer) resolve(data) @@ -238,44 +249,87 @@ export class Render { await page.setContent(data.file, data.pageGotoParams) } - /** 等待body加载完成 */ - await page.waitForSelector('body') + const timeout = Number(data?.pageGotoParams?.timeout) || 20000 + if (!data.waitForSelector) { + data.waitForSelector = ['body'] + } else if (!Array.isArray(data.waitForSelector)) { + data.waitForSelector = [data.waitForSelector || 'body'] + } else { + data.waitForSelector.push('body') + } + + const list: Promise[] = [] /** 等待指定元素加载完成 */ - if (data.waitForSelector) { - if (!Array.isArray(data.waitForSelector)) data.waitForSelector = [data.waitForSelector] - for (const selector of data.waitForSelector) { - await page.waitForSelector(selector).catch(() => { }) - } + const waitForSelector = async (selector: string) => { + const isExist = await this.checkElement(page, selector) + if (!isExist) return + await page.waitForSelector(selector, { timeout }).catch(() => { + console.warn(`[chrome] 页面元素 ${selector} 加载超时`) + }) } /** 等待特定函数完成 */ + const waitForFunction = async (func: string) => { + await page.waitForFunction(func, { timeout }).catch(() => { + console.warn(`[chrome] 函数 ${func} 加载超时`) + }) + } + + /** 等待特定请求完成 */ + const waitForRequest = async (req: string) => { + await page.waitForRequest(req, { timeout }).catch(() => { + console.warn(`[chrome] 请求 ${req} 加载超时`) + }) + } + + /** 等待特定响应完成 */ + const waitForResponse = async (res: string) => { + await page.waitForResponse(res, { timeout }).catch(() => { + console.warn(`[chrome] 响应 ${res} 加载超时`) + }) + } + + data.waitForSelector.forEach((selector) => { + list.push(waitForSelector(selector)) + }) + if (data.waitForFunction) { if (!Array.isArray(data.waitForFunction)) data.waitForFunction = [data.waitForFunction] - for (const func of data.waitForFunction) { - await page.waitForFunction(func).catch(() => { }) - } + data.waitForFunction.forEach((func) => { + list.push(waitForFunction(func)) + }) } - /** 等待特定请求完成 */ if (data.waitForRequest) { if (!Array.isArray(data.waitForRequest)) data.waitForRequest = [data.waitForRequest] - for (const req of data.waitForRequest) { - await page.waitForRequest(req).catch(() => { }) - } + data.waitForRequest.forEach((req) => { + list.push(waitForRequest(req)) + }) } - /** 等待特定响应完成 */ if (data.waitForResponse) { if (!Array.isArray(data.waitForResponse)) data.waitForResponse = [data.waitForResponse] - for (const res of data.waitForResponse) { - await page.waitForResponse(res).catch(() => { }) - } + data.waitForResponse.forEach((res) => { + list.push(waitForResponse(res)) + }) } + /** 等待所有任务完成 */ + await Promise.allSettled(list) return page } + /** + * 检查指定元素是否存在 + * @param page 页面实例 + * @param selector 选择器 + */ + async checkElement (page: Page, selector: string): Promise { + const result = await page.$(selector) + return !!result + } + /** * 获取页面元素 * @param page 页面实例 @@ -284,13 +338,16 @@ export class Render { async elementHandle (page: Page, name?: string) { try { if (name) { - const element = await page.$(name) || await page.$('#container') || await page.$('body') + const element = + (await page.$(name)) || + (await page.$('#container')) || + (await page.$('body')) return element } - const element = await page.$('#container') || await page.$('body') + const element = (await page.$('#container')) || (await page.$('body')) return element } catch (err) { - return await page.$('#container') || await page.$('body') + return (await page.$('#container')) || (await page.$('body')) } } @@ -300,7 +357,12 @@ export class Render { * @param width 视窗宽度 * @param height 视窗高度 */ - async setViewport (page: Page, width?: number, height?: number, deviceScaleFactor?: number) { + async setViewport ( + page: Page, + width?: number, + height?: number, + deviceScaleFactor?: number + ) { if (!width && !height && !deviceScaleFactor) return const setViewport = { width: Math.round(width || 1920), @@ -339,9 +401,9 @@ export class Render { req.continue() }) - page.on('response', request => delCount(request.url())) - page.on('requestfailed', request => delCount(request.url())) - page.on('requestfinished', request => delCount(request.url())) + page.on('response', (request) => delCount(request.url())) + page.on('requestfailed', (request) => delCount(request.url())) + page.on('requestfinished', (request) => delCount(request.url())) /** 加载页面 */ let result @@ -374,7 +436,7 @@ export class Render { const options = { path: data.path, type: data.type || 'jpeg', - quality: data.quality || 90 as number | undefined, + quality: data.quality || (90 as number | undefined), fullPage: data.fullPage || false, optimizeForSpeed: data.optimizeForSpeed || false, encoding: data.encoding || 'binary', @@ -445,13 +507,20 @@ export class Render { const list: Array = [] const boxWidth = box?.width ?? 1200 const boxHeight = box?.height ?? 2000 - const height = typeof data.multiPage === 'number' - ? data.multiPage - : (boxHeight >= 2000 ? 2000 : boxHeight) + const height = + typeof data.multiPage === 'number' + ? data.multiPage + : boxHeight >= 2000 + ? 2000 + : boxHeight const count = Math.ceil(boxHeight / height) for (let i = 0; i < count; i++) { - const { y, clipHeight } = this.calculatePageDimensions(i, height, boxHeight) + const { y, clipHeight } = this.calculatePageDimensions( + i, + height, + boxHeight + ) data.clip = { x: 0, y, width: boxWidth, height: clipHeight } const uint8Array = await this.screenshot(body, data, timeout) list.push(uint8Array) @@ -466,7 +535,11 @@ export class Render { * @param pageHeight 页面高度 * @param totalHeight 总高度 */ - private calculatePageDimensions (pageIndex: number, pageHeight: number, totalHeight: number) { + private calculatePageDimensions ( + pageIndex: number, + pageHeight: number, + totalHeight: number + ) { let y = pageIndex * pageHeight let clipHeight = Math.min(pageHeight, totalHeight - pageIndex * pageHeight) @@ -490,9 +563,11 @@ export class Render { if (!this.config.debug) { await page.close() if (!page.isClosed()) { - await page.close().catch((error) => process.emit('uncaughtException', error)) + await page + .close() + .catch((error) => process.emit('uncaughtException', error)) } } } } -} \ No newline at end of file +} diff --git a/packages/puppeteer-core/src/puppeteer/index.ts b/packages/puppeteer-core/src/puppeteer/index.ts index 9bb555f..a00737b 100644 --- a/packages/puppeteer-core/src/puppeteer/index.ts +++ b/packages/puppeteer-core/src/puppeteer/index.ts @@ -11,7 +11,6 @@ export const PUPPETEER_REVISIONS = Object.freeze({ firefox: 'stable_133.0', }) - export interface RunConfig { /** * 启动浏览器数量 @@ -61,7 +60,6 @@ export interface RunConfig { */ export type LaunchOptions = Parameters[0] & RunConfig - /** * @description Puppeteer多浏览器实例管理 */ diff --git a/packages/puppeteer/package.json b/packages/puppeteer/package.json index ff4aec9..761cf56 100644 --- a/packages/puppeteer/package.json +++ b/packages/puppeteer/package.json @@ -15,6 +15,7 @@ "type": "git", "url": "git+https://github.com/KarinJS/puppeteer.git" }, + "time": "2024-13-31", "license": "MIT", "author": "shijin", "type": "module",