fixup! added web-llm as llm provider

Author: eric2788

src/llms/web-llm.ts

@@ -1,24 +1,21 @@
-import { CreateMLCEngine, MLCEngine } from "@mlc-ai/web-llm";
+import type { MLCEngine } from "@mlc-ai/web-llm";
 import type { LLMProviders, Session } from "~llms";
 import type { SettingSchema } from "~options/fragments/llm";
 
-
 export default class WebLLM implements LLMProviders {
 
     private static readonly DEFAULT_MODEL: string = 'Qwen2-7B-Instruct-q4f32_1-MLC'
 
     private readonly model: string
-    private readonly engine: MLCEngine
 
     constructor(settings: SettingSchema) {
         this.model = settings.model || WebLLM.DEFAULT_MODEL
-        this.engine = new MLCEngine()
     }
 
     cumulative: boolean = true
 
     async validate(): Promise<void> {
-        await this.engine.reload(this.model)
+        await this.initializeEngine()
     }
 
     async prompt(chat: string): Promise<string> {
@@ -47,11 +44,7 @@ export default class WebLLM implements LLMProviders {
     }
 
     async asSession(): Promise<Session<LLMProviders>> {
-        const engine = await CreateMLCEngine(this.model, {
-            initProgressCallback: (progress) => {
-                console.log('初始化进度:', progress)
-            }
-        })
+        const engine = await this.initializeEngine()
         return {
             async prompt(chat: string) {
                 await engine.interruptGenerate()
@@ -71,7 +64,7 @@ export default class WebLLM implements LLMProviders {
                     stream: true
                 })
                 for await (const chunk of chunks) {
-                    yield chunk.choices[0]?.text|| "";
+                    yield chunk.choices[0]?.text || "";
                     if (chunk.usage) {
                         console.debug('Usage:', chunk.usage)
                     }
@@ -81,4 +74,13 @@ export default class WebLLM implements LLMProviders {
         }
     }
 
+    private async initializeEngine(): Promise<MLCEngine> {
+        const { CreateMLCEngine } = await import('@mlc-ai/web-llm')
+        return CreateMLCEngine(this.model, {
+            initProgressCallback: (progress) => {
+                console.log('初始化进度:', progress)
+            }
+        })
+    }
+
 }

tests/units/llm.spec.ts

@@ -3,6 +3,8 @@ import logger from '@tests/helpers/logger'
 import createLLMProvider from "~llms"
 
 
+test.slow()
+
 test('嘗試使用 Cloudflare AI 對話', { tag: "@scoped" }, async () => {
 
     test.skip(!process.env.CF_ACCOUNT_ID || !process.env.CF_API_TOKEN, '請設定 CF_ACCOUNT_ID 和 CF_API_TOKEN 環境變數')
@@ -93,4 +95,53 @@ test('嘗試使用 Remote Worker 對話', { tag: "@scoped" }, async () => {
     expect(msg).not.toBeUndefined()
     expect(msg).not.toBe('')
 
+})
+
+test('嘗試使用 Web LLM 對話', { tag: "@scoped" }, async ({ modules, page }) => {
+
+    test.setTimeout(0)
+    await modules['llm'].loadToPage()
+
+    logger.info('正在测试 json 返回请求...')
+    const res = await page.evaluate(async () => {
+        
+        const { llms } = window as any
+        const llm = llms.createLLMProvider({ provider: 'webllm', module: 'Phi-3-mini-128k-instruct-q0f16-MLC' })
+        const session = await llm.asSession()
+        {
+            (window as any).session = session
+        }
+        return await session.prompt('你好')
+    })
+
+    logger.info('response: ', res)
+    expect(res).not.toBeUndefined()
+    expect(res).not.toBe('')
+
+    logger.info('正在测试 SSE 请求...')
+    const msg = await page.evaluate(async () => {
+        const { session } = window as any
+        if (!session) {
+            console.error('session not found')
+            return undefined
+        }
+        const res = session.promptStream('地球为什么是圆的?')
+        let msg = '';
+        for await (const r of res) {
+            console.log('response: ', r)
+            msg = r
+        }
+        return msg
+    })
+
+    expect(msg).not.toBeUndefined()
+    expect(msg).not.toBe('')
+
+    logger.info("正在清除已下載的 LLM...")
+    await page.evaluate(async () => {
+        const { session } = window as any
+        if (!session) return
+        await session[Symbol.asyncDispose]()
+    })
+    
 })