From 8fe80bdbc6639eee6ec219aba7cc1d2738a7fceb Mon Sep 17 00:00:00 2001 From: Nestor Qin Date: Thu, 30 May 2024 01:41:42 -0400 Subject: [PATCH] [chore] format code and fix lint issues (#437) This PR formats the entire repo and fixes or suppresses all lint issues by running the following commands: - `npm run format` - `npm run lint` --- .eslintrc.cjs | 8 +- examples/README.md | 13 +- examples/cache-usage/README.md | 3 +- examples/cache-usage/src/cache_usage.html | 38 +- .../README.md | 14 +- .../src/background.ts | 5 +- .../src/content.js | 10 +- .../src/example.html | 31 +- .../src/manifest.json | 8 +- .../src/popup.css | 249 +++++----- .../src/popup.html | 41 +- .../src/popup.ts | 1 - examples/chrome-extension/README.md | 2 +- examples/chrome-extension/src/content.js | 10 +- examples/chrome-extension/src/example.html | 31 +- examples/chrome-extension/src/manifest.json | 8 +- .../chrome-extension/src/manifest_v2.json | 9 +- examples/chrome-extension/src/popup.css | 249 +++++----- examples/chrome-extension/src/popup.html | 41 +- .../src/function_calling.html | 26 +- .../src/get_started.html | 11 +- examples/get-started/src/get_started.html | 11 +- examples/json-mode/src/json_mode.html | 18 +- examples/json-schema/src/json_schema.html | 18 +- .../logit-processor/src/logit_processor.html | 26 +- .../logit-processor/src/my_logit_processor.ts | 16 +- .../src/multi_round_chat.html | 26 +- examples/next-simple-chat/.eslintrc.json | 3 - examples/next-simple-chat/next.config.js | 9 +- examples/next-simple-chat/postcss.config.js | 2 +- examples/next-simple-chat/src/pages/_app.tsx | 6 +- .../next-simple-chat/src/pages/_document.tsx | 4 +- .../next-simple-chat/src/pages/api/hello.ts | 10 +- examples/next-simple-chat/src/pages/index.tsx | 2 +- .../next-simple-chat/src/styles/globals.css | 20 +- .../src/utils/chat_component.tsx | 8 +- examples/next-simple-chat/tailwind.config.js | 14 +- examples/seed-to-reproduce/src/seed.html | 22 +- examples/service-worker/src/index.html | 11 +- examples/service-worker/src/sw.ts | 2 +- examples/simple-chat-js/index.css | 6 +- examples/simple-chat-js/index.html | 16 +- examples/simple-chat-ts/README.md | 6 +- examples/simple-chat-ts/src/llm_chat.css | 206 ++++---- examples/simple-chat-ts/src/llm_chat.html | 17 +- examples/simple-chat-upload/README.md | 6 +- examples/simple-chat-upload/package.json | 38 +- examples/simple-chat-upload/src/gh-config.js | 6 +- examples/simple-chat-upload/src/llm_chat.css | 206 ++++---- examples/simple-chat-upload/src/llm_chat.html | 29 +- .../simple-chat-upload/src/simple_chat.ts | 180 +++---- examples/streaming/src/streaming.html | 30 +- package-lock.json | 294 ++++++++++++ package.json | 5 +- src/extension_service_worker.ts | 2 +- src/service_worker.ts | 2 +- tests/conv_template.test.ts | 223 ++++----- tests/function_calling.test.ts | 275 +++++++---- tests/generation_config.test.ts | 176 +++---- tests/multi_round_chat.test.ts | 451 +++++++++--------- .../src/vram_requirements.html | 4 +- 61 files changed, 1845 insertions(+), 1369 deletions(-) delete mode 100644 examples/next-simple-chat/.eslintrc.json diff --git a/.eslintrc.cjs b/.eslintrc.cjs index c3240d54..71cfde1c 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -5,13 +5,15 @@ module.exports = { root: true, rules: { "@typescript-eslint/no-explicit-any": "off", - "@typescript-eslint/no-empty-function": "off" + "@typescript-eslint/no-empty-function": "off", + "@typescript-eslint/no-non-null-assertion": "off", }, overrides: [ { - "files": ["examples/**/*.js"], + "files": ["examples/**/*.js", "examples/**/*.ts"], "rules": { - "no-undef": "off" + "no-undef": "off", + "@typescript-eslint/no-unused-vars": "off" } } ] diff --git a/examples/README.md b/examples/README.md index 7d09a37f..22e42eb9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,21 +8,26 @@ Please send a pull request if you find things that belongs to here. Note that all examples below run in-browser and use WebGPU as a backend. #### Project List + - [get-started](get-started): minimum get started example with chat completion. [![Open on JSFiddle](https://img.shields.io/badge/open-JSFiddle-blue?logo=jsfiddle&logoColor=white)](https://jsfiddle.net/neetnestor/yac9gbwf/) [![Open on Codepen](https://img.shields.io/badge/open-codepen-gainsboro?logo=codepen)](https://codepen.io/neetnestor/pen/NWVdgey) + - [simple-chat-js](simple-chat-js): a mininum and complete chat bot app in vanilla JavaScript. [![Open on JSFiddle](https://img.shields.io/badge/open-JSFiddle-blue?logo=jsfiddle&logoColor=white)](https://jsfiddle.net/neetnestor/4nmgvsa2/) [![Open on Codepen](https://img.shields.io/badge/open-codepen-gainsboro?logo=codepen)](https://codepen.io/neetnestor/pen/vYwgZaG) + - [simple-chat-ts](simple-chat-ts): a mininum and complete chat bot app in TypeScript. - [get-started-web-worker](get-started-web-worker): same as get-started, but using web worker. - [next-simple-chat](next-simple-chat): a mininum and complete chat bot app with [Next.js](https://nextjs.org/). - [multi-round-chat](multi-round-chat): while APIs are functional, we internally optimize so that multi round chat usage can reuse KV cache #### Advanced OpenAI API Capabilities + These examples demonstrate various capabilities via WebLLM's OpenAI-like API. + - [streaming](streaming): return output as chunks in real-time in the form of an AsyncGenerator - [json-mode](json-mode): efficiently ensure output is in json format, see [OpenAI Reference](https://platform.openai.com/docs/guides/text-generation/chat-completions-api) for more. - [json-schema](json-schema): besides guaranteeing output to be in JSON, ensure output to adhere to a specific JSON schema specified the user @@ -30,17 +35,19 @@ These examples demonstrate various capabilities via WebLLM's OpenAI-like API. - [seed-to-reproduce](seed-to-reproduce): use seeding to ensure reproducible output with fields `seed`. #### Chrome Extension + - [chrome-extension](chrome-extension): chrome extension that does not have a persistent background - [chrome-extension-webgpu-service-worker](chrome-extension-webgpu-service-worker): chrome extension using service worker, hence having a persistent background #### Others + - [logit-processor](logit-processor): while `logit_bias` is supported, we additionally support stateful logit processing where users can specify their own rules. We also expose low-level API `forwardTokensAndSample()`. - [cache-usage](cache-usage): demonstrates how WebLLM supports both the [Cache API](https://developer.mozilla.org/en-US/docs/Web/API/Cache) and [IndexedDB cache](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API), and -users can pick with `appConfig.useIndexedDBCache`. Also demonstrates various cache utils such as checking -whether a model is cached, deleting a model's weights from cache, deleting a model library wasm from cache, etc. + users can pick with `appConfig.useIndexedDBCache`. Also demonstrates various cache utils such as checking + whether a model is cached, deleting a model's weights from cache, deleting a model library wasm from cache, etc. - [simple-chat-upload](simple-chat-upload): demonstrates how to upload local models to WebLLM instead of downloading via a URL link ## Demo Spaces -- [web-llm-embed](https://huggingface.co/spaces/matthoffner/web-llm-embed): document chat prototype using react-llm with transformers.js embeddings +- [web-llm-embed](https://huggingface.co/spaces/matthoffner/web-llm-embed): document chat prototype using react-llm with transformers.js embeddings - [DeVinci](https://x6occ-biaaa-aaaai-acqzq-cai.icp0.io/): AI chat app based on WebLLM and hosted on decentralized cloud platform diff --git a/examples/cache-usage/README.md b/examples/cache-usage/README.md index 196d444e..dab6d623 100644 --- a/examples/cache-usage/README.md +++ b/examples/cache-usage/README.md @@ -6,10 +6,9 @@ demonstrate the utility cache functions such as deleting models, checking if mod For more information about the two caches, see: https://developer.mozilla.org/en-US/docs/Web/API/Storage_API/Storage_quotas_and_eviction_criteria#what_technologies_store_data_in_the_browser. -To inspect the downloaded artifacts in your browser, open up developer console, go to application, +To inspect the downloaded artifacts in your browser, open up developer console, go to application, and you will find the artifacts under either `IndexedDB` or `Cache storage`. - To run the exapmle, you can do the following steps under this folder ```bash diff --git a/examples/cache-usage/src/cache_usage.html b/examples/cache-usage/src/cache_usage.html index 7a33a746..fd704b6d 100644 --- a/examples/cache-usage/src/cache_usage.html +++ b/examples/cache-usage/src/cache_usage.html @@ -1,24 +1,24 @@ - + - + - -

WebLLM Test Page

- Open console to see output -
-
- + +

WebLLM Test Page

+ Open console to see output +
+
+ -

Prompt

- +

Prompt

+ -

Response

- -
- +

Response

+ +
+ - - - \ No newline at end of file + + + diff --git a/examples/chrome-extension-webgpu-service-worker/README.md b/examples/chrome-extension-webgpu-service-worker/README.md index 6b67c0f6..8cff5339 100644 --- a/examples/chrome-extension-webgpu-service-worker/README.md +++ b/examples/chrome-extension-webgpu-service-worker/README.md @@ -2,7 +2,6 @@ ![Chrome Extension](https://github.com/mlc-ai/mlc-llm/assets/11940172/0d94cc73-eff1-4128-a6e4-70dc879f04e0) - > [!WARNING] > Service worker support in WebGPU is enabled by default in [Chrome 124](https://chromiumdash.appspot.com/commit/8d78510e4aca5ac3cd8ee4a33e96b404eaa43246). > If you are using Chrome 123, go to `chrome://flags/#enable-experimental-web-platform-features`, enable the `#enable-experimental-web-platform-features` flag, and **relaunch the browser**. @@ -10,11 +9,12 @@ This example shows how we can create a Chrome extension using WebGPU and service worker. - The project structure is as follows: - - `manifest.json`: A required file that lists important information about the structure and behavior of that extension. Here we are using manifest V3. - - `popup.ts`: Script of the extension pop-up window. - - `background.ts`: Script of the service worker. An extension service worker is loaded when it is needed, and unloaded when it goes dormant. - - `content.js`: Content script that interacts with DOM. + - `manifest.json`: A required file that lists important information about the structure and behavior of that extension. Here we are using manifest V3. + - `popup.ts`: Script of the extension pop-up window. + - `background.ts`: Script of the service worker. An extension service worker is loaded when it is needed, and unloaded when it goes dormant. + - `content.js`: Content script that interacts with DOM. - Run + ```bash npm install npm run build @@ -22,8 +22,8 @@ This example shows how we can create a Chrome extension using WebGPU and service This will create a new directory at `./dist/`. To load the extension into Chrome, go to Extensions > Manage Extensions and select Load Unpacked. Add the `./dist/` directory. You can now pin the extension to your toolbar and use it to chat with your favorite model! -**Note**: This example disables chatting using the contents of the active tab by default. +**Note**: This example disables chatting using the contents of the active tab by default. To enable it, set `useContext` in `popup.ts` to `true`. More info about this feature can be found -[here](https://github.com/mlc-ai/web-llm/pull/190). +[here](https://github.com/mlc-ai/web-llm/pull/190). However, if the web content is too large, it might run into issues. We recommend using `example.html` to test this feature. diff --git a/examples/chrome-extension-webgpu-service-worker/src/background.ts b/examples/chrome-extension-webgpu-service-worker/src/background.ts index 4b467347..f5035c7c 100644 --- a/examples/chrome-extension-webgpu-service-worker/src/background.ts +++ b/examples/chrome-extension-webgpu-service-worker/src/background.ts @@ -1,4 +1,7 @@ -import { ExtensionServiceWorkerMLCEngineHandler, MLCEngine } from "@mlc-ai/web-llm"; +import { + ExtensionServiceWorkerMLCEngineHandler, + MLCEngine, +} from "@mlc-ai/web-llm"; // Hookup an engine to a service worker handler const engine = new MLCEngine(); diff --git a/examples/chrome-extension-webgpu-service-worker/src/content.js b/examples/chrome-extension-webgpu-service-worker/src/content.js index 9054a4f7..86dab072 100644 --- a/examples/chrome-extension-webgpu-service-worker/src/content.js +++ b/examples/chrome-extension-webgpu-service-worker/src/content.js @@ -1,6 +1,6 @@ // Only the content script is able to access the DOM -chrome.runtime.onConnect.addListener(function(port) { - port.onMessage.addListener(function(msg) { - port.postMessage({contents: document.body.innerHTML}); - }); - }); \ No newline at end of file +chrome.runtime.onConnect.addListener(function (port) { + port.onMessage.addListener(function (msg) { + port.postMessage({ contents: document.body.innerHTML }); + }); +}); diff --git a/examples/chrome-extension-webgpu-service-worker/src/example.html b/examples/chrome-extension-webgpu-service-worker/src/example.html index 11936595..aa8db631 100644 --- a/examples/chrome-extension-webgpu-service-worker/src/example.html +++ b/examples/chrome-extension-webgpu-service-worker/src/example.html @@ -1,11 +1,20 @@ -In the year 2154, humanity had colonized several planets in the distant reaches of the galaxy. The planet of Xylophia-IV was one of the most remote and inhospitable, with temperatures often dropping to -200 degrees Celsius. Despite these harsh conditions, a team of scientists had established a research station on the planet to study the unique geological formations and exotic flora and fauna. - -One day, while conducting a routine survey of the planet's surface, the team discovered an strange object buried deep in the ice. As they examined it closer, they realized it was a small, metallic capsule with a glowing blue symbol etched onto its surface. - -The team's leader, a brilliant scientist named Dr. Maria Rodriguez, was immediately intrigued by the capsule's mysterious origins. She ordered her team to bring it back to the research station for further analysis. - -After weeks of studying the capsule, the team finally cracked the code to the symbol etched onto its surface. It was a message from an alien race, warning Earth of an impending attack from an unknown threat. - -The team was shocked and dismayed by the news, but they knew they had to act quickly to warn the rest of humanity. They transmitted the message to the nearest space station, which relayed it to Earth's government. - -As the threat of attack loomed near, the team remained on high alert, ready to face whatever dangers lay ahead. They had uncovered a secrets of the universe, and now they were determined to protect their planet and its inhabitants at all costs. \ No newline at end of file +In the year 2154, humanity had colonized several planets in the distant reaches +of the galaxy. The planet of Xylophia-IV was one of the most remote and +inhospitable, with temperatures often dropping to -200 degrees Celsius. Despite +these harsh conditions, a team of scientists had established a research station +on the planet to study the unique geological formations and exotic flora and +fauna. One day, while conducting a routine survey of the planet's surface, the +team discovered an strange object buried deep in the ice. As they examined it +closer, they realized it was a small, metallic capsule with a glowing blue +symbol etched onto its surface. The team's leader, a brilliant scientist named +Dr. Maria Rodriguez, was immediately intrigued by the capsule's mysterious +origins. She ordered her team to bring it back to the research station for +further analysis. After weeks of studying the capsule, the team finally cracked +the code to the symbol etched onto its surface. It was a message from an alien +race, warning Earth of an impending attack from an unknown threat. The team was +shocked and dismayed by the news, but they knew they had to act quickly to warn +the rest of humanity. They transmitted the message to the nearest space station, +which relayed it to Earth's government. As the threat of attack loomed near, the +team remained on high alert, ready to face whatever dangers lay ahead. They had +uncovered a secrets of the universe, and now they were determined to protect +their planet and its inhabitants at all costs. diff --git a/examples/chrome-extension-webgpu-service-worker/src/manifest.json b/examples/chrome-extension-webgpu-service-worker/src/manifest.json index 4e8b2b0f..22144225 100644 --- a/examples/chrome-extension-webgpu-service-worker/src/manifest.json +++ b/examples/chrome-extension-webgpu-service-worker/src/manifest.json @@ -26,9 +26,5 @@ "service_worker": "background.ts", "type": "module" }, - "permissions": [ - "storage", - "tabs", - "webNavigation" - ] -} \ No newline at end of file + "permissions": ["storage", "tabs", "webNavigation"] +} diff --git a/examples/chrome-extension-webgpu-service-worker/src/popup.css b/examples/chrome-extension-webgpu-service-worker/src/popup.css index 7285d4fd..7984721d 100644 --- a/examples/chrome-extension-webgpu-service-worker/src/popup.css +++ b/examples/chrome-extension-webgpu-service-worker/src/popup.css @@ -7,229 +7,234 @@ } html { - font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Helvetica, Arial, + font-family: + -apple-system, + BlinkMacSystemFont, + Segoe UI, + Helvetica, + Arial, sans-serif; color: #222; } body { - margin: 0; - padding: 0.5rem; - background-color: #778da9; - width: 320px; - font-size: small; + margin: 0; + padding: 0.5rem; + background-color: #778da9; + width: 320px; + font-size: small; } p { - margin: 0; + margin: 0; } /* LOADING BAR */ #loadingContainer { - margin-bottom: 15px; - width: 300px; - height: 8px; - } + margin-bottom: 15px; + width: 300px; + height: 8px; +} /* INPUT AREA */ #query-input { - border: 1px solid #ccc; - border-radius: 4px; + border: 1px solid #ccc; + border-radius: 4px; } .input-container { - display: flex; - flex-direction: row; - align-items: center; + display: flex; + flex-direction: row; + align-items: center; } .input-container input { - width: 100%; - outline: none; - padding: 0.5rem; - margin-right: 0.5rem; + width: 100%; + outline: none; + padding: 0.5rem; + margin-right: 0.5rem; } /* SUBMIT BUTTON */ .btn { - background-color: #1b263b; - color: white; - font-size: small; - cursor: pointer; - border-radius: 4px; - border: none; - padding: 0.5rem; + background-color: #1b263b; + color: white; + font-size: small; + cursor: pointer; + border-radius: 4px; + border: none; + padding: 0.5rem; } .btn:hover { - background-color: #d0d0d0; + background-color: #d0d0d0; } .btn:disabled { - background-color: #a7a7a7; - color: rgb(255, 255, 255); - cursor: default; + background-color: #a7a7a7; + color: rgb(255, 255, 255); + cursor: default; } .btn img { - width: 1rem; - height: 1rem; + width: 1rem; + height: 1rem; } /* LOADING */ .stage { - display: flex; - justify-content: center; - align-items: center; - position: relative; - margin: 0 -5%; - overflow: hidden; + display: flex; + justify-content: center; + align-items: center; + position: relative; + margin: 0 -5%; + overflow: hidden; } #loading-indicator { - display: none; - color: white; - margin-top: 0.5rem; + display: none; + color: white; + margin-top: 0.5rem; } .dot-flashing { - position: relative; - width: 10px; - height: 10px; - border-radius: 5px; - background-color: #1b263b; - color: #1b263b; - animation: dot-flashing 0.4s infinite linear alternate; - animation-delay: 0.2s; + position: relative; + width: 10px; + height: 10px; + border-radius: 5px; + background-color: #1b263b; + color: #1b263b; + animation: dot-flashing 0.4s infinite linear alternate; + animation-delay: 0.2s; } .dot-flashing::before, .dot-flashing::after { - content: ""; - display: inline-block; - position: absolute; - top: 0; + content: ""; + display: inline-block; + position: absolute; + top: 0; } .dot-flashing::before { - left: -15px; - width: 10px; - height: 10px; - border-radius: 5px; - background-color: #1b263b; - color: #1b263b; - animation: dot-flashing 0.4s infinite alternate; - animation-delay: 0s; + left: -15px; + width: 10px; + height: 10px; + border-radius: 5px; + background-color: #1b263b; + color: #1b263b; + animation: dot-flashing 0.4s infinite alternate; + animation-delay: 0s; } .dot-flashing::after { - left: 15px; - width: 10px; - height: 10px; - border-radius: 5px; - background-color: #1b263b; - color: #1b263b; - animation: dot-flashing 0.4s infinite alternate; - animation-delay: 0.4s; + left: 15px; + width: 10px; + height: 10px; + border-radius: 5px; + background-color: #1b263b; + color: #1b263b; + animation: dot-flashing 0.4s infinite alternate; + animation-delay: 0.4s; } @keyframes dot-flashing { - 0% { - background-color: #1b263b; - } + 0% { + background-color: #1b263b; + } - 50%, - 100% { - background-color: #415a77; - } + 50%, + 100% { + background-color: #415a77; + } } /* ANSWERS */ #queriesAnswersContainer { - display: block; - color: white; - margin-top: 0.5rem; + display: block; + color: white; + margin-top: 0.5rem; } #answer { - color: #333333; + color: #333333; } #answerWrapper { - display: none; - background-color: #ffd166; - border-radius: 8px; - padding: 0.5rem; - margin-top: 0.5rem; + display: none; + background-color: #ffd166; + border-radius: 8px; + padding: 0.5rem; + margin-top: 0.5rem; } .queriesAnswers { - border-radius: 8px; - background-color: #ffd166;; - padding: 0.5rem; - color: #333333; + border-radius: 8px; + background-color: #ffd166; + padding: 0.5rem; + color: #333333; } #lastQuery { - color: rgb(188, 188, 188); + color: rgb(188, 188, 188); } #lastAnswer { - color: white; - margin-top: 0.5rem; + color: white; + margin-top: 0.5rem; } #lastRequest { - padding: 0.5rem; - margin-top: 0.5rem; - background-color: #333333; - border-radius: 4px; + padding: 0.5rem; + margin-top: 0.5rem; + background-color: #333333; + border-radius: 4px; } /* ANSWER OPTIONS */ .timeStamp { - color: #9a8c98; + color: #9a8c98; } .copyRow { - display: flex; - flex-direction: row; - align-items: end; - justify-content: space-between; - color: #a7a7a7; - margin-top: 0.5rem; + display: flex; + flex-direction: row; + align-items: end; + justify-content: space-between; + color: #a7a7a7; + margin-top: 0.5rem; } .copyText { - display: none; - color: #a7a7a7; - margin-right: 0.5rem; + display: none; + color: #a7a7a7; + margin-right: 0.5rem; } .copyButton { - color: #415a77; - background-color: transparent; - border: none; - cursor: pointer; - padding: 0; - margin-left: 0.5rem; + color: #415a77; + background-color: transparent; + border: none; + cursor: pointer; + padding: 0; + margin-left: 0.5rem; } .copyButton:hover { - color: #5e80a7; - background-color: transparent; + color: #5e80a7; + background-color: transparent; } .removeButton { - color: #415a77; - background-color: transparent; - border: none; - cursor: pointer; - padding: 0; + color: #415a77; + background-color: transparent; + border: none; + cursor: pointer; + padding: 0; } .removeButton:hover { - color: #5e80a7; - background-color: transparent; + color: #5e80a7; + background-color: transparent; } diff --git a/examples/chrome-extension-webgpu-service-worker/src/popup.html b/examples/chrome-extension-webgpu-service-worker/src/popup.html index 5a775ae3..5cbfe8d1 100644 --- a/examples/chrome-extension-webgpu-service-worker/src/popup.html +++ b/examples/chrome-extension-webgpu-service-worker/src/popup.html @@ -1,31 +1,44 @@ - + - + Chatbot - - + + -
- - + +
-
+
-
-
- - -
+
+
+ + +
diff --git a/examples/chrome-extension-webgpu-service-worker/src/popup.ts b/examples/chrome-extension-webgpu-service-worker/src/popup.ts index e8aae139..ec83c734 100644 --- a/examples/chrome-extension-webgpu-service-worker/src/popup.ts +++ b/examples/chrome-extension-webgpu-service-worker/src/popup.ts @@ -11,7 +11,6 @@ import { MLCEngineInterface, InitProgressReport, } from "@mlc-ai/web-llm"; -import { prebuiltAppConfig } from "@mlc-ai/web-llm"; import { ProgressBar, Line } from "progressbar.js"; /***************** UI elements *****************/ diff --git a/examples/chrome-extension/README.md b/examples/chrome-extension/README.md index c9e0ee8f..88d48170 100644 --- a/examples/chrome-extension/README.md +++ b/examples/chrome-extension/README.md @@ -9,4 +9,4 @@ npm install npm run build ``` -This will create a new directory at `chrome-extension/dist/`. To load the extension into Chrome, go to Extensions > Manage Extensions and select Load Unpacked. Add the `chrome-extension/dist/` directory. You can now pin the extension to your toolbar and use it to chat with your favorite model! \ No newline at end of file +This will create a new directory at `chrome-extension/dist/`. To load the extension into Chrome, go to Extensions > Manage Extensions and select Load Unpacked. Add the `chrome-extension/dist/` directory. You can now pin the extension to your toolbar and use it to chat with your favorite model! diff --git a/examples/chrome-extension/src/content.js b/examples/chrome-extension/src/content.js index 9054a4f7..86dab072 100644 --- a/examples/chrome-extension/src/content.js +++ b/examples/chrome-extension/src/content.js @@ -1,6 +1,6 @@ // Only the content script is able to access the DOM -chrome.runtime.onConnect.addListener(function(port) { - port.onMessage.addListener(function(msg) { - port.postMessage({contents: document.body.innerHTML}); - }); - }); \ No newline at end of file +chrome.runtime.onConnect.addListener(function (port) { + port.onMessage.addListener(function (msg) { + port.postMessage({ contents: document.body.innerHTML }); + }); +}); diff --git a/examples/chrome-extension/src/example.html b/examples/chrome-extension/src/example.html index 11936595..aa8db631 100644 --- a/examples/chrome-extension/src/example.html +++ b/examples/chrome-extension/src/example.html @@ -1,11 +1,20 @@ -In the year 2154, humanity had colonized several planets in the distant reaches of the galaxy. The planet of Xylophia-IV was one of the most remote and inhospitable, with temperatures often dropping to -200 degrees Celsius. Despite these harsh conditions, a team of scientists had established a research station on the planet to study the unique geological formations and exotic flora and fauna. - -One day, while conducting a routine survey of the planet's surface, the team discovered an strange object buried deep in the ice. As they examined it closer, they realized it was a small, metallic capsule with a glowing blue symbol etched onto its surface. - -The team's leader, a brilliant scientist named Dr. Maria Rodriguez, was immediately intrigued by the capsule's mysterious origins. She ordered her team to bring it back to the research station for further analysis. - -After weeks of studying the capsule, the team finally cracked the code to the symbol etched onto its surface. It was a message from an alien race, warning Earth of an impending attack from an unknown threat. - -The team was shocked and dismayed by the news, but they knew they had to act quickly to warn the rest of humanity. They transmitted the message to the nearest space station, which relayed it to Earth's government. - -As the threat of attack loomed near, the team remained on high alert, ready to face whatever dangers lay ahead. They had uncovered a secrets of the universe, and now they were determined to protect their planet and its inhabitants at all costs. \ No newline at end of file +In the year 2154, humanity had colonized several planets in the distant reaches +of the galaxy. The planet of Xylophia-IV was one of the most remote and +inhospitable, with temperatures often dropping to -200 degrees Celsius. Despite +these harsh conditions, a team of scientists had established a research station +on the planet to study the unique geological formations and exotic flora and +fauna. One day, while conducting a routine survey of the planet's surface, the +team discovered an strange object buried deep in the ice. As they examined it +closer, they realized it was a small, metallic capsule with a glowing blue +symbol etched onto its surface. The team's leader, a brilliant scientist named +Dr. Maria Rodriguez, was immediately intrigued by the capsule's mysterious +origins. She ordered her team to bring it back to the research station for +further analysis. After weeks of studying the capsule, the team finally cracked +the code to the symbol etched onto its surface. It was a message from an alien +race, warning Earth of an impending attack from an unknown threat. The team was +shocked and dismayed by the news, but they knew they had to act quickly to warn +the rest of humanity. They transmitted the message to the nearest space station, +which relayed it to Earth's government. As the threat of attack loomed near, the +team remained on high alert, ready to face whatever dangers lay ahead. They had +uncovered a secrets of the universe, and now they were determined to protect +their planet and its inhabitants at all costs. diff --git a/examples/chrome-extension/src/manifest.json b/examples/chrome-extension/src/manifest.json index fa780902..7d8458a5 100644 --- a/examples/chrome-extension/src/manifest.json +++ b/examples/chrome-extension/src/manifest.json @@ -16,9 +16,5 @@ "default_title": "MLCBot", "default_popup": "popup.html" }, - "permissions": [ - "storage", - "tabs", - "webNavigation" - ] -} \ No newline at end of file + "permissions": ["storage", "tabs", "webNavigation"] +} diff --git a/examples/chrome-extension/src/manifest_v2.json b/examples/chrome-extension/src/manifest_v2.json index e17c8402..79bba360 100644 --- a/examples/chrome-extension/src/manifest_v2.json +++ b/examples/chrome-extension/src/manifest_v2.json @@ -19,10 +19,5 @@ "js": ["content.js"] } ], - "permissions": [ - "storage", - "tabs", - "webNavigation", - "activeTab" - ] -} \ No newline at end of file + "permissions": ["storage", "tabs", "webNavigation", "activeTab"] +} diff --git a/examples/chrome-extension/src/popup.css b/examples/chrome-extension/src/popup.css index 7285d4fd..7984721d 100644 --- a/examples/chrome-extension/src/popup.css +++ b/examples/chrome-extension/src/popup.css @@ -7,229 +7,234 @@ } html { - font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Helvetica, Arial, + font-family: + -apple-system, + BlinkMacSystemFont, + Segoe UI, + Helvetica, + Arial, sans-serif; color: #222; } body { - margin: 0; - padding: 0.5rem; - background-color: #778da9; - width: 320px; - font-size: small; + margin: 0; + padding: 0.5rem; + background-color: #778da9; + width: 320px; + font-size: small; } p { - margin: 0; + margin: 0; } /* LOADING BAR */ #loadingContainer { - margin-bottom: 15px; - width: 300px; - height: 8px; - } + margin-bottom: 15px; + width: 300px; + height: 8px; +} /* INPUT AREA */ #query-input { - border: 1px solid #ccc; - border-radius: 4px; + border: 1px solid #ccc; + border-radius: 4px; } .input-container { - display: flex; - flex-direction: row; - align-items: center; + display: flex; + flex-direction: row; + align-items: center; } .input-container input { - width: 100%; - outline: none; - padding: 0.5rem; - margin-right: 0.5rem; + width: 100%; + outline: none; + padding: 0.5rem; + margin-right: 0.5rem; } /* SUBMIT BUTTON */ .btn { - background-color: #1b263b; - color: white; - font-size: small; - cursor: pointer; - border-radius: 4px; - border: none; - padding: 0.5rem; + background-color: #1b263b; + color: white; + font-size: small; + cursor: pointer; + border-radius: 4px; + border: none; + padding: 0.5rem; } .btn:hover { - background-color: #d0d0d0; + background-color: #d0d0d0; } .btn:disabled { - background-color: #a7a7a7; - color: rgb(255, 255, 255); - cursor: default; + background-color: #a7a7a7; + color: rgb(255, 255, 255); + cursor: default; } .btn img { - width: 1rem; - height: 1rem; + width: 1rem; + height: 1rem; } /* LOADING */ .stage { - display: flex; - justify-content: center; - align-items: center; - position: relative; - margin: 0 -5%; - overflow: hidden; + display: flex; + justify-content: center; + align-items: center; + position: relative; + margin: 0 -5%; + overflow: hidden; } #loading-indicator { - display: none; - color: white; - margin-top: 0.5rem; + display: none; + color: white; + margin-top: 0.5rem; } .dot-flashing { - position: relative; - width: 10px; - height: 10px; - border-radius: 5px; - background-color: #1b263b; - color: #1b263b; - animation: dot-flashing 0.4s infinite linear alternate; - animation-delay: 0.2s; + position: relative; + width: 10px; + height: 10px; + border-radius: 5px; + background-color: #1b263b; + color: #1b263b; + animation: dot-flashing 0.4s infinite linear alternate; + animation-delay: 0.2s; } .dot-flashing::before, .dot-flashing::after { - content: ""; - display: inline-block; - position: absolute; - top: 0; + content: ""; + display: inline-block; + position: absolute; + top: 0; } .dot-flashing::before { - left: -15px; - width: 10px; - height: 10px; - border-radius: 5px; - background-color: #1b263b; - color: #1b263b; - animation: dot-flashing 0.4s infinite alternate; - animation-delay: 0s; + left: -15px; + width: 10px; + height: 10px; + border-radius: 5px; + background-color: #1b263b; + color: #1b263b; + animation: dot-flashing 0.4s infinite alternate; + animation-delay: 0s; } .dot-flashing::after { - left: 15px; - width: 10px; - height: 10px; - border-radius: 5px; - background-color: #1b263b; - color: #1b263b; - animation: dot-flashing 0.4s infinite alternate; - animation-delay: 0.4s; + left: 15px; + width: 10px; + height: 10px; + border-radius: 5px; + background-color: #1b263b; + color: #1b263b; + animation: dot-flashing 0.4s infinite alternate; + animation-delay: 0.4s; } @keyframes dot-flashing { - 0% { - background-color: #1b263b; - } + 0% { + background-color: #1b263b; + } - 50%, - 100% { - background-color: #415a77; - } + 50%, + 100% { + background-color: #415a77; + } } /* ANSWERS */ #queriesAnswersContainer { - display: block; - color: white; - margin-top: 0.5rem; + display: block; + color: white; + margin-top: 0.5rem; } #answer { - color: #333333; + color: #333333; } #answerWrapper { - display: none; - background-color: #ffd166; - border-radius: 8px; - padding: 0.5rem; - margin-top: 0.5rem; + display: none; + background-color: #ffd166; + border-radius: 8px; + padding: 0.5rem; + margin-top: 0.5rem; } .queriesAnswers { - border-radius: 8px; - background-color: #ffd166;; - padding: 0.5rem; - color: #333333; + border-radius: 8px; + background-color: #ffd166; + padding: 0.5rem; + color: #333333; } #lastQuery { - color: rgb(188, 188, 188); + color: rgb(188, 188, 188); } #lastAnswer { - color: white; - margin-top: 0.5rem; + color: white; + margin-top: 0.5rem; } #lastRequest { - padding: 0.5rem; - margin-top: 0.5rem; - background-color: #333333; - border-radius: 4px; + padding: 0.5rem; + margin-top: 0.5rem; + background-color: #333333; + border-radius: 4px; } /* ANSWER OPTIONS */ .timeStamp { - color: #9a8c98; + color: #9a8c98; } .copyRow { - display: flex; - flex-direction: row; - align-items: end; - justify-content: space-between; - color: #a7a7a7; - margin-top: 0.5rem; + display: flex; + flex-direction: row; + align-items: end; + justify-content: space-between; + color: #a7a7a7; + margin-top: 0.5rem; } .copyText { - display: none; - color: #a7a7a7; - margin-right: 0.5rem; + display: none; + color: #a7a7a7; + margin-right: 0.5rem; } .copyButton { - color: #415a77; - background-color: transparent; - border: none; - cursor: pointer; - padding: 0; - margin-left: 0.5rem; + color: #415a77; + background-color: transparent; + border: none; + cursor: pointer; + padding: 0; + margin-left: 0.5rem; } .copyButton:hover { - color: #5e80a7; - background-color: transparent; + color: #5e80a7; + background-color: transparent; } .removeButton { - color: #415a77; - background-color: transparent; - border: none; - cursor: pointer; - padding: 0; + color: #415a77; + background-color: transparent; + border: none; + cursor: pointer; + padding: 0; } .removeButton:hover { - color: #5e80a7; - background-color: transparent; + color: #5e80a7; + background-color: transparent; } diff --git a/examples/chrome-extension/src/popup.html b/examples/chrome-extension/src/popup.html index 5a775ae3..5cbfe8d1 100644 --- a/examples/chrome-extension/src/popup.html +++ b/examples/chrome-extension/src/popup.html @@ -1,31 +1,44 @@ - + - + Chatbot - - + + -
- - + +
-
+
-
-
- - -
+
+
+ + +
diff --git a/examples/function-calling/src/function_calling.html b/examples/function-calling/src/function_calling.html index 4656c5c8..a8baff62 100644 --- a/examples/function-calling/src/function_calling.html +++ b/examples/function-calling/src/function_calling.html @@ -1,16 +1,16 @@ - + - + - -

WebLLM Test Page

- Open console to see output -
-
- + +

WebLLM Test Page

+ Open console to see output +
+
+ - - - \ No newline at end of file + + + diff --git a/examples/get-started-web-worker/src/get_started.html b/examples/get-started-web-worker/src/get_started.html index a376ef62..18c28791 100644 --- a/examples/get-started-web-worker/src/get_started.html +++ b/examples/get-started-web-worker/src/get_started.html @@ -1,13 +1,13 @@ - +

WebLLM Test Page

Open console to see output -
-
+
+

Prompt

@@ -15,8 +15,9 @@

Prompt

Response

-
+
+ diff --git a/examples/get-started/src/get_started.html b/examples/get-started/src/get_started.html index 4b7fdf76..f49432c3 100644 --- a/examples/get-started/src/get_started.html +++ b/examples/get-started/src/get_started.html @@ -1,13 +1,13 @@ - +

WebLLM Test Page

Open console to see output -
-
+
+

Prompt

@@ -15,8 +15,9 @@

Prompt

Response

-
+
+ diff --git a/examples/json-mode/src/json_mode.html b/examples/json-mode/src/json_mode.html index 7f6c7673..4532c0ca 100644 --- a/examples/json-mode/src/json_mode.html +++ b/examples/json-mode/src/json_mode.html @@ -1,16 +1,16 @@ - + - + - +

WebLLM Test Page

Open console to see output. -
-
+
+
- - \ No newline at end of file + + diff --git a/examples/json-schema/src/json_schema.html b/examples/json-schema/src/json_schema.html index e5f7d825..1cf636ad 100644 --- a/examples/json-schema/src/json_schema.html +++ b/examples/json-schema/src/json_schema.html @@ -1,16 +1,16 @@ - + - + - +

WebLLM Test Page

Open console to see output. -
-
+
+
- - \ No newline at end of file + + diff --git a/examples/logit-processor/src/logit_processor.html b/examples/logit-processor/src/logit_processor.html index 9b10b8cd..58047d27 100644 --- a/examples/logit-processor/src/logit_processor.html +++ b/examples/logit-processor/src/logit_processor.html @@ -1,16 +1,16 @@ - + - + - -

WebLLM Logit Processor Test Page

- Open console to see the effect of your logit processor. -
-
- + +

WebLLM Logit Processor Test Page

+ Open console to see the effect of your logit processor. +
+
+ - - - \ No newline at end of file + + + diff --git a/examples/logit-processor/src/my_logit_processor.ts b/examples/logit-processor/src/my_logit_processor.ts index cc2aa03d..c28e1514 100644 --- a/examples/logit-processor/src/my_logit_processor.ts +++ b/examples/logit-processor/src/my_logit_processor.ts @@ -1,21 +1,21 @@ import * as webllm from "@mlc-ai/web-llm"; - + // Define LogitProcessor export class MyLogitProcessor implements webllm.LogitProcessor { private tokenSequence: Array = []; processLogits(logits: Float32Array): Float32Array { - logits[0] = 100.0; // should be enough so that we always sample token 0 below - return logits; + logits[0] = 100.0; // should be enough so that we always sample token 0 below + return logits; } processSampledToken(token: number): void { - this.tokenSequence.push(token); - console.log("processSampledToken: " + this.tokenSequence.length); + this.tokenSequence.push(token); + console.log("processSampledToken: " + this.tokenSequence.length); } resetState(): void { - this.tokenSequence = []; - console.log("resetState"); + this.tokenSequence = []; + console.log("resetState"); } -} \ No newline at end of file +} diff --git a/examples/multi-round-chat/src/multi_round_chat.html b/examples/multi-round-chat/src/multi_round_chat.html index bb9e7b05..367bb1bc 100644 --- a/examples/multi-round-chat/src/multi_round_chat.html +++ b/examples/multi-round-chat/src/multi_round_chat.html @@ -1,16 +1,16 @@ - + - + - -

WebLLM Test Page

- Open console to see output -
-
- + +

WebLLM Test Page

+ Open console to see output +
+
+ - - - \ No newline at end of file + + + diff --git a/examples/next-simple-chat/.eslintrc.json b/examples/next-simple-chat/.eslintrc.json deleted file mode 100644 index bffb357a..00000000 --- a/examples/next-simple-chat/.eslintrc.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "next/core-web-vitals" -} diff --git a/examples/next-simple-chat/next.config.js b/examples/next-simple-chat/next.config.js index 59057101..369a10d9 100644 --- a/examples/next-simple-chat/next.config.js +++ b/examples/next-simple-chat/next.config.js @@ -2,21 +2,20 @@ const nextConfig = { reactStrictMode: true, - webpack: (config, { isServer }) => { // Fixes npm packages that depend on `fs` module if (!isServer) { config.resolve.fallback = { ...config.resolve.fallback, // if you miss it, all the other options in fallback, specified - // by next.js will be dropped. Doesn't make much sense, but how it is + // by next.js will be dropped. Doesn't make much sense, but how it is fs: false, // the solution module: false, perf_hooks: false, }; } - return config + return config; }, -} +}; -module.exports = nextConfig +module.exports = nextConfig; diff --git a/examples/next-simple-chat/postcss.config.js b/examples/next-simple-chat/postcss.config.js index 33ad091d..12a703d9 100644 --- a/examples/next-simple-chat/postcss.config.js +++ b/examples/next-simple-chat/postcss.config.js @@ -3,4 +3,4 @@ module.exports = { tailwindcss: {}, autoprefixer: {}, }, -} +}; diff --git a/examples/next-simple-chat/src/pages/_app.tsx b/examples/next-simple-chat/src/pages/_app.tsx index 6de02d15..5f38c82a 100644 --- a/examples/next-simple-chat/src/pages/_app.tsx +++ b/examples/next-simple-chat/src/pages/_app.tsx @@ -1,6 +1,6 @@ -import '~/styles/globals.css' -import type { AppProps } from 'next/app' +import "~/styles/globals.css"; +import type { AppProps } from "next/app"; export default function App({ Component, pageProps }: AppProps) { - return + return ; } diff --git a/examples/next-simple-chat/src/pages/_document.tsx b/examples/next-simple-chat/src/pages/_document.tsx index 54e8bf3e..b2fff8b4 100644 --- a/examples/next-simple-chat/src/pages/_document.tsx +++ b/examples/next-simple-chat/src/pages/_document.tsx @@ -1,4 +1,4 @@ -import { Html, Head, Main, NextScript } from 'next/document' +import { Html, Head, Main, NextScript } from "next/document"; export default function Document() { return ( @@ -9,5 +9,5 @@ export default function Document() { - ) + ); } diff --git a/examples/next-simple-chat/src/pages/api/hello.ts b/examples/next-simple-chat/src/pages/api/hello.ts index f8bcc7e5..ea77e8f3 100644 --- a/examples/next-simple-chat/src/pages/api/hello.ts +++ b/examples/next-simple-chat/src/pages/api/hello.ts @@ -1,13 +1,13 @@ // Next.js API route support: https://nextjs.org/docs/api-routes/introduction -import type { NextApiRequest, NextApiResponse } from 'next' +import type { NextApiRequest, NextApiResponse } from "next"; type Data = { - name: string -} + name: string; +}; export default function handler( req: NextApiRequest, - res: NextApiResponse + res: NextApiResponse, ) { - res.status(200).json({ name: 'John Doe' }) + res.status(200).json({ name: "John Doe" }); } diff --git a/examples/next-simple-chat/src/pages/index.tsx b/examples/next-simple-chat/src/pages/index.tsx index dc8edfff..a8df791a 100644 --- a/examples/next-simple-chat/src/pages/index.tsx +++ b/examples/next-simple-chat/src/pages/index.tsx @@ -18,7 +18,7 @@ export default function Home() {
- +
); diff --git a/examples/next-simple-chat/src/styles/globals.css b/examples/next-simple-chat/src/styles/globals.css index c5473cb9..3afe8349 100644 --- a/examples/next-simple-chat/src/styles/globals.css +++ b/examples/next-simple-chat/src/styles/globals.css @@ -101,12 +101,12 @@ s .chatui-header { } .error-msg .msg-bubble { - border-bottom-left-radius: 0; - color: #f15959; + border-bottom-left-radius: 0; + color: #f15959; } .init-msg .msg-bubble { - border-bottom-left-radius: 0; + border-bottom-left-radius: 0; } .right-msg { @@ -139,16 +139,16 @@ s .chatui-header { } .chatui-btn { - margin-left: 10px; - background: #579ffb; - color: #fff; - font-weight: bold; - cursor: pointer; - padding: 10px; + margin-left: 10px; + background: #579ffb; + color: #fff; + font-weight: bold; + cursor: pointer; + padding: 10px; } .chatui-btn:hover { - background: #577bfb; + background: #577bfb; } .chatui-chat { diff --git a/examples/next-simple-chat/src/utils/chat_component.tsx b/examples/next-simple-chat/src/utils/chat_component.tsx index c39643eb..9a392c1e 100644 --- a/examples/next-simple-chat/src/utils/chat_component.tsx +++ b/examples/next-simple-chat/src/utils/chat_component.tsx @@ -4,7 +4,7 @@ import ChatUI from "~/utils/chat_ui"; const ChatComponent = () => { const [messages, setMessages] = useState<{ kind: string; text: string }[]>( - [] + [], ); const [prompt, setPrompt] = useState(""); const [runtimeStats, setRuntimeStats] = useState(""); @@ -78,7 +78,11 @@ const ChatComponent = () => { + -

- Step 2: Chat -

+

Step 2: Chat

@@ -34,5 +27,4 @@ - diff --git a/examples/simple-chat-ts/README.md b/examples/simple-chat-ts/README.md index 4b1e0d8f..336b7d49 100644 --- a/examples/simple-chat-ts/README.md +++ b/examples/simple-chat-ts/README.md @@ -19,15 +19,17 @@ Due to the differences in command-line tools between Unix/Linux and Windows syst ### Steps for Windows Users 1. **Create a Node.js Script File**: + - In the `examples\simple-chat` directory, create a file named `copy-config.js`. - Add the following code to handle file copying: ```javascript - const fs = require('fs'); + const fs = require("fs"); // Copy file - fs.copyFileSync('src/gh-config.js', 'src/app-config.js'); + fs.copyFileSync("src/gh-config.js", "src/app-config.js"); ``` 2. **Modify `package.json`**: + - In the `scripts` section of your `package.json`, replace Unix-style `cp` commands with our new Node.js script. For example: ```json "scripts": { diff --git a/examples/simple-chat-ts/src/llm_chat.css b/examples/simple-chat-ts/src/llm_chat.css index 395714ed..79525baa 100644 --- a/examples/simple-chat-ts/src/llm_chat.css +++ b/examples/simple-chat-ts/src/llm_chat.css @@ -1,188 +1,188 @@ .chatui { - display: flex; - position: relative; - flex-flow: column wrap; - justify-content: space-between; - width: 100%; - max-width: 867px; - margin: 25px 10px; - height: 600px; - border: 2px solid #ddd; - border-radius: 5px; - background-color: #1F2027; + display: flex; + position: relative; + flex-flow: column wrap; + justify-content: space-between; + width: 100%; + max-width: 867px; + margin: 25px 10px; + height: 600px; + border: 2px solid #ddd; + border-radius: 5px; + background-color: #1f2027; } .chatui-select-wrapper { - display: flex; - justify-content: center; - background-color: #1F2027; - padding: 10px 0; + display: flex; + justify-content: center; + background-color: #1f2027; + padding: 10px 0; } #chatui-select { - width: 350px; - background-color: #1F2027; - color: white; - border: none; + width: 350px; + background-color: #1f2027; + color: white; + border: none; } #chatui-select:focus { - outline: none; + outline: none; } #chatui-select::-webkit-scrollbar { - display: none; + display: none; } #chatui-select option { - background-color: #1F2027; - color: white; + background-color: #1f2027; + color: white; } #chatui-select option:hover { - background-color: #474747; - color: white; + background-color: #474747; + color: white; } s .chatui-header { - display: flex; - justify-content: space-between; - padding: 10px; - border-bottom: 2px solid #ddd; - background: #eee; - color: #666; + display: flex; + justify-content: space-between; + padding: 10px; + border-bottom: 2px solid #ddd; + background: #eee; + color: #666; } /* Used to remove tiny white lines in android devices; not sure if there is a better way */ *, *::before, *::after { - box-sizing: content-box; + box-sizing: content-box; } .chatui-chat { - flex: 1; - overflow-y: auto; - padding: 10px; - background-color: #1F2027; + flex: 1; + overflow-y: auto; + padding: 10px; + background-color: #1f2027; } .chatui-chat::-webkit-scrollbar { - width: 6px; + width: 6px; } .chatui-chat::-webkit-scrollbar-track { - background: #1F2027; + background: #1f2027; } .chatui-chat::-webkit-scrollbar-thumb { - background: #888; + background: #888; } .chatui-chat::-webkit-scrollbar-thumb:hover { - background: #555; + background: #555; } .msg { - display: flex; - align-items: flex-end; - margin-bottom: 10px; + display: flex; + align-items: flex-end; + margin-bottom: 10px; } .msg:last-of-type { - margin: 0; + margin: 0; } .msg-bubble { - background-color: #f0f0f0; - border-radius: 8px; - padding: 16px; - margin: 5px auto; - width: calc(100% - 20px); - box-sizing: border-box; - color: black; - border: none; - font-size: medium; - margin-left: auto; - margin-right: auto; + background-color: #f0f0f0; + border-radius: 8px; + padding: 16px; + margin: 5px auto; + width: calc(100% - 20px); + box-sizing: border-box; + color: black; + border: none; + font-size: medium; + margin-left: auto; + margin-right: auto; } .left-msg .msg-bubble { - background-color: #343541; - color: #ececec; + background-color: #343541; + color: #ececec; } .error-msg .msg-bubble { - background-color: #343541; - color: #f15959; + background-color: #343541; + color: #f15959; } .init-msg .msg-bubble { - background-color: #343541; - color: #ececec; + background-color: #343541; + color: #ececec; } .right-msg .msg-bubble { - background-color: #444654; - color: #ececec; + background-color: #444654; + color: #ececec; } .chatui-inputarea { - display: flex; - padding: 10px; - border-top: 2px solid transparent; - background-color: #1F2027; + display: flex; + padding: 10px; + border-top: 2px solid transparent; + background-color: #1f2027; } .chatui-inputarea * { - padding: 10px; - border: none; - border-radius: 3px; - font-size: 1em; - color: white; - background: rgba(0, 0, 0, 0.3); + padding: 10px; + border: none; + border-radius: 3px; + font-size: 1em; + color: white; + background: rgba(0, 0, 0, 0.3); } .chatui-input { - flex: 1; - background-color: #40414F; - color: white; + flex: 1; + background-color: #40414f; + color: white; } .chatui-reset-btn { - margin-left: 10px; - background-color: #40414F; - color: #fff; - font-weight: bold; - cursor: pointer; - background-image: url('img/reset.png'); - background-repeat: no-repeat; - background-position: center; - width: 40px; - background-repeat: no-repeat; - background-position: center; - background-size: 20px 20px; + margin-left: 10px; + background-color: #40414f; + color: #fff; + font-weight: bold; + cursor: pointer; + background-image: url("img/reset.png"); + background-repeat: no-repeat; + background-position: center; + width: 40px; + background-repeat: no-repeat; + background-position: center; + background-size: 20px 20px; } .chatui-reset-btn:hover { - background-color: #03a33e; + background-color: #03a33e; } .chatui-send-btn { - margin-left: 10px; - background-color: #40414F; - color: #fff; - font-weight: bold; - cursor: pointer; - background-image: url('img/plane.png'); - background-repeat: no-repeat; - background-position: center; - width: 40px; - background-repeat: no-repeat; - background-position: center; - background-size: 20px 20px; + margin-left: 10px; + background-color: #40414f; + color: #fff; + font-weight: bold; + cursor: pointer; + background-image: url("img/plane.png"); + background-repeat: no-repeat; + background-position: center; + width: 40px; + background-repeat: no-repeat; + background-position: center; + background-size: 20px 20px; } .chatui-send-btn:hover { - background-color: #03a33e; -} \ No newline at end of file + background-color: #03a33e; +} diff --git a/examples/simple-chat-ts/src/llm_chat.html b/examples/simple-chat-ts/src/llm_chat.html index a54fbc2b..774ffef9 100644 --- a/examples/simple-chat-ts/src/llm_chat.html +++ b/examples/simple-chat-ts/src/llm_chat.html @@ -1,15 +1,18 @@ - +
-
- -
-
+
+
+
- +
diff --git a/examples/simple-chat-upload/README.md b/examples/simple-chat-upload/README.md index 4b1e0d8f..336b7d49 100644 --- a/examples/simple-chat-upload/README.md +++ b/examples/simple-chat-upload/README.md @@ -19,15 +19,17 @@ Due to the differences in command-line tools between Unix/Linux and Windows syst ### Steps for Windows Users 1. **Create a Node.js Script File**: + - In the `examples\simple-chat` directory, create a file named `copy-config.js`. - Add the following code to handle file copying: ```javascript - const fs = require('fs'); + const fs = require("fs"); // Copy file - fs.copyFileSync('src/gh-config.js', 'src/app-config.js'); + fs.copyFileSync("src/gh-config.js", "src/app-config.js"); ``` 2. **Modify `package.json`**: + - In the `scripts` section of your `package.json`, replace Unix-style `cp` commands with our new Node.js script. For example: ```json "scripts": { diff --git a/examples/simple-chat-upload/package.json b/examples/simple-chat-upload/package.json index 52befc36..d78b9a9c 100644 --- a/examples/simple-chat-upload/package.json +++ b/examples/simple-chat-upload/package.json @@ -1,20 +1,20 @@ { - "name": "simple-chat", - "version": "0.1.0", - "private": true, - "scripts": { - "start": "cp src/gh-config.js src/app-config.js && parcel src/llm_chat.html --port 8883", - "build": "cp src/gh-config.js src/app-config.js && parcel build src/llm_chat.html --dist-dir lib --no-content-hash" - }, - "devDependencies": { - "buffer": "^5.7.1", - "parcel": "^2.8.3", - "process": "^0.11.10", - "tslib": "^2.3.1", - "typescript": "^4.9.5", - "url": "^0.11.3" - }, - "dependencies": { - "@mlc-ai/web-llm": "^0.2.31" - } -} \ No newline at end of file + "name": "simple-chat", + "version": "0.1.0", + "private": true, + "scripts": { + "start": "cp src/gh-config.js src/app-config.js && parcel src/llm_chat.html --port 8883", + "build": "cp src/gh-config.js src/app-config.js && parcel build src/llm_chat.html --dist-dir lib --no-content-hash" + }, + "devDependencies": { + "buffer": "^5.7.1", + "parcel": "^2.8.3", + "process": "^0.11.10", + "tslib": "^2.3.1", + "typescript": "^4.9.5", + "url": "^0.11.3" + }, + "dependencies": { + "@mlc-ai/web-llm": "^0.2.31" + } +} diff --git a/examples/simple-chat-upload/src/gh-config.js b/examples/simple-chat-upload/src/gh-config.js index b7bc6f36..733509aa 100644 --- a/examples/simple-chat-upload/src/gh-config.js +++ b/examples/simple-chat-upload/src/gh-config.js @@ -1,6 +1,6 @@ import { prebuiltAppConfig } from "@mlc-ai/web-llm"; export default { - "model_list": prebuiltAppConfig.model_list, - "use_web_worker": true -} + model_list: prebuiltAppConfig.model_list, + use_web_worker: true, +}; diff --git a/examples/simple-chat-upload/src/llm_chat.css b/examples/simple-chat-upload/src/llm_chat.css index 395714ed..79525baa 100644 --- a/examples/simple-chat-upload/src/llm_chat.css +++ b/examples/simple-chat-upload/src/llm_chat.css @@ -1,188 +1,188 @@ .chatui { - display: flex; - position: relative; - flex-flow: column wrap; - justify-content: space-between; - width: 100%; - max-width: 867px; - margin: 25px 10px; - height: 600px; - border: 2px solid #ddd; - border-radius: 5px; - background-color: #1F2027; + display: flex; + position: relative; + flex-flow: column wrap; + justify-content: space-between; + width: 100%; + max-width: 867px; + margin: 25px 10px; + height: 600px; + border: 2px solid #ddd; + border-radius: 5px; + background-color: #1f2027; } .chatui-select-wrapper { - display: flex; - justify-content: center; - background-color: #1F2027; - padding: 10px 0; + display: flex; + justify-content: center; + background-color: #1f2027; + padding: 10px 0; } #chatui-select { - width: 350px; - background-color: #1F2027; - color: white; - border: none; + width: 350px; + background-color: #1f2027; + color: white; + border: none; } #chatui-select:focus { - outline: none; + outline: none; } #chatui-select::-webkit-scrollbar { - display: none; + display: none; } #chatui-select option { - background-color: #1F2027; - color: white; + background-color: #1f2027; + color: white; } #chatui-select option:hover { - background-color: #474747; - color: white; + background-color: #474747; + color: white; } s .chatui-header { - display: flex; - justify-content: space-between; - padding: 10px; - border-bottom: 2px solid #ddd; - background: #eee; - color: #666; + display: flex; + justify-content: space-between; + padding: 10px; + border-bottom: 2px solid #ddd; + background: #eee; + color: #666; } /* Used to remove tiny white lines in android devices; not sure if there is a better way */ *, *::before, *::after { - box-sizing: content-box; + box-sizing: content-box; } .chatui-chat { - flex: 1; - overflow-y: auto; - padding: 10px; - background-color: #1F2027; + flex: 1; + overflow-y: auto; + padding: 10px; + background-color: #1f2027; } .chatui-chat::-webkit-scrollbar { - width: 6px; + width: 6px; } .chatui-chat::-webkit-scrollbar-track { - background: #1F2027; + background: #1f2027; } .chatui-chat::-webkit-scrollbar-thumb { - background: #888; + background: #888; } .chatui-chat::-webkit-scrollbar-thumb:hover { - background: #555; + background: #555; } .msg { - display: flex; - align-items: flex-end; - margin-bottom: 10px; + display: flex; + align-items: flex-end; + margin-bottom: 10px; } .msg:last-of-type { - margin: 0; + margin: 0; } .msg-bubble { - background-color: #f0f0f0; - border-radius: 8px; - padding: 16px; - margin: 5px auto; - width: calc(100% - 20px); - box-sizing: border-box; - color: black; - border: none; - font-size: medium; - margin-left: auto; - margin-right: auto; + background-color: #f0f0f0; + border-radius: 8px; + padding: 16px; + margin: 5px auto; + width: calc(100% - 20px); + box-sizing: border-box; + color: black; + border: none; + font-size: medium; + margin-left: auto; + margin-right: auto; } .left-msg .msg-bubble { - background-color: #343541; - color: #ececec; + background-color: #343541; + color: #ececec; } .error-msg .msg-bubble { - background-color: #343541; - color: #f15959; + background-color: #343541; + color: #f15959; } .init-msg .msg-bubble { - background-color: #343541; - color: #ececec; + background-color: #343541; + color: #ececec; } .right-msg .msg-bubble { - background-color: #444654; - color: #ececec; + background-color: #444654; + color: #ececec; } .chatui-inputarea { - display: flex; - padding: 10px; - border-top: 2px solid transparent; - background-color: #1F2027; + display: flex; + padding: 10px; + border-top: 2px solid transparent; + background-color: #1f2027; } .chatui-inputarea * { - padding: 10px; - border: none; - border-radius: 3px; - font-size: 1em; - color: white; - background: rgba(0, 0, 0, 0.3); + padding: 10px; + border: none; + border-radius: 3px; + font-size: 1em; + color: white; + background: rgba(0, 0, 0, 0.3); } .chatui-input { - flex: 1; - background-color: #40414F; - color: white; + flex: 1; + background-color: #40414f; + color: white; } .chatui-reset-btn { - margin-left: 10px; - background-color: #40414F; - color: #fff; - font-weight: bold; - cursor: pointer; - background-image: url('img/reset.png'); - background-repeat: no-repeat; - background-position: center; - width: 40px; - background-repeat: no-repeat; - background-position: center; - background-size: 20px 20px; + margin-left: 10px; + background-color: #40414f; + color: #fff; + font-weight: bold; + cursor: pointer; + background-image: url("img/reset.png"); + background-repeat: no-repeat; + background-position: center; + width: 40px; + background-repeat: no-repeat; + background-position: center; + background-size: 20px 20px; } .chatui-reset-btn:hover { - background-color: #03a33e; + background-color: #03a33e; } .chatui-send-btn { - margin-left: 10px; - background-color: #40414F; - color: #fff; - font-weight: bold; - cursor: pointer; - background-image: url('img/plane.png'); - background-repeat: no-repeat; - background-position: center; - width: 40px; - background-repeat: no-repeat; - background-position: center; - background-size: 20px 20px; + margin-left: 10px; + background-color: #40414f; + color: #fff; + font-weight: bold; + cursor: pointer; + background-image: url("img/plane.png"); + background-repeat: no-repeat; + background-position: center; + width: 40px; + background-repeat: no-repeat; + background-position: center; + background-size: 20px 20px; } .chatui-send-btn:hover { - background-color: #03a33e; -} \ No newline at end of file + background-color: #03a33e; +} diff --git a/examples/simple-chat-upload/src/llm_chat.html b/examples/simple-chat-upload/src/llm_chat.html index 1547dfb0..4e26567a 100644 --- a/examples/simple-chat-upload/src/llm_chat.html +++ b/examples/simple-chat-upload/src/llm_chat.html @@ -1,27 +1,32 @@ - +
-
- -
-
+
+
+
- /> - + />
- +
- - -
diff --git a/examples/simple-chat-upload/src/simple_chat.ts b/examples/simple-chat-upload/src/simple_chat.ts index 86889e58..8fdf1f9e 100644 --- a/examples/simple-chat-upload/src/simple_chat.ts +++ b/examples/simple-chat-upload/src/simple_chat.ts @@ -33,7 +33,9 @@ class ChatUI { // get the elements chatUI.uiChat = getElementAndCheck("chatui-chat"); chatUI.uiChatInput = getElementAndCheck("chatui-input") as HTMLInputElement; - chatUI.uiChatInfoLabel = getElementAndCheck("chatui-info-label") as HTMLLabelElement; + chatUI.uiChatInfoLabel = getElementAndCheck( + "chatui-info-label", + ) as HTMLLabelElement; // register event handlers getElementAndCheck("chatui-reset-btn").onclick = () => { chatUI.onReset(); @@ -52,8 +54,8 @@ class ChatUI { // phone) can only handle small models and make all other models unselectable. Otherwise, the // browser may crash. See https://github.com/mlc-ai/web-llm/issues/209. // Also use GPU vendor to decide whether it is a mobile device (hence with limited resources). - const androidMaxStorageBufferBindingSize = 1 << 27; // 128MB - const mobileVendors = new Set(["qualcomm", "arm"]) + const androidMaxStorageBufferBindingSize = 1 << 27; // 128MB + const mobileVendors = new Set(["qualcomm", "arm"]); let restrictModels = false; let maxStorageBufferBindingSize: number; let gpuVendor: string; @@ -67,24 +69,34 @@ class ChatUI { console.log(err.stack); return; } - if ((gpuVendor.length != 0 && mobileVendors.has(gpuVendor)) || - (maxStorageBufferBindingSize <= androidMaxStorageBufferBindingSize)) { - chatUI.appendMessage("init", "Your device seems to have " + - "limited resources, so we restrict the selectable models."); + if ( + (gpuVendor.length != 0 && mobileVendors.has(gpuVendor)) || + maxStorageBufferBindingSize <= androidMaxStorageBufferBindingSize + ) { + chatUI.appendMessage( + "init", + "Your device seems to have " + + "limited resources, so we restrict the selectable models.", + ); restrictModels = true; } // Populate modelSelector - const modelSelector = getElementAndCheck("chatui-select") as HTMLSelectElement; + const modelSelector = getElementAndCheck( + "chatui-select", + ) as HTMLSelectElement; for (let i = 0; i < chatUI.config.model_list.length; ++i) { const item = chatUI.config.model_list[i]; const opt = document.createElement("option"); opt.value = item.model_id; opt.innerHTML = item.model_id; - opt.selected = (i == 0); + opt.selected = i == 0; if ( - (restrictModels && (item.low_resource_required === undefined || !item.low_resource_required)) || - (item.buffer_size_required_bytes && maxStorageBufferBindingSize < item.buffer_size_required_bytes) + (restrictModels && + (item.low_resource_required === undefined || + !item.low_resource_required)) || + (item.buffer_size_required_bytes && + maxStorageBufferBindingSize < item.buffer_size_required_bytes) ) { // Either on a low-resource device and not a low-resource model // Or device's maxStorageBufferBindingSize does not satisfy the model's need (if specified) @@ -92,7 +104,11 @@ class ChatUI { opt.disabled = !params.has("bypassRestrictions"); opt.selected = false; } - if (!modelSelector.lastChild?.textContent?.startsWith(opt.value.split('-')[0])) { + if ( + !modelSelector.lastChild?.textContent?.startsWith( + opt.value.split("-")[0], + ) + ) { modelSelector.appendChild(document.createElement("hr")); } modelSelector.appendChild(opt); @@ -105,7 +121,7 @@ class ChatUI { }; return chatUI; - } + }; /** * Push a task to the execution queue. @@ -189,12 +205,12 @@ class ChatUI { const msgText = msg.getElementsByClassName("msg-text"); if (msgText.length != 1) throw Error("Expect msg-text"); if (msgText[0].innerHTML == text) return; - const list = text.split('\n').map((t) => { - const item = document.createElement('div'); + const list = text.split("\n").map((t) => { + const item = document.createElement("div"); item.textContent = t; return item; }); - msgText[0].innerHTML = ''; + msgText[0].innerHTML = ""; list.forEach((item) => msgText[0].append(item)); this.uiChat.scrollTo(0, this.uiChat.scrollHeight); } @@ -220,7 +236,7 @@ class ChatUI { this.appendMessage("init", ""); const initProgressCallback = (report) => { this.updateLastMessage("init", report.text); - } + }; this.engine.setInitProgressCallback(initProgressCallback); try { @@ -258,11 +274,14 @@ class ChatUI { this.uiChatInput.setAttribute("placeholder", "Generating..."); this.appendMessage("left", ""); - this.chatHistory.push({ "role": "user", "content": prompt }); + this.chatHistory.push({ role: "user", content: prompt }); try { let curMessage = ""; - const completion = await this.engine.chat.completions.create({ stream: true, messages: this.chatHistory }); + const completion = await this.engine.chat.completions.create({ + stream: true, + messages: this.chatHistory, + }); // TODO(Charlie): Processing of � requires changes for await (const chunk of completion) { const curDelta = chunk.choices[0].delta.content; @@ -273,8 +292,8 @@ class ChatUI { } this.uiChatInfoLabel.innerHTML = await this.engine.runtimeStatsText(); const finalMessage = await this.engine.getMessage(); - this.updateLastMessage("left", finalMessage); // TODO: Remove this after � issue is fixed - this.chatHistory.push({ "role": "assistant", "content": finalMessage }); + this.updateLastMessage("left", finalMessage); // TODO: Remove this after � issue is fixed + this.chatHistory.push({ role: "assistant", content: finalMessage }); } catch (err) { this.appendMessage("error", "Generate error, " + err.toString()); console.log(err.stack); @@ -290,22 +309,23 @@ let engine: webllm.MLCEngineInterface; // Here we do not use `CreateMLCEngine()` but instantiate an engine that is not loaded with model if (useWebWorker) { - engine = new webllm.WebWorkerMLCEngine(new Worker( - new URL('./worker.ts', import.meta.url), - { type: 'module' } - )); + engine = new webllm.WebWorkerMLCEngine( + new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }), + ); } else { engine = new webllm.MLCEngine(); } ChatUI.CreateAsync(engine); - -function getFileType(file: File){ - if (file.name.includes("wasm")){ +function getFileType(file: File) { + if (file.name.includes("wasm")) { return "webllm/wasm"; - } else if (file.name.includes(".bin") || file.name.includes("ndarray-cache.json")) { + } else if ( + file.name.includes(".bin") || + file.name.includes("ndarray-cache.json") + ) { return "webllm/model"; - } else if (file.name.includes("mlc-chat-config.json")){ + } else if (file.name.includes("mlc-chat-config.json")) { return "webllm/config"; } else { console.log("No model file suffix found"); @@ -313,13 +333,13 @@ function getFileType(file: File){ } } -async function uploadToIndexedDB(file: File){ +async function uploadToIndexedDB(file: File) { let db; const request = indexedDB.open(getFileType(file), 1); request.onupgradeneeded = (event) => { db = (event.target as IDBOpenDBRequest).result; - if (!db.objectStoreNames.contains('urls')) { - db.createObjectStore('urls', { keyPath: 'url' }); + if (!db.objectStoreNames.contains("urls")) { + db.createObjectStore("urls", { keyPath: "url" }); } }; request.onsuccess = (event) => { @@ -328,74 +348,74 @@ async function uploadToIndexedDB(file: File){ request.onerror = (event) => { console.error("Database error: ", (event.target as IDBOpenDBRequest).error); }; - const transaction = db.transaction('files', 'readwrite'); - const store = transaction.objectStore('files'); + const transaction = db.transaction("files", "readwrite"); + const store = transaction.objectStore("files"); const reader = new FileReader(); reader.onload = async (e) => { - if (e.target === null || e.target.result === null){ + if (e.target === null || e.target.result === null) { console.error("Do not read any files"); return; } - const url = file.name; + const url = file.name; store.add(e.target.result, url); }; - transaction.oncomplete = function() { - alert('All files have been uploaded to IndexedDB.'); + transaction.oncomplete = function () { + alert("All files have been uploaded to IndexedDB."); }; - transaction.onerror = function(event) { - console.error('Error uploading files:', event); + transaction.onerror = function (event) { + console.error("Error uploading files:", event); }; } -async function cacheFile(file:File, response:Response) { +async function cacheFile(file: File, response: Response) { try { - const cache = await caches.open(getFileType(file)); // Ensure getFileType is a synchronous function or awaited if async - console.log('Put response into cache:', response); - await cache.put(file.name, response); + const cache = await caches.open(getFileType(file)); // Ensure getFileType is a synchronous function or awaited if async + console.log("Put response into cache:", response); + await cache.put(file.name, response); } catch (error) { - console.error('Failed to cache the file:', error); + console.error("Failed to cache the file:", error); } } async function uploadFiles(): Promise { - const input = document.getElementById('file-input') as HTMLInputElement; - if (!input.files || input.files.length === 0) { - alert('No files selected.'); - return; + const input = document.getElementById("file-input") as HTMLInputElement; + if (!input.files || input.files.length === 0) { + alert("No files selected."); + return; + } + if (appConfig.useIndexedDBCache) { + for (const file of input.files) { + uploadToIndexedDB(file); } - if (appConfig.useIndexedDBCache){ - for (const file of input.files){ - uploadToIndexedDB(file); - } - } - else { - for (const file of input.files) { - const reader = new FileReader(); - reader.onload = async (e) => { - if (e.target === null || e.target.result === null){ - console.error("Do not read any files"); - return; - } - const arrayBuffer = e.target.result as ArrayBuffer; - const response = new Response(arrayBuffer, { - status: 200, - statusText: 'OK', - headers: { - 'Content-Type': 'application/octet-stream', - 'Content-Length': arrayBuffer.byteLength.toString(), - }, - - }); - await cacheFile(file, response); - }; - if (file.name.includes("mlc-chat-config.json") || file.name.includes("ndarray-cache.json")){ - reader.readAsText(file); - } else { - reader.readAsArrayBuffer(file); + } else { + for (const file of input.files) { + const reader = new FileReader(); + reader.onload = async (e) => { + if (e.target === null || e.target.result === null) { + console.error("Do not read any files"); + return; } + const arrayBuffer = e.target.result as ArrayBuffer; + const response = new Response(arrayBuffer, { + status: 200, + statusText: "OK", + headers: { + "Content-Type": "application/octet-stream", + "Content-Length": arrayBuffer.byteLength.toString(), + }, + }); + await cacheFile(file, response); + }; + if ( + file.name.includes("mlc-chat-config.json") || + file.name.includes("ndarray-cache.json") + ) { + reader.readAsText(file); + } else { + reader.readAsArrayBuffer(file); } } - + } } (window as any).uploadFiles = uploadFiles; diff --git a/examples/streaming/src/streaming.html b/examples/streaming/src/streaming.html index 2e7d85a4..cc870e56 100644 --- a/examples/streaming/src/streaming.html +++ b/examples/streaming/src/streaming.html @@ -1,17 +1,17 @@ - + - + - -

WebLLM Test Page

- Open console to see output -
-
- -

Response

- - - - \ No newline at end of file + +

WebLLM Test Page

+ Open console to see output +
+
+ +

Response

+ + + + diff --git a/package-lock.json b/package-lock.json index 439c3faa..1b257f24 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ }, "devDependencies": { "@mlc-ai/web-tokenizers": "^0.1.3", + "@next/eslint-plugin-next": "^14.2.3", "@rollup/plugin-commonjs": "^20.0.0", "@rollup/plugin-node-resolve": "^13.0.4", "@types/chrome": "^0.0.266", @@ -820,6 +821,102 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -1260,6 +1357,61 @@ "integrity": "sha512-WSt3TvycV8MfRvOh6v6bfY7rRHIOebOUVOaqpCCZjCPlsEaiM/gcN850ASd2XYirSlAY0EwBTM0TtkZ0YPD/BQ==", "dev": true }, + "node_modules/@next/eslint-plugin-next": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-14.2.3.tgz", + "integrity": "sha512-L3oDricIIjgj1AVnRdRor21gI7mShlSwU/1ZGHmqM3LzHhXXhdkrfeNY5zif25Bi5Dd7fiJHsbhoZCHfXYvlAw==", + "dev": true, + "dependencies": { + "glob": "10.3.10" + } + }, + "node_modules/@next/eslint-plugin-next/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@next/eslint-plugin-next/node_modules/glob": { + "version": "10.3.10", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz", + "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==", + "dev": true, + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.5", + "minimatch": "^9.0.1", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0", + "path-scurry": "^1.10.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@next/eslint-plugin-next/node_modules/minimatch": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", + "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1295,6 +1447,16 @@ "node": ">= 8" } }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@pkgr/core": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.1.tgz", @@ -2809,6 +2971,12 @@ "node": ">=8" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true + }, "node_modules/electron-to-chromium": { "version": "1.4.653", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.653.tgz", @@ -3499,6 +3667,34 @@ "node": ">=0.10.0" } }, + "node_modules/foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/foreground-child/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/form-data": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", @@ -4334,6 +4530,24 @@ "node": ">=8" } }, + "node_modules/jackspeak": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "dev": true, + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", @@ -6309,6 +6523,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "dev": true, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/mixin-deep": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", @@ -6697,6 +6920,31 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", + "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==", + "dev": true, + "engines": { + "node": "14 || >=16.14" + } + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -8064,6 +8312,21 @@ "node": ">=8" } }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -8076,6 +8339,19 @@ "node": ">=8" } }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-bom": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", @@ -8809,6 +9085,24 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/package.json b/package.json index 1c3b9f84..a10a769c 100644 --- a/package.json +++ b/package.json @@ -7,9 +7,9 @@ "type": "module", "scripts": { "build": "rollup -c && ./cleanup-index-js.sh", - "lint": "npx eslint ./src/ && npx prettier ./src --check", + "lint": "npx eslint ./src/ ./tests/ ./examples/ && npx prettier ./src/ ./tests/ ./examples/ --check", "test": "yarn jest", - "format": "prettier --write \"./src/\"", + "format": "prettier --write \"./src/\" \"./examples/\" \"./tests/\"", "prepare": "husky" }, "files": [ @@ -28,6 +28,7 @@ "homepage": "https://github.com/mlc-ai/web-llm", "devDependencies": { "@mlc-ai/web-tokenizers": "^0.1.3", + "@next/eslint-plugin-next": "^14.2.3", "@rollup/plugin-commonjs": "^20.0.0", "@rollup/plugin-node-resolve": "^13.0.4", "@types/chrome": "^0.0.266", diff --git a/src/extension_service_worker.ts b/src/extension_service_worker.ts index 1c575253..4e8279d8 100644 --- a/src/extension_service_worker.ts +++ b/src/extension_service_worker.ts @@ -2,7 +2,7 @@ import * as tvmjs from "tvmjs"; import log from "loglevel"; import { AppConfig, ChatOptions, MLCEngineConfig } from "./config"; import { ReloadParams, WorkerRequest } from "./message"; -import { LogLevel, MLCEngineInterface } from "./types"; +import { MLCEngineInterface } from "./types"; import { ChatWorker, MLCEngineWorkerHandler, diff --git a/src/service_worker.ts b/src/service_worker.ts index eba3e564..dc9c6639 100644 --- a/src/service_worker.ts +++ b/src/service_worker.ts @@ -2,7 +2,7 @@ import * as tvmjs from "tvmjs"; import log from "loglevel"; import { AppConfig, ChatOptions, MLCEngineConfig } from "./config"; import { ReloadParams, WorkerRequest, WorkerResponse } from "./message"; -import { MLCEngineInterface, InitProgressReport, LogLevel } from "./types"; +import { MLCEngineInterface, InitProgressReport } from "./types"; import { MLCEngineWorkerHandler, WebWorkerMLCEngine, diff --git a/tests/conv_template.test.ts b/tests/conv_template.test.ts index ec16e8e8..007bf722 100644 --- a/tests/conv_template.test.ts +++ b/tests/conv_template.test.ts @@ -1,113 +1,116 @@ -import { ChatConfig, ConvTemplateConfig, Role } from '../src/config' -import { getConversation } from '../src/conversation' -import { MLCEngine } from '../src/engine' -import { ChatCompletionRequest } from "../src/openai_api_protocols/chat_completion" +import { ChatConfig, Role } from "../src/config"; +import { getConversation } from "../src/conversation"; +import { describe, expect, test } from "@jest/globals"; +describe("Test conversation template", () => { + test("Test from json", () => { + const config_str = + "{" + + ' "model_type": "llama",' + + ' "quantization": "q4f16_1",' + + ' "model_config": {' + + ' "hidden_size": 4096,' + + ' "intermediate_size": 11008,' + + ' "num_attention_heads": 32,' + + ' "num_hidden_layers": 32,' + + ' "rms_norm_eps": 1e-05,' + + ' "vocab_size": 32000,' + + ' "position_embedding_base": 10000,' + + ' "context_window_size": 4096,' + + ' "prefill_chunk_size": 4096,' + + ' "num_key_value_heads": 32,' + + ' "head_dim": 128,' + + ' "tensor_parallel_shards": 1,' + + ' "max_batch_size": 80' + + " }," + + ' "vocab_size": 32000,' + + ' "context_window_size": 4096,' + + ' "sliding_window_size": -1,' + + ' "prefill_chunk_size": 4096,' + + ' "attention_sink_size": -1,' + + ' "tensor_parallel_shards": 1,' + + ' "mean_gen_len": 128,' + + ' "max_gen_len": 512,' + + ' "shift_fill_factor": 0.3,' + + ' "temperature": 0.6,' + + ' "presence_penalty": 0.0,' + + ' "frequency_penalty": 0.0,' + + ' "repetition_penalty": 1.0,' + + ' "top_p": 0.9,' + + ' "conv_template": {' + + ' "name": "llama-2",' + + ' "system_template": "[INST] <>\\n{system_message}\\n<>\\n\\n",' + + ' "system_message": "You are a helpful, respectful and honest assistant.",' + + ' "system_prefix_token_ids": [' + + " 1" + + " ]," + + ' "add_role_after_system_message": false,' + + ' "roles": {' + + ' "user": "[INST]",' + + ' "assistant": "[/INST]",' + + ' "tool": "[INST]"' + + " }," + + ' "role_templates": {' + + ' "user": "{user_message}",' + + ' "assistant": "{assistant_message}",' + + ' "tool": "{tool_message}"' + + " }," + + ' "messages": [],' + + ' "seps": [' + + ' " "' + + " ]," + + ' "role_content_sep": " ",' + + ' "role_empty_sep": " ",' + + ' "stop_str": [' + + ' "[INST]"' + + " ]," + + ' "stop_token_ids": [' + + " 2" + + " ]," + + ' "function_string": "",' + + ' "use_function_calling": false' + + " }," + + ' "pad_token_id": 0,' + + ' "bos_token_id": 1,' + + ' "eos_token_id": 2,' + + ' "tokenizer_files": [' + + ' "tokenizer.model",' + + ' "tokenizer.json",' + + ' "tokenizer_config.json"' + + " ]," + + ' "version": "0.1.0"' + + "}"; + const config_json = JSON.parse(config_str); + const config = { ...config_json } as ChatConfig; + const conversation = getConversation(config.conv_template); + const config_obj = conversation.config; -import { describe, expect, test } from '@jest/globals'; + expect(config_obj.system_template).toEqual( + "[INST] <>\n{system_message}\n<>\n\n", + ); + expect(config_obj.system_message).toEqual( + "You are a helpful, respectful and honest assistant.", + ); + expect(config_obj.roles.user).toEqual("[INST]"); + expect(config_obj.roles.assistant).toEqual("[/INST]"); + expect(config_obj.role_templates?.user).toEqual("{user_message}"); + expect(config_obj.role_templates?.assistant).toEqual("{assistant_message}"); + expect(config_obj.role_content_sep).toEqual(" "); + expect(config_obj.role_empty_sep).toEqual(" "); + expect(config_obj.seps).toEqual([" "]); + expect(config_obj.stop_str).toEqual(["[INST]"]); + expect(config_obj.stop_token_ids).toEqual([2]); + expect(config_obj.system_prefix_token_ids).toEqual([1]); + expect(config_obj.add_role_after_system_message).toBe(false); -describe('Test conversation template', () => { - test('Test from json', () => { - const config_str = "{" + - " \"model_type\": \"llama\"," + - " \"quantization\": \"q4f16_1\"," + - " \"model_config\": {" + - " \"hidden_size\": 4096," + - " \"intermediate_size\": 11008," + - " \"num_attention_heads\": 32," + - " \"num_hidden_layers\": 32," + - " \"rms_norm_eps\": 1e-05," + - " \"vocab_size\": 32000," + - " \"position_embedding_base\": 10000," + - " \"context_window_size\": 4096," + - " \"prefill_chunk_size\": 4096," + - " \"num_key_value_heads\": 32," + - " \"head_dim\": 128," + - " \"tensor_parallel_shards\": 1," + - " \"max_batch_size\": 80" + - " }," + - " \"vocab_size\": 32000," + - " \"context_window_size\": 4096," + - " \"sliding_window_size\": -1," + - " \"prefill_chunk_size\": 4096," + - " \"attention_sink_size\": -1," + - " \"tensor_parallel_shards\": 1," + - " \"mean_gen_len\": 128," + - " \"max_gen_len\": 512," + - " \"shift_fill_factor\": 0.3," + - " \"temperature\": 0.6," + - " \"presence_penalty\": 0.0," + - " \"frequency_penalty\": 0.0," + - " \"repetition_penalty\": 1.0," + - " \"top_p\": 0.9," + - " \"conv_template\": {" + - " \"name\": \"llama-2\"," + - " \"system_template\": \"[INST] <>\\n{system_message}\\n<>\\n\\n\"," + - " \"system_message\": \"You are a helpful, respectful and honest assistant.\"," + - " \"system_prefix_token_ids\": [" + - " 1" + - " ]," + - " \"add_role_after_system_message\": false," + - " \"roles\": {" + - " \"user\": \"[INST]\"," + - " \"assistant\": \"[/INST]\"," + - " \"tool\": \"[INST]\"" + - " }," + - " \"role_templates\": {" + - " \"user\": \"{user_message}\"," + - " \"assistant\": \"{assistant_message}\"," + - " \"tool\": \"{tool_message}\"" + - " }," + - " \"messages\": []," + - " \"seps\": [" + - " \" \"" + - " ]," + - " \"role_content_sep\": \" \"," + - " \"role_empty_sep\": \" \"," + - " \"stop_str\": [" + - " \"[INST]\"" + - " ]," + - " \"stop_token_ids\": [" + - " 2" + - " ]," + - " \"function_string\": \"\"," + - " \"use_function_calling\": false" + - " }," + - " \"pad_token_id\": 0," + - " \"bos_token_id\": 1," + - " \"eos_token_id\": 2," + - " \"tokenizer_files\": [" + - " \"tokenizer.model\"," + - " \"tokenizer.json\"," + - " \"tokenizer_config.json\"" + - " ]," + - " \"version\": \"0.1.0\"" + - "}"; - const config_json = JSON.parse(config_str); - const config = { ...config_json } as ChatConfig; - const conversation = getConversation(config.conv_template) - const config_obj = conversation.config; - - expect(config_obj.system_template).toEqual("[INST] <>\n{system_message}\n<>\n\n"); - expect(config_obj.system_message).toEqual("You are a helpful, respectful and honest assistant."); - expect(config_obj.roles.user).toEqual("[INST]"); - expect(config_obj.roles.assistant).toEqual("[/INST]"); - expect(config_obj.role_templates?.user).toEqual("{user_message}"); - expect(config_obj.role_templates?.assistant).toEqual("{assistant_message}"); - expect(config_obj.role_content_sep).toEqual(" "); - expect(config_obj.role_empty_sep).toEqual(" "); - expect(config_obj.seps).toEqual([" "]); - expect(config_obj.stop_str).toEqual(["[INST]"]); - expect(config_obj.stop_token_ids).toEqual([2]); - expect(config_obj.system_prefix_token_ids).toEqual([1]); - expect(config_obj.add_role_after_system_message).toBe(false); - - conversation.appendMessage(Role.user, "test1"); - conversation.appendMessage(Role.assistant, "test2"); - conversation.appendMessage(Role.user, "test3"); - conversation.appendReplyHeader(Role.assistant); - const prompt = conversation.getPromptArray().join(""); - expect(prompt).toEqual("[INST] <>\nYou are a helpful, respectful and honest assistant.\n<>\n\n test1 [/INST] test2 [INST] test3 [/INST] ") - console.log(prompt) - }); -}) \ No newline at end of file + conversation.appendMessage(Role.user, "test1"); + conversation.appendMessage(Role.assistant, "test2"); + conversation.appendMessage(Role.user, "test3"); + conversation.appendReplyHeader(Role.assistant); + const prompt = conversation.getPromptArray().join(""); + expect(prompt).toEqual( + "[INST] <>\nYou are a helpful, respectful and honest assistant.\n<>\n\n test1 [/INST] test2 [INST] test3 [/INST] ", + ); + console.log(prompt); + }); +}); diff --git a/tests/function_calling.test.ts b/tests/function_calling.test.ts index 6658b308..66a0e7e0 100644 --- a/tests/function_calling.test.ts +++ b/tests/function_calling.test.ts @@ -1,108 +1,195 @@ -import { Role } from '../src/config' -import { getConversation } from '../src/conversation' -import { MLCEngine } from '../src/engine' -import { ChatCompletionRequest } from "../src/openai_api_protocols/chat_completion" +import { Role } from "../src/config"; +import { getConversation } from "../src/conversation"; +import { MLCEngine } from "../src/engine"; +import { ChatCompletionRequest } from "../src/openai_api_protocols/chat_completion"; +import { describe, expect, test } from "@jest/globals"; -import { describe, expect, test } from '@jest/globals'; +describe("Test conversation template", () => { + test("Test getPromptArrayInternal", () => { + const conv = getConversation("gorilla"); + conv.appendMessage( + Role.user, + 'Call me an Uber ride type "Plus" in Berkeley at zipcode 94704 in 10 minutes', + "Tom", + ); + const prompt_array = conv.getPromptArray(); -describe('Test conversation template', () => { - test('Test getPromptArrayInternal', () => { - const conv = getConversation("gorilla"); - conv.appendMessage(Role.user, "Call me an Uber ride type \"Plus\" in Berkeley at zipcode 94704 in 10 minutes", "Tom"); - const prompt_array = conv.getPromptArray(); + expect(prompt_array).toEqual([ + "A chat between a curious user and an artificial intelligence assistant. " + + "The assistant gives helpful, detailed, and polite answers to the user's questions.\n", + 'Tom: <> Call me an Uber ride type "Plus" in Berkeley at zipcode 94704 in 10 minutes <> \n', + ]); + }); - expect(prompt_array).toEqual([ - "A chat between a curious user and an artificial intelligence assistant. " + - "The assistant gives helpful, detailed, and polite answers to the user's questions.\n", - "Tom: <> Call me an Uber ride type \"Plus\" in Berkeley at zipcode 94704 in 10 minutes <> \n"]); - }); + test("Test getPromptArrayInternal function call", () => { + const conv = getConversation("gorilla"); + conv.appendMessage( + Role.user, + 'Call me an Uber ride type "Plus" in Berkeley at zipcode 94704 in 10 minutes', + ); + conv.use_function_calling = true; + conv.function_string = JSON.stringify([ + { + name: "Uber Carpool", + api_name: "uber.ride", + description: + "Find suitable ride for customers given the location, type of ride, and the amount of time the customer is willing to wait as parameters", + parameters: [ + { + name: "loc", + description: "Location of the starting place of the Uber ride", + }, + { + name: "type", + enum: ["plus", "comfort", "black"], + description: "Types of Uber ride user is ordering", + }, + { + name: "time", + description: + "The amount of time in minutes the customer is willing to wait", + }, + ], + }, + ]); + const prompt_array = conv.getPromptArray(); - test('Test getPromptArrayInternal function call', () => { - const conv = getConversation("gorilla"); - conv.appendMessage(Role.user, "Call me an Uber ride type \"Plus\" in Berkeley at zipcode 94704 in 10 minutes"); - conv.use_function_calling = true; - conv.function_string = JSON.stringify([{ - "name": "Uber Carpool", - "api_name": "uber.ride", - "description": "Find suitable ride for customers given the location, type of ride, and the amount of time the customer is willing to wait as parameters", - "parameters": [ - { "name": "loc", "description": "Location of the starting place of the Uber ride" }, - { "name": "type", "enum": ["plus", "comfort", "black"], "description": "Types of Uber ride user is ordering" }, - { "name": "time", "description": "The amount of time in minutes the customer is willing to wait" } - ] - }]); - const prompt_array = conv.getPromptArray(); - - expect(prompt_array).toEqual([ - "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\n", - "USER: <> Call me an Uber ride type \"Plus\" in Berkeley at zipcode 94704 in 10 minutes <> [{\"name\":\"Uber Carpool\",\"api_name\":\"uber.ride\",\"description\":\"Find suitable ride for customers given the location, type of ride, and the amount of time the customer is willing to wait as parameters\",\"parameters\":[{\"name\":\"loc\",\"description\":\"Location of the starting place of the Uber ride\"},{\"name\":\"type\",\"enum\":[\"plus\",\"comfort\",\"black\"],\"description\":\"Types of Uber ride user is ordering\"},{\"name\":\"time\",\"description\":\"The amount of time in minutes the customer is willing to wait\"}]}]\n" - ]); - }) + expect(prompt_array).toEqual([ + "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\n", + 'USER: <> Call me an Uber ride type "Plus" in Berkeley at zipcode 94704 in 10 minutes <> [{"name":"Uber Carpool","api_name":"uber.ride","description":"Find suitable ride for customers given the location, type of ride, and the amount of time the customer is willing to wait as parameters","parameters":[{"name":"loc","description":"Location of the starting place of the Uber ride"},{"name":"type","enum":["plus","comfort","black"],"description":"Types of Uber ride user is ordering"},{"name":"time","description":"The amount of time in minutes the customer is willing to wait"}]}]\n', + ]); + }); }); -describe('Test MLCEngine', () => { - test('Test getFunctionCallUsage none', () => { - const engine = new MLCEngine(); - - const request: ChatCompletionRequest = { - model: "gorilla-openfunctions-v1-q4f16_1_MLC", - messages: [ - { role: "system", content: "You are a helpful assistant." }, - { role: "user", content: "Hello!" }, - ], - tool_choice: 'none', - tools: [ - { type: 'function', function: { description: 'A', name: 'fn_A', parameters: { foo: 'bar' } } }, - { type: 'function', function: { description: 'B', name: 'fn_B', parameters: { foo: 'bar' } } }, - { type: 'function', function: { description: 'C', name: 'fn_C', parameters: { foo: 'bar' } } }, - ], - }; +describe("Test MLCEngine", () => { + test("Test getFunctionCallUsage none", () => { + const engine = new MLCEngine(); - expect((engine as any).getFunctionCallUsage(request)).toEqual(""); - }); + const request: ChatCompletionRequest = { + model: "gorilla-openfunctions-v1-q4f16_1_MLC", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Hello!" }, + ], + tool_choice: "none", + tools: [ + { + type: "function", + function: { + description: "A", + name: "fn_A", + parameters: { foo: "bar" }, + }, + }, + { + type: "function", + function: { + description: "B", + name: "fn_B", + parameters: { foo: "bar" }, + }, + }, + { + type: "function", + function: { + description: "C", + name: "fn_C", + parameters: { foo: "bar" }, + }, + }, + ], + }; - test('Test getFunctionCallUsage auto', () => { - const engine = new MLCEngine(); + expect((engine as any).getFunctionCallUsage(request)).toEqual(""); + }); - const request: ChatCompletionRequest = { - model: "gorilla-openfunctions-v1-q4f16_1_MLC", - messages: [ - { role: "system", content: "You are a helpful assistant." }, - { role: "user", content: "Hello!" }, - ], - tool_choice: 'auto', - tools: [ - { type: 'function', function: { description: 'A', name: 'fn_A', parameters: { foo: 'bar' } } }, - { type: 'function', function: { description: 'B', name: 'fn_B', parameters: { foo: 'bar' } } }, - { type: 'function', function: { description: 'C', name: 'fn_C', parameters: { foo: 'bar' } } }, - ], - }; - expect((engine as any).getFunctionCallUsage(request)).toEqual("[{\"description\":\"A\",\"name\":\"fn_A\",\"parameters\":{\"foo\":\"bar\"}},{\"description\":\"B\",\"name\":\"fn_B\",\"parameters\":{\"foo\":\"bar\"}},{\"description\":\"C\",\"name\":\"fn_C\",\"parameters\":{\"foo\":\"bar\"}}]"); - }); + test("Test getFunctionCallUsage auto", () => { + const engine = new MLCEngine(); - test('Test getFunctionCallUsage function', () => { - const engine = new MLCEngine(); + const request: ChatCompletionRequest = { + model: "gorilla-openfunctions-v1-q4f16_1_MLC", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Hello!" }, + ], + tool_choice: "auto", + tools: [ + { + type: "function", + function: { + description: "A", + name: "fn_A", + parameters: { foo: "bar" }, + }, + }, + { + type: "function", + function: { + description: "B", + name: "fn_B", + parameters: { foo: "bar" }, + }, + }, + { + type: "function", + function: { + description: "C", + name: "fn_C", + parameters: { foo: "bar" }, + }, + }, + ], + }; + expect((engine as any).getFunctionCallUsage(request)).toEqual( + '[{"description":"A","name":"fn_A","parameters":{"foo":"bar"}},{"description":"B","name":"fn_B","parameters":{"foo":"bar"}},{"description":"C","name":"fn_C","parameters":{"foo":"bar"}}]', + ); + }); - const request: ChatCompletionRequest = { - model: "gorilla-openfunctions-v1-q4f16_1_MLC", - messages: [ - { role: "system", content: "You are a helpful assistant." }, - { role: "user", content: "Hello!" }, - ], - tool_choice: { - type: 'function', - function: { - name: 'fn_B', - }, - }, - tools: [ - { type: 'function', function: { description: 'A', name: 'fn_A', parameters: { foo: 'bar' } } }, - { type: 'function', function: { description: 'B', name: 'fn_B', parameters: { foo: 'bar' } } }, - { type: 'function', function: { description: 'C', name: 'fn_C', parameters: { foo: 'bar' } } }, - ], - }; - expect((engine as any).getFunctionCallUsage(request)).toEqual("[{\"description\":\"B\",\"name\":\"fn_B\",\"parameters\":{\"foo\":\"bar\"}}]"); + test("Test getFunctionCallUsage function", () => { + const engine = new MLCEngine(); - }); + const request: ChatCompletionRequest = { + model: "gorilla-openfunctions-v1-q4f16_1_MLC", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Hello!" }, + ], + tool_choice: { + type: "function", + function: { + name: "fn_B", + }, + }, + tools: [ + { + type: "function", + function: { + description: "A", + name: "fn_A", + parameters: { foo: "bar" }, + }, + }, + { + type: "function", + function: { + description: "B", + name: "fn_B", + parameters: { foo: "bar" }, + }, + }, + { + type: "function", + function: { + description: "C", + name: "fn_C", + parameters: { foo: "bar" }, + }, + }, + ], + }; + expect((engine as any).getFunctionCallUsage(request)).toEqual( + '[{"description":"B","name":"fn_B","parameters":{"foo":"bar"}}]', + ); + }); }); diff --git a/tests/generation_config.test.ts b/tests/generation_config.test.ts index 7dfafcbf..8f672840 100644 --- a/tests/generation_config.test.ts +++ b/tests/generation_config.test.ts @@ -1,99 +1,99 @@ -import { GenerationConfig, postInitAndCheckGenerationConfigValues } from "../src/config" -import { describe, expect, test } from '@jest/globals'; +import { + GenerationConfig, + postInitAndCheckGenerationConfigValues, +} from "../src/config"; +import { describe, expect, test } from "@jest/globals"; -describe('Check generation config illegal values', () => { - test('High-level unsupported fields', () => { - expect(() => { - const genConfig: GenerationConfig = { - max_gen_len: 0, - } - postInitAndCheckGenerationConfigValues(genConfig) - }).toThrow("`max_gen_len` should be greater than zero."); - }); +describe("Check generation config illegal values", () => { + test("High-level unsupported fields", () => { + expect(() => { + const genConfig: GenerationConfig = { + max_gen_len: 0, + }; + postInitAndCheckGenerationConfigValues(genConfig); + }).toThrow("`max_gen_len` should be greater than zero."); + }); - test('logit_bias exceeds range', () => { - expect(() => { - const genConfig: GenerationConfig = { - max_gen_len: 10, - logit_bias: { - "1355": 155 - } - }; - postInitAndCheckGenerationConfigValues(genConfig) - }).toThrow("logit_bias should be in range [-100, 100];"); - }); + test("logit_bias exceeds range", () => { + expect(() => { + const genConfig: GenerationConfig = { + max_gen_len: 10, + logit_bias: { + "1355": 155, + }, + }; + postInitAndCheckGenerationConfigValues(genConfig); + }).toThrow("logit_bias should be in range [-100, 100];"); + }); - test('logit_bias invalid key', () => { - expect(() => { - const genConfig: GenerationConfig = { - max_gen_len: 10, - logit_bias: { - "thisRaisesError": 50 - } - }; - postInitAndCheckGenerationConfigValues(genConfig) - }).toThrow("Expect logit_bias's keys to be number represented in string"); - }); + test("logit_bias invalid key", () => { + expect(() => { + const genConfig: GenerationConfig = { + max_gen_len: 10, + logit_bias: { + thisRaisesError: 50, + }, + }; + postInitAndCheckGenerationConfigValues(genConfig); + }).toThrow("Expect logit_bias's keys to be number represented in string"); + }); - test('top_logprobs out of range', () => { - expect(() => { - const genConfig: GenerationConfig = { - logprobs: true, - top_logprobs: 6, - max_gen_len: 10 - }; - postInitAndCheckGenerationConfigValues(genConfig) - }).toThrow("`top_logprobs` should be in range [0,5]"); - }); + test("top_logprobs out of range", () => { + expect(() => { + const genConfig: GenerationConfig = { + logprobs: true, + top_logprobs: 6, + max_gen_len: 10, + }; + postInitAndCheckGenerationConfigValues(genConfig); + }).toThrow("`top_logprobs` should be in range [0,5]"); + }); - test('top_logprobs set without setting logprobs', () => { - expect(() => { - const genConfig: GenerationConfig = { - top_logprobs: 3, - max_gen_len: 10 - }; - postInitAndCheckGenerationConfigValues(genConfig) - }).toThrow("`logprobs` must be true if `top_logprobs` is set"); - }); + test("top_logprobs set without setting logprobs", () => { + expect(() => { + const genConfig: GenerationConfig = { + top_logprobs: 3, + max_gen_len: 10, + }; + postInitAndCheckGenerationConfigValues(genConfig); + }).toThrow("`logprobs` must be true if `top_logprobs` is set"); + }); - test('top_logprobs set though logprobs is false', () => { - expect(() => { - const genConfig: GenerationConfig = { - logprobs: false, - top_logprobs: 3, - max_gen_len: 10 - }; - postInitAndCheckGenerationConfigValues(genConfig) - }).toThrow("`logprobs` must be true if `top_logprobs` is set"); - }); + test("top_logprobs set though logprobs is false", () => { + expect(() => { + const genConfig: GenerationConfig = { + logprobs: false, + top_logprobs: 3, + max_gen_len: 10, + }; + postInitAndCheckGenerationConfigValues(genConfig); + }).toThrow("`logprobs` must be true if `top_logprobs` is set"); + }); }); -describe('Check generation post init', () => { - test('Only set one of presence or frequency penalty', () => { +describe("Check generation post init", () => { + test("Only set one of presence or frequency penalty", () => { + const genConfig: GenerationConfig = { + frequency_penalty: 1.5, + }; + postInitAndCheckGenerationConfigValues(genConfig); + expect(genConfig.presence_penalty).toBe(0.0); + }); - const genConfig: GenerationConfig = { - frequency_penalty: 1.5, - }; - postInitAndCheckGenerationConfigValues(genConfig); - expect(genConfig.presence_penalty).toBe(0.0); - }); + test("Set logprobs without setting top_logprobs", () => { + const genConfig: GenerationConfig = { + logprobs: true, + }; + postInitAndCheckGenerationConfigValues(genConfig); + expect(genConfig.top_logprobs).toBe(0); + }); - test('Set logprobs without setting top_logprobs', () => { - - const genConfig: GenerationConfig = { - logprobs: true, - }; - postInitAndCheckGenerationConfigValues(genConfig); - expect(genConfig.top_logprobs).toBe(0); - }); - - test('Set both logprobs and top_logprobs', () => { - - const genConfig: GenerationConfig = { - logprobs: true, - top_logprobs: 2, - }; - postInitAndCheckGenerationConfigValues(genConfig); - expect(genConfig.top_logprobs).toBe(2); - }); + test("Set both logprobs and top_logprobs", () => { + const genConfig: GenerationConfig = { + logprobs: true, + top_logprobs: 2, + }; + postInitAndCheckGenerationConfigValues(genConfig); + expect(genConfig.top_logprobs).toBe(2); + }); }); diff --git a/tests/multi_round_chat.test.ts b/tests/multi_round_chat.test.ts index 840e415c..51ad8aff 100644 --- a/tests/multi_round_chat.test.ts +++ b/tests/multi_round_chat.test.ts @@ -1,223 +1,234 @@ - -import { describe, expect, test } from '@jest/globals'; +import { describe, expect, test } from "@jest/globals"; import { - ChatCompletionMessageParam, - ChatCompletionRequest, - ChatCompletionUserMessageParam, + ChatCompletionMessageParam, + ChatCompletionRequest, + ChatCompletionUserMessageParam, } from "../src/openai_api_protocols/chat_completion"; -import { MLCEngine } from '../src/engine'; -import { Conversation, compareConversationObject } from '../src/conversation'; -import { ChatConfig, Role } from '../src/config'; - -const configStr = "{" + - " \"conv_template\": {" + - " \"name\": \"llama-2\"," + - " \"system_template\": \"[INST] <>\\n{system_message}\\n<>\\n\\n\"," + - " \"system_message\": \"You are a helpful, respectful and honest assistant.\"," + - " \"system_prefix_token_ids\": [" + - " 1" + - " ]," + - " \"add_role_after_system_message\": false," + - " \"roles\": {" + - " \"user\": \"[INST]\"," + - " \"assistant\": \"[/INST]\"," + - " \"tool\": \"[INST]\"" + - " }," + - " \"role_templates\": {" + - " \"user\": \"{user_message}\"," + - " \"assistant\": \"{assistant_message}\"," + - " \"tool\": \"{tool_message}\"" + - " }," + - " \"messages\": []," + - " \"seps\": [" + - " \" \"" + - " ]," + - " \"role_content_sep\": \" \"," + - " \"role_empty_sep\": \" \"," + - " \"stop_str\": [" + - " \"[INST]\"" + - " ]," + - " \"stop_token_ids\": [" + - " 2" + - " ]," + - " \"function_string\": \"\"," + - " \"use_function_calling\": false" + - " }" + - "}"; - -describe('Test multi-round chatting', () => { - test('Test is multi-round', () => { - // Setups - const config_json = JSON.parse(configStr); - const chatConfig = { ...config_json } as ChatConfig; - const engine = new MLCEngine(); - - // Simulate request0 - const messages: ChatCompletionMessageParam[] = [ - { - "role": "system", - "content": "[INST] <>\n\nYou are a helpful, respectful and honest assistant. " + - "Be as happy as you can when speaking please.\n<>\n\n " - }, - { "role": "user", "content": "Provide me three US states." }, - ]; - const request0: ChatCompletionRequest = { - messages: messages, - }; - - // Simulate processing of request0, appending response to convA (done by LLMChatPipeline) - const conv0: Conversation = - ((engine as any).getConversationFromChatCompletionRequest(request0, chatConfig)); - conv0.appendMessage(Role.user, "Provide me three US states."); - const reply0 = "California, New York, Nevada."; - conv0.appendMessage(Role.assistant, reply0); // simulated response - - // Simulate request1, where user maintain the chat history, appending the resposne - const newMessages = [...messages]; - newMessages.push({ "role": "assistant", "content": reply0 }); - newMessages.push({ "role": "user", "content": "Two more please" }); // next input - - const request1: ChatCompletionRequest = { - messages: newMessages, - } - const conv1: Conversation = - ((engine as any).getConversationFromChatCompletionRequest(request1, chatConfig)); - - expect(compareConversationObject(conv0, conv1)).toBe(true); - }); - - test('Test is NOT multi-round due to multiple new inputs', () => { - // Setups - const config_json = JSON.parse(configStr); - const chatConfig = { ...config_json } as ChatConfig; - const engine = new MLCEngine(); - - // Simulate request0 - const messages: ChatCompletionMessageParam[] = [ - { - "role": "system", - "content": "[INST] <>\n\nYou are a helpful, respectful and honest assistant. " + - "Be as happy as you can when speaking please.\n<>\n\n " - }, - { "role": "user", "content": "Provide me three US states." }, - ]; - const request0: ChatCompletionRequest = { - messages: messages, - }; - - // Simulate processing of request0, appending response to convA (done by LLMChatPipeline) - const conv0: Conversation = - ((engine as any).getConversationFromChatCompletionRequest(request0, chatConfig)); - conv0.appendMessage(Role.user, "Provide me three US states."); - const reply0 = "California, New York, Nevada."; - conv0.appendMessage(Role.assistant, reply0); // simulated response - - // Simulate request1, where user maintain the chat history, appending the resposne - const newMessages = [...messages]; - newMessages.push({ "role": "assistant", "content": reply0 }); - newMessages.push({ "role": "user", "content": "Two more please" }); // next input - - // Code above same as previous tests - // Add one more round of chat history - newMessages.push({ "role": "assistant", "content": "Pennsylvania, Florida" }); // next response - newMessages.push({ "role": "user", "content": "Thank you!" }); // next input - - const request1: ChatCompletionRequest = { - messages: newMessages, - } - const conv1: Conversation = - ((engine as any).getConversationFromChatCompletionRequest(request1, chatConfig)); - - expect(compareConversationObject(conv0, conv1)).toBe(false); - }); - - test('Test is NOT multi-round due to change in system prompt', () => { - // Setups - const config_json = JSON.parse(configStr); - const chatConfig = { ...config_json } as ChatConfig; - const engine = new MLCEngine(); - - // Simulate request0 - const messages: ChatCompletionMessageParam[] = [ - { - "role": "system", - "content": "[INST] <>\n\nYou are a helpful, respectful and honest assistant. " + - "Be as happy as you can when speaking please.\n<>\n\n " - }, - { "role": "user", "content": "Provide me three US states." }, - ]; - const request0: ChatCompletionRequest = { - messages: messages, - }; - - // Simulate processing of request0, appending response to convA (done by LLMChatPipeline) - const conv0: Conversation = - ((engine as any).getConversationFromChatCompletionRequest(request0, chatConfig)); - conv0.appendMessage(Role.user, "Provide me three US states."); - const reply0 = "California, New York, Nevada."; - conv0.appendMessage(Role.assistant, reply0); // simulated response - - // Simulate request1, where user maintain the chat history, appending the resposne - const newMessages = [...messages]; - newMessages.push({ "role": "assistant", "content": reply0 }); - newMessages.push({ "role": "user", "content": "Two more please" }); // next input - - // Code above same as previous tests - // Changed system prompt, should be false - newMessages[0].content = "No system prompt"; - - const request1: ChatCompletionRequest = { - messages: newMessages, - } - const conv1: Conversation = - ((engine as any).getConversationFromChatCompletionRequest(request1, chatConfig)); - - expect(compareConversationObject(conv0, conv1)).toBe(false); - }); - - test('Test is NOT multi-round due to change in role name', () => { - // Setups - const config_json = JSON.parse(configStr); - const chatConfig = { ...config_json } as ChatConfig; - const engine = new MLCEngine(); - - // Simulate request0 - const messages: ChatCompletionMessageParam[] = [ - { - "role": "system", - "content": "[INST] <>\n\nYou are a helpful, respectful and honest assistant. " + - "Be as happy as you can when speaking please.\n<>\n\n " - }, - { "role": "user", "content": "Provide me three US states." }, - ]; - const request0: ChatCompletionRequest = { - messages: messages, - }; - - // Simulate processing of request0, appending response to convA (done by LLMChatPipeline) - const conv0: Conversation = - ((engine as any).getConversationFromChatCompletionRequest(request0, chatConfig)); - conv0.appendMessage(Role.user, "Provide me three US states."); - const reply0 = "California, New York, Nevada."; - conv0.appendMessage(Role.assistant, reply0); // simulated response - - // Simulate request1, where user maintain the chat history, appending the resposne - const newMessages = [...messages]; - newMessages.push({ "role": "assistant", "content": reply0 }); - newMessages.push({ "role": "user", "content": "Two more please" }); // next input - - // Code above same as previous tests - // Changed system prompt, should be false - (newMessages[1] as ChatCompletionUserMessageParam).name = "Bob"; - - const request1: ChatCompletionRequest = { - messages: newMessages, - } - const conv1: Conversation = - ((engine as any).getConversationFromChatCompletionRequest(request1, chatConfig)); - - expect(compareConversationObject(conv0, conv1)).toBe(false); - }); - -}) \ No newline at end of file +import { MLCEngine } from "../src/engine"; +import { Conversation, compareConversationObject } from "../src/conversation"; +import { ChatConfig, Role } from "../src/config"; + +const configStr = + "{" + + ' "conv_template": {' + + ' "name": "llama-2",' + + ' "system_template": "[INST] <>\\n{system_message}\\n<>\\n\\n",' + + ' "system_message": "You are a helpful, respectful and honest assistant.",' + + ' "system_prefix_token_ids": [' + + " 1" + + " ]," + + ' "add_role_after_system_message": false,' + + ' "roles": {' + + ' "user": "[INST]",' + + ' "assistant": "[/INST]",' + + ' "tool": "[INST]"' + + " }," + + ' "role_templates": {' + + ' "user": "{user_message}",' + + ' "assistant": "{assistant_message}",' + + ' "tool": "{tool_message}"' + + " }," + + ' "messages": [],' + + ' "seps": [' + + ' " "' + + " ]," + + ' "role_content_sep": " ",' + + ' "role_empty_sep": " ",' + + ' "stop_str": [' + + ' "[INST]"' + + " ]," + + ' "stop_token_ids": [' + + " 2" + + " ]," + + ' "function_string": "",' + + ' "use_function_calling": false' + + " }" + + "}"; + +describe("Test multi-round chatting", () => { + test("Test is multi-round", () => { + // Setups + const config_json = JSON.parse(configStr); + const chatConfig = { ...config_json } as ChatConfig; + const engine = new MLCEngine(); + + // Simulate request0 + const messages: ChatCompletionMessageParam[] = [ + { + role: "system", + content: + "[INST] <>\n\nYou are a helpful, respectful and honest assistant. " + + "Be as happy as you can when speaking please.\n<>\n\n ", + }, + { role: "user", content: "Provide me three US states." }, + ]; + const request0: ChatCompletionRequest = { + messages: messages, + }; + + // Simulate processing of request0, appending response to convA (done by LLMChatPipeline) + const conv0: Conversation = ( + engine as any + ).getConversationFromChatCompletionRequest(request0, chatConfig); + conv0.appendMessage(Role.user, "Provide me three US states."); + const reply0 = "California, New York, Nevada."; + conv0.appendMessage(Role.assistant, reply0); // simulated response + + // Simulate request1, where user maintain the chat history, appending the resposne + const newMessages = [...messages]; + newMessages.push({ role: "assistant", content: reply0 }); + newMessages.push({ role: "user", content: "Two more please" }); // next input + + const request1: ChatCompletionRequest = { + messages: newMessages, + }; + const conv1: Conversation = ( + engine as any + ).getConversationFromChatCompletionRequest(request1, chatConfig); + + expect(compareConversationObject(conv0, conv1)).toBe(true); + }); + + test("Test is NOT multi-round due to multiple new inputs", () => { + // Setups + const config_json = JSON.parse(configStr); + const chatConfig = { ...config_json } as ChatConfig; + const engine = new MLCEngine(); + + // Simulate request0 + const messages: ChatCompletionMessageParam[] = [ + { + role: "system", + content: + "[INST] <>\n\nYou are a helpful, respectful and honest assistant. " + + "Be as happy as you can when speaking please.\n<>\n\n ", + }, + { role: "user", content: "Provide me three US states." }, + ]; + const request0: ChatCompletionRequest = { + messages: messages, + }; + + // Simulate processing of request0, appending response to convA (done by LLMChatPipeline) + const conv0: Conversation = ( + engine as any + ).getConversationFromChatCompletionRequest(request0, chatConfig); + conv0.appendMessage(Role.user, "Provide me three US states."); + const reply0 = "California, New York, Nevada."; + conv0.appendMessage(Role.assistant, reply0); // simulated response + + // Simulate request1, where user maintain the chat history, appending the resposne + const newMessages = [...messages]; + newMessages.push({ role: "assistant", content: reply0 }); + newMessages.push({ role: "user", content: "Two more please" }); // next input + + // Code above same as previous tests + // Add one more round of chat history + newMessages.push({ role: "assistant", content: "Pennsylvania, Florida" }); // next response + newMessages.push({ role: "user", content: "Thank you!" }); // next input + + const request1: ChatCompletionRequest = { + messages: newMessages, + }; + const conv1: Conversation = ( + engine as any + ).getConversationFromChatCompletionRequest(request1, chatConfig); + + expect(compareConversationObject(conv0, conv1)).toBe(false); + }); + + test("Test is NOT multi-round due to change in system prompt", () => { + // Setups + const config_json = JSON.parse(configStr); + const chatConfig = { ...config_json } as ChatConfig; + const engine = new MLCEngine(); + + // Simulate request0 + const messages: ChatCompletionMessageParam[] = [ + { + role: "system", + content: + "[INST] <>\n\nYou are a helpful, respectful and honest assistant. " + + "Be as happy as you can when speaking please.\n<>\n\n ", + }, + { role: "user", content: "Provide me three US states." }, + ]; + const request0: ChatCompletionRequest = { + messages: messages, + }; + + // Simulate processing of request0, appending response to convA (done by LLMChatPipeline) + const conv0: Conversation = ( + engine as any + ).getConversationFromChatCompletionRequest(request0, chatConfig); + conv0.appendMessage(Role.user, "Provide me three US states."); + const reply0 = "California, New York, Nevada."; + conv0.appendMessage(Role.assistant, reply0); // simulated response + + // Simulate request1, where user maintain the chat history, appending the resposne + const newMessages = [...messages]; + newMessages.push({ role: "assistant", content: reply0 }); + newMessages.push({ role: "user", content: "Two more please" }); // next input + + // Code above same as previous tests + // Changed system prompt, should be false + newMessages[0].content = "No system prompt"; + + const request1: ChatCompletionRequest = { + messages: newMessages, + }; + const conv1: Conversation = ( + engine as any + ).getConversationFromChatCompletionRequest(request1, chatConfig); + + expect(compareConversationObject(conv0, conv1)).toBe(false); + }); + + test("Test is NOT multi-round due to change in role name", () => { + // Setups + const config_json = JSON.parse(configStr); + const chatConfig = { ...config_json } as ChatConfig; + const engine = new MLCEngine(); + + // Simulate request0 + const messages: ChatCompletionMessageParam[] = [ + { + role: "system", + content: + "[INST] <>\n\nYou are a helpful, respectful and honest assistant. " + + "Be as happy as you can when speaking please.\n<>\n\n ", + }, + { role: "user", content: "Provide me three US states." }, + ]; + const request0: ChatCompletionRequest = { + messages: messages, + }; + + // Simulate processing of request0, appending response to convA (done by LLMChatPipeline) + const conv0: Conversation = ( + engine as any + ).getConversationFromChatCompletionRequest(request0, chatConfig); + conv0.appendMessage(Role.user, "Provide me three US states."); + const reply0 = "California, New York, Nevada."; + conv0.appendMessage(Role.assistant, reply0); // simulated response + + // Simulate request1, where user maintain the chat history, appending the resposne + const newMessages = [...messages]; + newMessages.push({ role: "assistant", content: reply0 }); + newMessages.push({ role: "user", content: "Two more please" }); // next input + + // Code above same as previous tests + // Changed system prompt, should be false + (newMessages[1] as ChatCompletionUserMessageParam).name = "Bob"; + + const request1: ChatCompletionRequest = { + messages: newMessages, + }; + const conv1: Conversation = ( + engine as any + ).getConversationFromChatCompletionRequest(request1, chatConfig); + + expect(compareConversationObject(conv0, conv1)).toBe(false); + }); +}); diff --git a/utils/vram_requirements/src/vram_requirements.html b/utils/vram_requirements/src/vram_requirements.html index 5b5ccc5d..f3f0d626 100644 --- a/utils/vram_requirements/src/vram_requirements.html +++ b/utils/vram_requirements/src/vram_requirements.html @@ -7,8 +7,8 @@

vRAM Requirement Report

Open console to see logs -
-
+
+