From 1a499a32b9a25ab0f2ff534adc136ad35eb76483 Mon Sep 17 00:00:00 2001 From: Linchenn Date: Wed, 6 Mar 2024 10:36:36 -0800 Subject: [PATCH 1/6] Add LLM Inference sample for Web --- examples/llm_inference/js/README.md | 19 +++++++ examples/llm_inference/js/index.html | 31 +++++++++++ examples/llm_inference/js/index.js | 82 ++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 examples/llm_inference/js/README.md create mode 100644 examples/llm_inference/js/index.html create mode 100644 examples/llm_inference/js/index.js diff --git a/examples/llm_inference/js/README.md b/examples/llm_inference/js/README.md new file mode 100644 index 00000000..fe149de8 --- /dev/null +++ b/examples/llm_inference/js/README.md @@ -0,0 +1,19 @@ +# MediaPipe LLM Inference task for web + +## Overview + +This web sample demonstrates how to use the LLM Inference API to run common text-to-text generation tasks like information retrieval, email drafting, and document summarization, on web. + + +## Prerequisites + +* A device that can access Chrome and has WebGPU support (Desktop and Laptop are recommanded). + +## Running the demo + +Follow the following instructions to run the sample on your device: +1. Make a folder for the task, named as `llm_task`, and copy the [index.html](https://github.com/googlesamples/mediapipe/blob/main/examples/llm_inference/js/index.html) and [index.js](https://github.com/googlesamples/mediapipe/blob/main/examples/llm_inference/js/index.js) files into your `llm_task` folder. +2. Download one of the [compatible models](https://developers.google.com/mediapipe/solutions/genai/llm_inference#models) that you want to run, into the `llm_task` folder. +3. In your `index.js` file, update []`modelFileName`](https://github.com/googlesamples/mediapipe/blob/main/examples/llm_inference/js/index.js#L23) with your model file's name. +4. Run `python3 -m http.server 8000` under the `llm_task` folder to host the three files. +5. Open `localhost:8000` in Chrome. Then the button on the webpage will be enabled when the task is ready (~10 seconds). diff --git a/examples/llm_inference/js/index.html b/examples/llm_inference/js/index.html new file mode 100644 index 00000000..e0987bc9 --- /dev/null +++ b/examples/llm_inference/js/index.html @@ -0,0 +1,31 @@ + + + + + + BYOM-Web Demo + + + Input:
+
+
+
+ Result:
+ + + + + + \ No newline at end of file diff --git a/examples/llm_inference/js/index.js b/examples/llm_inference/js/index.js new file mode 100644 index 00000000..d447fc3e --- /dev/null +++ b/examples/llm_inference/js/index.js @@ -0,0 +1,82 @@ +// Copyright 2024 The MediaPipe Authors. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ---------------------------------------------------------------------------------------- // + +import {FilesetResolver, LlmInference} from 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-genai'; + +const input = document.getElementById('input'); +const output = document.getElementById('output'); +const submit = document.getElementById('submit'); + +const modelFileName = 'llm.tflite'; /* Update the file name */ + +/** + * Display tokens to the output text box. + */ +function displayNewTokens(tokens, complete) { + if (output.textContent != null && output.textContent.length > 0) { + output.textContent += tokens; + } else { + output.textContent = tokens; + } + + if (complete) { + if (output.textContent == null || output.textContent.length === 0) { + output.textContent = 'Result is empty'; + } + } +} + +/** + * Main function to run LLM Inference. + */ +async function runDemo() { + const genaiFileset = await FilesetResolver.forGenAiTasks( + 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-genai/wasm'); + const llmInferenceOptions = { + baseOptions: {modelAssetPath: modelFileName}, + }; + + let llmInference; + + submit.onclick = () => { + output.textContent = ''; + submit.disabled = true; + llmInference.generateResponse(input.value, displayNewTokens).finally(() => { + submit.disabled = false; + }); + }; + + submit.value = 'Loading the model...' + LlmInference + .createFromOptions( + genaiFileset, + llmInferenceOptions, + ) + .then(llm => { + llmInference = llm; + submit.disabled = false; + submit.value = 'Get Response' + }); + + window.onbeforeunload = () => { + if (llmInference) { + llmInference.close(); + llmInference = null; + } + }; +} + +runDemo(); \ No newline at end of file From d49e454d967de7988100e652b627118ebd7e5180 Mon Sep 17 00:00:00 2001 From: Linchenn Date: Wed, 6 Mar 2024 10:57:40 -0800 Subject: [PATCH 2/6] Update index.html --- examples/llm_inference/js/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_inference/js/index.html b/examples/llm_inference/js/index.html index e0987bc9..2cad3264 100644 --- a/examples/llm_inference/js/index.html +++ b/examples/llm_inference/js/index.html @@ -15,7 +15,7 @@ - BYOM-Web Demo + LLM Inference Web Demo Input:
From a798462a6710f17be10a87a5e611f1f435f006a6 Mon Sep 17 00:00:00 2001 From: Linchenn Date: Wed, 6 Mar 2024 11:11:04 -0800 Subject: [PATCH 3/6] Update README.md --- examples/llm_inference/js/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_inference/js/README.md b/examples/llm_inference/js/README.md index fe149de8..db0962f9 100644 --- a/examples/llm_inference/js/README.md +++ b/examples/llm_inference/js/README.md @@ -7,7 +7,7 @@ This web sample demonstrates how to use the LLM Inference API to run common text ## Prerequisites -* A device that can access Chrome and has WebGPU support (Desktop and Laptop are recommanded). +* A browser with WebGPU support (such as Chrome, Edge or Opera). ## Running the demo From e8e3803394cf6f76022e11079c9dfb1eec1b0445 Mon Sep 17 00:00:00 2001 From: Linchenn Date: Wed, 6 Mar 2024 11:11:47 -0800 Subject: [PATCH 4/6] empty line --- examples/llm_inference/js/index.html | 3 +-- examples/llm_inference/js/index.js | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/llm_inference/js/index.html b/examples/llm_inference/js/index.html index 2cad3264..70ad817d 100644 --- a/examples/llm_inference/js/index.html +++ b/examples/llm_inference/js/index.html @@ -27,5 +27,4 @@ - - \ No newline at end of file + diff --git a/examples/llm_inference/js/index.js b/examples/llm_inference/js/index.js index d447fc3e..99907efc 100644 --- a/examples/llm_inference/js/index.js +++ b/examples/llm_inference/js/index.js @@ -79,4 +79,4 @@ async function runDemo() { }; } -runDemo(); \ No newline at end of file +runDemo(); From e72b5de2138c0910a7e3a6d02401ed24eb677f61 Mon Sep 17 00:00:00 2001 From: Linchenn Date: Wed, 6 Mar 2024 11:40:18 -0800 Subject: [PATCH 5/6] Update index.js --- examples/llm_inference/js/index.js | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/examples/llm_inference/js/index.js b/examples/llm_inference/js/index.js index 99907efc..bab7ebf8 100644 --- a/examples/llm_inference/js/index.js +++ b/examples/llm_inference/js/index.js @@ -26,14 +26,10 @@ const modelFileName = 'llm.tflite'; /* Update the file name */ * Display tokens to the output text box. */ function displayNewTokens(tokens, complete) { - if (output.textContent != null && output.textContent.length > 0) { - output.textContent += tokens; - } else { - output.textContent = tokens; - } + output.textContent += tokens; if (complete) { - if (output.textContent == null || output.textContent.length === 0) { + if (!output.textContent) { output.textContent = 'Result is empty'; } } @@ -45,38 +41,29 @@ function displayNewTokens(tokens, complete) { async function runDemo() { const genaiFileset = await FilesetResolver.forGenAiTasks( 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-genai/wasm'); - const llmInferenceOptions = { - baseOptions: {modelAssetPath: modelFileName}, - }; - let llmInference; submit.onclick = () => { output.textContent = ''; submit.disabled = true; - llmInference.generateResponse(input.value, displayNewTokens).finally(() => { + llmInference.generateResponse(input.value, displayNewTokens).then(() => { submit.disabled = false; }); }; submit.value = 'Loading the model...' LlmInference - .createFromOptions( + .createFromModelPath( genaiFileset, - llmInferenceOptions, + modelFileName, ) .then(llm => { llmInference = llm; submit.disabled = false; submit.value = 'Get Response' + }).catch(() =>{ + alert('Failed to initialize the task.'); }); - - window.onbeforeunload = () => { - if (llmInference) { - llmInference.close(); - llmInference = null; - } - }; } runDemo(); From 66ed93b079e0870692e543aa0d927fcf191cced1 Mon Sep 17 00:00:00 2001 From: Linchenn Date: Wed, 6 Mar 2024 11:49:22 -0800 Subject: [PATCH 6/6] Update index.js --- examples/llm_inference/js/index.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/examples/llm_inference/js/index.js b/examples/llm_inference/js/index.js index bab7ebf8..63ca0235 100644 --- a/examples/llm_inference/js/index.js +++ b/examples/llm_inference/js/index.js @@ -32,6 +32,7 @@ function displayNewTokens(tokens, complete) { if (!output.textContent) { output.textContent = 'Result is empty'; } + submit.disabled = false; } } @@ -46,17 +47,12 @@ async function runDemo() { submit.onclick = () => { output.textContent = ''; submit.disabled = true; - llmInference.generateResponse(input.value, displayNewTokens).then(() => { - submit.disabled = false; - }); + llmInference.generateResponse(input.value, displayNewTokens); }; submit.value = 'Loading the model...' LlmInference - .createFromModelPath( - genaiFileset, - modelFileName, - ) + .createFromModelPath(genaiFileset, modelFileName) .then(llm => { llmInference = llm; submit.disabled = false;