Agents

.env

@@ -87,6 +87,7 @@ PUBLIC_APP_COLOR=blue # can be any of tailwind colors: https://tailwindcss.com/d
 PUBLIC_APP_DATA_SHARING=#set to 1 to enable options & text regarding data sharing
 PUBLIC_APP_DISCLAIMER=#set to 1 to show a disclaimer on login page
 
+TOOLS = []
 # PUBLIC_APP_NAME=HuggingChat
 # PUBLIC_APP_ASSETS=huggingchat
 # PUBLIC_APP_COLOR=yellow

README.md

@@ -162,11 +162,11 @@ MODELS=`[
 
 You can change things like the parameters, or customize the preprompt to better suit your needs. You can also add more models by adding more objects to the array, with different preprompts for example.
 
-#### Custom prompt templates:
+#### Custom prompt templates
 
 By default the prompt is constructed using `userMessageToken`, `assistantMessageToken`, `userMessageEndToken`, `assistantMessageEndToken`, `preprompt` parameters and a series of default templates.
 
-However, these templates can be modified by setting the `chatPromptTemplate` and `webSearchQueryPromptTemplate` parameters. Note that if WebSearch is not enabled, only `chatPromptTemplate` needs to be set. The template language is https://handlebarsjs.com. The templates have access to the model's prompt parameters (`preprompt`, etc.). However, if the templates are specified it is recommended to inline the prompt parameters, as using the references (`{{preprompt}}`) is deprecated.
+However, these templates can be modified by setting the `chatPromptTemplate` and `webSearchQueryPromptTemplate` parameters. Note that if WebSearch is not enabled, only `chatPromptTemplate` needs to be set. The template language is <https://handlebarsjs.com>. The templates have access to the model's prompt parameters (`preprompt`, etc.). However, if the templates are specified it is recommended to inline the prompt parameters, as using the references (`{{preprompt}}`) is deprecated.
 
 For example:
 
@@ -300,6 +300,37 @@ If the model being hosted will be available on multiple servers/instances add th
 
 ```
 
+### Tools
+
+chat-ui supports two tools currently:
+
+- `webSearch`
+- `textToImage`
+
+You can enable them by adding the following JSON to your `.env.local`:
+
+```
+TOOLS = `[
+  {
+    "name" : "textToImage",
+    "model" : "[model name form the hub here]"
+  },
+  {
+    "name" : "webSearch",
+    "key" : {
+      "type" : "serper",
+      "apiKey" : "[your key here]"
+    }
+  }
+]`
+```
+
+Or a subset of these if you only want to enable some of the tools.
+
+The web search key `type` can be either `serper` or `serpapi`.
+
+The `textToImage` model can be [any model from the hub](https://huggingface.co/tasks/text-to-image) that matches the right task as long as the inference endpoint for it is enabled.
+
 ## Deploying to a HF Space
 
 Create a `DOTENV_LOCAL` secret to your HF space with the content of your .env.local, and they will be picked up automatically when you run.

src/lib/components/OpenWebSearchResults.svelte

@@ -1,5 +1,10 @@
 <script lang="ts">
-	import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
+	import type {
+		AgentUpdate,
+		ErrorUpdate,
+		MessageUpdate,
+		WebSearchUpdate,
+	} from "$lib/types/MessageUpdate";
 	import CarbonCaretRight from "~icons/carbon/caret-right";
 
 	import CarbonCheckmark from "~icons/carbon/checkmark-filled";
@@ -9,11 +14,19 @@
 
 	export let loading = false;
 	export let classNames = "";
-	export let webSearchMessages: WebSearchUpdate[] = [];
+	export let messages: MessageUpdate[] = [];
 
 	let detailsOpen: boolean;
 	let error: boolean;
-	$: error = webSearchMessages[webSearchMessages.length - 1]?.messageType === "error";
+
+	$: messagesToDisplay = messages.filter(
+		(el) =>
+			el.type === "agent" ||
+			(el.type === "webSearch" && el.messageType !== "sources") ||
+			el.type === "error"
+	) as Array<WebSearchUpdate | AgentUpdate | ErrorUpdate>;
+
+	$: error = messages.some((el) => el.type === "error");
 </script>
 
 <details
@@ -31,56 +44,46 @@
 			<CarbonCheckmark class="my-auto text-gray-500" />
 		{/if}
 		<span class="px-2 font-medium" class:text-red-700={error} class:dark:text-red-500={error}
-			>Web search
+			>Tools
 		</span>
 		<div class="my-auto transition-all" class:rotate-90={detailsOpen}>
 			<CarbonCaretRight />
 		</div>
 	</summary>
 
 	<div class="content px-5 pb-5 pt-4">
-		{#if webSearchMessages.length === 0}
+		{#if messagesToDisplay.length === 0}
 			<div class="mx-auto w-fit">
 				<EosIconsLoading class="mb-3 h-4 w-4" />
 			</div>
 		{:else}
 			<ol>
-				{#each webSearchMessages as message}
-					{#if message.messageType === "update"}
-						<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
-							<div class="flex items-start">
+				{#each messagesToDisplay as message}
+					<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
+						<div class="flex items-start">
+							{#if message.type === "error"}
+								<CarbonError class=" -ml-2 -mt-0.5 h-4 w-4 text-red-700 dark:text-red-500" />
+							{:else}
 								<div
 									class="-ml-1.5 h-3 w-3 flex-none rounded-full bg-gray-200 dark:bg-gray-600 {loading
 										? 'group-last:animate-pulse group-last:bg-gray-300 group-last:dark:bg-gray-500'
 										: ''}"
 								/>
-								<h3 class="text-md -mt-1.5 pl-2.5 text-gray-800 dark:text-gray-100">
-									{message.message}
-								</h3>
-							</div>
-							{#if message.args}
-								<p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
-									{message.args}
-								</p>
-							{/if}
-						</li>
-					{:else if message.messageType === "error"}
-						<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
-							<div class="flex items-start">
-								<CarbonError
-									class="-ml-1.5 h-3 w-3 flex-none scale-110 text-red-700 dark:text-red-500"
-								/>
-								<h3 class="text-md -mt-1.5 pl-2.5 text-red-700 dark:text-red-500">
-									{message.message}
-								</h3>
-							</div>
-							{#if message.args}
-								<p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
-									{message.args}
-								</p>
 							{/if}
-						</li>
-					{/if}
+							<h3
+								class="text-md -mt-1.5 pl-2.5 text-gray-800 dark:text-gray-100"
+								class:text-red-700={message.type === "error"}
+								class:dark:text-red-500={message.type === "error"}
+							>
+								{message.message}
+							</h3>
+						</div>
+						{#if message.type === "webSearch" && message.args}
+							<p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
+								{message.args}
+							</p>
+						{/if}
+					</li>
 				{/each}
 			</ol>
 		{/if}

src/lib/components/WebSearchToggle.svelte

@@ -3,25 +3,56 @@
 	import CarbonInformation from "~icons/carbon/information";
 	import Switch from "./Switch.svelte";
 
-	const toggle = () => ($webSearchParameters.useSearch = !$webSearchParameters.useSearch);
+	export let tools: {
+		webSearch: boolean;
+		textToImage: boolean;
+	};
+
+	const toggleWebSearch = () => ($webSearchParameters.useSearch = !$webSearchParameters.useSearch);
+	const toggleSDXL = () => ($webSearchParameters.useSDXL = !$webSearchParameters.useSDXL);
 </script>
 
 <div
-	class="flex h-9 cursor-pointer select-none items-center gap-2 rounded-xl border bg-white p-1.5 shadow-sm hover:shadow-none dark:border-gray-800 dark:bg-gray-900"
-	on:click={toggle}
-	on:keypress={toggle}
+	class="flex flex-col flex-nowrap  rounded-xl border bg-white p-1.5 shadow-sm hover:shadow-none dark:border-gray-800 dark:bg-gray-900"
 >
-	<Switch name="useSearch" bind:checked={$webSearchParameters.useSearch} on:click on:keypress />
-	<div class="whitespace-nowrap text-sm text-gray-800 dark:text-gray-200">Search web</div>
-	<div class="group relative w-max">
-		<CarbonInformation class="text-xs text-gray-500" />
+	{#if tools.webSearch}
+		<div
+			class="flex h-9 w-full cursor-pointer select-none items-center gap-2"
+			on:click={toggleWebSearch}
+			on:keypress={toggleWebSearch}
+		>
+			<Switch name="useSearch" bind:checked={$webSearchParameters.useSearch} on:click on:keypress />
+			<div class="whitespace-nowrap text-sm text-gray-800 dark:text-gray-200">Web Search</div>
+			<div class="group relative mr-auto w-max">
+				<CarbonInformation class="text-xs text-gray-500" />
+				<div
+					class="pointer-events-none absolute -top-20 left-1/2 w-max -translate-x-1/2 rounded-md bg-gray-100 p-2 opacity-0 transition-opacity group-hover:opacity-100 dark:bg-gray-800"
+				>
+					<p class="max-w-sm text-sm text-gray-800 dark:text-gray-200">
+						When enabled, the request will be completed with relevant context fetched from the web.
+					</p>
+				</div>
+			</div>
+		</div>
+	{/if}
+	{#if tools.textToImage}
 		<div
-			class="pointer-events-none absolute -top-20 left-1/2 w-max -translate-x-1/2 rounded-md bg-gray-100 p-2 opacity-0 transition-opacity group-hover:opacity-100 dark:bg-gray-800"
+			class="flex h-9 cursor-pointer select-none items-center gap-2"
+			on:click={toggleSDXL}
+			on:keypress={toggleSDXL}
 		>
-			<p class="max-w-sm text-sm text-gray-800 dark:text-gray-200">
-				When enabled, the model will try to complement its answer with information queried from the
-				web.
-			</p>
+			<Switch name="useSearch" bind:checked={$webSearchParameters.useSDXL} on:click on:keypress />
+			<div class="whitespace-nowrap text-sm text-gray-800 dark:text-gray-200">SDXL Images</div>
+			<div class="group relative w-max">
+				<CarbonInformation class="text-xs text-gray-500" />
+				<div
+					class="pointer-events-none absolute -top-20 left-1/2 w-max -translate-x-1/2 rounded-md bg-gray-100 p-2 opacity-0 transition-opacity group-hover:opacity-100 dark:bg-gray-800"
+				>
+					<p class="max-w-sm text-sm text-gray-800 dark:text-gray-200">
+						When enabled, the model will try to generate images to go along with the answers.
+					</p>
+				</div>
+			</div>
 		</div>
-	</div>
+	{/if}
 </div>

src/lib/components/chat/ChatMessage.svelte

@@ -2,8 +2,7 @@
 	import { marked } from "marked";
 	import markedKatex from "marked-katex-extension";
 	import type { Message } from "$lib/types/Message";
-	import { afterUpdate, createEventDispatcher } from "svelte";
-	import { deepestChild } from "$lib/utils/deepestChild";
+	import { createEventDispatcher } from "svelte";
 	import { page } from "$app/stores";
 
 	import CodeBlock from "../CodeBlock.svelte";
@@ -17,7 +16,7 @@
 	import type { Model } from "$lib/types/Model";
 
 	import OpenWebSearchResults from "../OpenWebSearchResults.svelte";
-	import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
+	import type { MessageUpdate, WebSearchUpdate } from "$lib/types/MessageUpdate";
 
 	function sanitizeMd(md: string) {
 		let ret = md
@@ -49,16 +48,14 @@
 	export let readOnly = false;
 	export let isTapped = false;
 
-	export let webSearchMessages: WebSearchUpdate[];
+	export let updateMessages: MessageUpdate[];
 
 	const dispatch = createEventDispatcher<{
 		retry: { content: string; id: Message["id"] };
 		vote: { score: Message["score"]; id: Message["id"] };
 	}>();
 
 	let contentEl: HTMLElement;
-	let loadingEl: IconLoading;
-	let pendingTimeout: ReturnType<typeof setTimeout>;
 	let isCopied = false;
 
 	const renderer = new marked.Renderer();
@@ -89,40 +86,19 @@
 
 	$: tokens = marked.lexer(sanitizeMd(message.content));
 
-	afterUpdate(() => {
-		loadingEl?.$destroy();
-		clearTimeout(pendingTimeout);
-
-		// Add loading animation to the last message if update takes more than 600ms
-		if (loading) {
-			pendingTimeout = setTimeout(() => {
-				if (contentEl) {
-					loadingEl = new IconLoading({
-						target: deepestChild(contentEl),
-						props: { classNames: "loading inline ml-2" },
-					});
-				}
-			}, 600);
-		}
-	});
-
-	let searchUpdates: WebSearchUpdate[] = [];
-
-	$: searchUpdates = ((webSearchMessages.length > 0
-		? webSearchMessages
-		: message.updates?.filter(({ type }) => type === "webSearch")) ?? []) as WebSearchUpdate[];
-
 	$: downloadLink =
 		message.from === "user" ? `${$page.url.pathname}/message/${message.id}/prompt` : undefined;
 
 	let webSearchIsDone = true;
 
 	$: webSearchIsDone =
-		searchUpdates.length > 0 && searchUpdates[searchUpdates.length - 1].messageType === "sources";
+		updateMessages.length > 0 && updateMessages[updateMessages.length - 1].type === "finalAnswer";
 
 	$: webSearchSources =
-		searchUpdates &&
-		searchUpdates?.filter(({ messageType }) => messageType === "sources")?.[0]?.sources;
+		updateMessages &&
+		(updateMessages?.filter(({ type }) => type === "webSearch") as WebSearchUpdate[]).filter(
+			({ messageType }) => messageType === "sources"
+		)?.[0]?.sources;
 
 	$: if (isCopied) {
 		setTimeout(() => {
@@ -145,21 +121,48 @@
 		<div
 			class="relative min-h-[calc(2rem+theme(spacing[3.5])*2)] min-w-[60px] break-words rounded-2xl border border-gray-100 bg-gradient-to-br from-gray-50 px-5 py-3.5 text-gray-600 prose-pre:my-2 dark:border-gray-800 dark:from-gray-800/40 dark:text-gray-300"
 		>
-			{#if searchUpdates && searchUpdates.length > 0}
+			{#if updateMessages && updateMessages.filter(({ type }) => type === "agent").length > 0}
 				<OpenWebSearchResults
 					classNames={tokens.length ? "mb-3.5" : ""}
-					webSearchMessages={searchUpdates}
-					loading={!(searchUpdates[searchUpdates.length - 1]?.messageType === "sources")}
+					messages={updateMessages}
+					loading={!(updateMessages[updateMessages.length - 1]?.type === "finalAnswer")}
 				/>
 			{/if}
-			{#if !message.content && (webSearchIsDone || (webSearchMessages && webSearchMessages.length === 0))}
+			{#if !message.content && (webSearchIsDone || (updateMessages && updateMessages.length === 0))}
 				<IconLoading />
 			{/if}
 
 			<div
 				class="prose max-w-none dark:prose-invert max-sm:prose-sm prose-headings:font-semibold prose-h1:text-lg prose-h2:text-base prose-h3:text-base prose-pre:bg-gray-800 dark:prose-pre:bg-gray-900"
 				bind:this={contentEl}
 			>
+				{#if message.files && message.files.length > 0}
+					<div class="mx-auto grid w-fit grid-cols-2 gap-5">
+						{#each message.files as file}
+							<div class="flex flex-col flex-nowrap gap-0">
+								{#if file.mime?.startsWith("image")}
+									<img
+										src={$page.url.pathname + "/output/" + file.sha256}
+										alt="tool output"
+										class="my-2 aspect-auto"
+									/>
+								{:else if file.mime?.startsWith("audio")}
+									<audio controls class="my-2">
+										<source src={$page.url.pathname + "/output/" + file.sha256} type="audio/wav" />
+									</audio>
+								{/if}
+								{#if file.model}
+									<span class="text-sm"
+										>Content generated using <a href={`https://huggingface.co/${file.model}`}
+											>{file.model}</a
+										></span
+									>
+								{/if}
+							</div>
+						{/each}
+					</div>
+					<div class="my-5 w-full border-b-2 border-gray-300 dark:border-gray-700" />
+				{/if}
 				{#each tokens as token}
 					{#if token.type === "code"}
 						<CodeBlock lang={token.lang} code={unsanitizeMd(token.text)} />