From ec83e81c4cfafc8f4008e4550406c14ac0a1ce00 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Fri, 25 Oct 2024 07:34:14 +0000 Subject: [PATCH 001/143] fix: use custom tool template only for huggingchat --- src/lib/server/models.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/server/models.ts b/src/lib/server/models.ts index b321aca9d6b..7ba20dc7713 100644 --- a/src/lib/server/models.ts +++ b/src/lib/server/models.ts @@ -130,7 +130,7 @@ async function getChatPromptRender( // or use the `rag` mode without the citations const id = m.id ?? m.name; - if (id.startsWith("CohereForAI")) { + if (isHuggingChat && id.startsWith("CohereForAI")) { formattedMessages = [ { role: "system", @@ -157,7 +157,7 @@ async function getChatPromptRender( }, ...formattedMessages, ]; - } else if (id.startsWith("meta-llama")) { + } else if (isHuggingChat && id.startsWith("meta-llama")) { const results = toolResults.flatMap((result) => { if (result.status === ToolResultStatus.Error) { return [ From c83861a64bcb036807c65cee78298dfd7211c5dc Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Fri, 25 Oct 2024 08:06:13 +0000 Subject: [PATCH 002/143] fix(endpoint): fallback to hf token for openai endpoint type --- src/lib/server/endpoints/openai/endpointOai.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/server/endpoints/openai/endpointOai.ts b/src/lib/server/endpoints/openai/endpointOai.ts index 693c6068cef..0a9413aeec1 100644 --- a/src/lib/server/endpoints/openai/endpointOai.ts +++ b/src/lib/server/endpoints/openai/endpointOai.ts @@ -86,7 +86,7 @@ export const endpointOAIParametersSchema = z.object({ model: z.any(), type: z.literal("openai"), baseURL: z.string().url().default("https://api.openai.com/v1"), - apiKey: z.string().default(env.OPENAI_API_KEY ?? "sk-"), + apiKey: z.string().default(env.OPENAI_API_KEY || env.HF_TOKEN || "sk-"), completion: z .union([z.literal("completions"), z.literal("chat_completions")]) .default("chat_completions"), @@ -135,7 +135,7 @@ export async function endpointOai( } const openai = new OpenAI({ - apiKey: apiKey ?? "sk-", + apiKey: apiKey || "sk-", baseURL, defaultHeaders, defaultQuery, From 938e1d89da3fdc87870b8f18dde283038a8d5634 Mon Sep 17 00:00:00 2001 From: goupilew Date: Mon, 28 Oct 2024 10:33:31 +0100 Subject: [PATCH 003/143] [feat] Add support for endpoint web sources (#1544) * [feat] Add support for endpoint web sources * chores: lint --------- Co-authored-by: Nathan Sarrazin --- src/lib/components/chat/ChatMessage.svelte | 28 +++++++++++++++++++ src/lib/server/endpoints/endpoints.ts | 5 ++-- .../server/endpoints/google/endpointVertex.ts | 28 +++++++++++++++++-- src/lib/server/textGeneration/generate.ts | 7 ++++- src/lib/types/MessageUpdate.ts | 1 + 5 files changed, 63 insertions(+), 6 deletions(-) diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index ff4f9f37915..05fe2ad43dd 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -23,10 +23,12 @@ import OpenWebSearchResults from "../OpenWebSearchResults.svelte"; import { + MessageUpdateType, MessageWebSearchUpdateType, type MessageToolUpdate, type MessageWebSearchSourcesUpdate, type MessageWebSearchUpdate, + type MessageFinalAnswerUpdate, } from "$lib/types/MessageUpdate"; import { base } from "$app/paths"; import { useConvTreeStore } from "$lib/stores/convTree"; @@ -149,6 +151,10 @@ $: searchUpdates = (message.updates?.filter(({ type }) => type === "webSearch") ?? []) as MessageWebSearchUpdate[]; + $: messageFinalAnswer = message.updates?.find( + ({ type }) => type === MessageUpdateType.FinalAnswer + ) as MessageFinalAnswerUpdate; + // filter all updates with type === "tool" then group them by uuid field $: toolUpdates = message.updates @@ -314,7 +320,29 @@ {/each} {/if} + + + {#if messageFinalAnswer?.webSources && messageFinalAnswer.webSources.length} +
+
Sources:
+ {#each messageFinalAnswer.webSources as { uri, title }} + + {title} favicon +
{title}
+ + {/each} +
+ {/if} + {#if !loading && (message.content || toolUpdates)}
{ + const uri = chunk.web?.uri ?? chunk.retrievedContext?.uri; + const title = chunk.web?.title ?? chunk.retrievedContext?.title; + + if (!uri || !title) { + return null; + } + + return { + uri, + title, + }; + }) + .filter((source) => source !== null); + + if (candidateWebSources) { + webSources.push(...candidateWebSources); + } + const content = firstPart.text; generatedText += content; - const output: TextGenerationStreamOutput = { + const output: TextGenerationStreamOutputWithToolsAndWebSources = { token: { id: tokenId++, text: content, @@ -194,6 +215,7 @@ export function endpointVertex(input: z.input Date: Mon, 28 Oct 2024 16:20:21 +0530 Subject: [PATCH 004/143] fix(ui): swap delete confirmation buttons for better UX (#1546) (#1547) Co-authored-by: Aditya Medhane Co-authored-by: Nathan Sarrazin --- src/lib/components/NavConversationItem.svelte | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/lib/components/NavConversationItem.svelte b/src/lib/components/NavConversationItem.svelte index adc4d508a2d..3baca2f7713 100644 --- a/src/lib/components/NavConversationItem.svelte +++ b/src/lib/components/NavConversationItem.svelte @@ -52,6 +52,14 @@
{#if confirmDelete} + - {:else} + {#if file.type === "hash"} + {#await fetch(urlNotTrailing + "/output/" + file.value).then((res) => res.text())} +
+ +
+ {:then result} + 
{result}
+ {/await} + {:else} + 
{atob(file.value)}
+ {/if} + {/if} {/if} - {/if} + + diff --git a/src/lib/server/endpoints/preprocessMessages.ts b/src/lib/server/endpoints/preprocessMessages.ts index e38dd9abc26..6b9560a45fa 100644 --- a/src/lib/server/endpoints/preprocessMessages.ts +++ b/src/lib/server/endpoints/preprocessMessages.ts @@ -11,7 +11,8 @@ export async function preprocessMessages( ): Promise { return Promise.resolve(messages) .then((msgs) => addWebSearchContext(msgs, webSearch)) - .then((msgs) => downloadFiles(msgs, convId)); + .then((msgs) => downloadFiles(msgs, convId)) + .then((msgs) => injectClipboardFiles(msgs)); } function addWebSearchContext(messages: Message[], webSearch: Message["webSearch"]) { @@ -54,3 +55,21 @@ async function downloadFiles(messages: Message[], convId: ObjectId): Promise { + const plaintextFiles = message.files + ?.filter((file) => file.mime === "application/vnd.chatui.clipboard") + .map((file) => Buffer.from(file.value, "base64").toString("utf-8")); + + if (!plaintextFiles || plaintextFiles.length === 0) return message; + + return { + ...message, + content: `${plaintextFiles.join("\n\n")}\n\n${message.content}`, + files: message.files?.filter((file) => file.mime !== "application/vnd.chatui.clipboard"), + }; + }) + ); +} From fffd510dc2c15fee907e4424ef6a1b9bf330ae35 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Wed, 30 Oct 2024 18:31:25 +0000 Subject: [PATCH 007/143] feat: add setting option to disable content pasting --- scripts/populate.ts | 2 +- src/lib/components/chat/ChatWindow.svelte | 2 +- src/lib/stores/settings.ts | 1 + src/lib/types/Settings.ts | 2 ++ src/routes/+layout.server.ts | 1 + src/routes/settings/(nav)/+page.svelte | 21 ++++++++++++++++++--- src/routes/settings/(nav)/+server.ts | 1 + 7 files changed, 25 insertions(+), 5 deletions(-) diff --git a/scripts/populate.ts b/scripts/populate.ts index 929192100fc..9f9de1fd0da 100644 --- a/scripts/populate.ts +++ b/scripts/populate.ts @@ -150,9 +150,9 @@ async function seed() { createdAt: faker.date.recent({ days: 30 }), updatedAt: faker.date.recent({ days: 30 }), disableStream: faker.datatype.boolean(0.25), + directPaste: faker.datatype.boolean(0.25), customPrompts: {}, assistants: [], - disableStream: faker.datatype.boolean(0.25), }; await collections.settings.updateOne( { userId: user._id }, diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte index a30904647ae..f47878c2700 100644 --- a/src/lib/components/chat/ChatWindow.svelte +++ b/src/lib/components/chat/ChatWindow.svelte @@ -92,7 +92,7 @@ const onPaste = (e: ClipboardEvent) => { const textContent = e.clipboardData?.getData("text"); - if (textContent && textContent.length > 256) { + if (!$settings.directPaste && textContent && textContent.length > 256) { e.preventDefault(); pastedLongContent = true; setTimeout(() => { diff --git a/src/lib/stores/settings.ts b/src/lib/stores/settings.ts index 83c12599d54..c8e9615552e 100644 --- a/src/lib/stores/settings.ts +++ b/src/lib/stores/settings.ts @@ -17,6 +17,7 @@ type SettingsStore = { assistants: Array; tools?: Array; disableStream: boolean; + directPaste: boolean; }; type SettingsStoreWritable = Writable & { diff --git a/src/lib/types/Settings.ts b/src/lib/types/Settings.ts index f45aa0bf0b4..b66b0c8023b 100644 --- a/src/lib/types/Settings.ts +++ b/src/lib/types/Settings.ts @@ -23,6 +23,7 @@ export interface Settings extends Timestamps { assistants?: Assistant["_id"][]; tools?: string[]; disableStream: boolean; + directPaste: boolean; } export type SettingsEditable = Omit; @@ -35,4 +36,5 @@ export const DEFAULT_SETTINGS = { assistants: [], tools: [], disableStream: false, + directPaste: false, } satisfies SettingsEditable; diff --git a/src/routes/+layout.server.ts b/src/routes/+layout.server.ts index 5ce048cd0a3..a9ea681f2cc 100644 --- a/src/routes/+layout.server.ts +++ b/src/routes/+layout.server.ts @@ -174,6 +174,7 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => { .filter((el) => !el.isHidden && el.isOnByDefault) .map((el) => el._id.toString()), disableStream: settings?.disableStream ?? DEFAULT_SETTINGS.disableStream, + directPaste: settings?.directPaste ?? DEFAULT_SETTINGS.directPaste, }, models: models.map((model) => ({ id: model.id, diff --git a/src/routes/settings/(nav)/+page.svelte b/src/routes/settings/(nav)/+page.svelte index 46f3e7adbf5..76be962b0fe 100644 --- a/src/routes/settings/(nav)/+page.svelte +++ b/src/routes/settings/(nav)/+page.svelte @@ -25,7 +25,7 @@ > -
+
{#if envPublic.PUBLIC_APP_DATA_SHARING === "1"}