From c2c499c31ad59cc55257fe47d277bbf7145ef5b4 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 09:50:05 +0100 Subject: [PATCH 01/61] 1 --- packages/ui-utils/src/types.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/ui-utils/src/types.ts b/packages/ui-utils/src/types.ts index d1108f87f8c0..ef14c295843c 100644 --- a/packages/ui-utils/src/types.ts +++ b/packages/ui-utils/src/types.ts @@ -84,8 +84,25 @@ Tool invocations (that can be tool calls or tool results, depending on whether o that the assistant made as part of this message. */ toolInvocations?: Array; + + parts: Array; } +export type TextUIPart = { + type: 'text'; + text: string; +}; + +export type ReasoningUIPart = { + type: 'reasoning'; + reasoning: string; +}; + +export type ToolInvocationUIPart = { + type: 'tool-invocation'; + toolInvocation: ToolInvocation; +}; + export type CreateMessage = Omit & { id?: Message['id']; }; From 447ca78f0e3c79255a382e5e40d254f69c554daf Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 10:36:19 +0100 Subject: [PATCH 02/61] 2 --- .../next-openai/app/use-chat-tools/page.tsx | 125 +++++++++--------- packages/react/src/use-assistant.ts | 5 +- packages/react/src/use-chat.ts | 50 ++++--- packages/ui-utils/src/call-chat-api.ts | 1 + .../ui-utils/src/process-chat-response.ts | 110 ++++++++++++--- packages/vue/src/use-assistant.ts | 3 + 6 files changed, 192 insertions(+), 102 deletions(-) diff --git a/examples/next-openai/app/use-chat-tools/page.tsx b/examples/next-openai/app/use-chat-tools/page.tsx index 4df23edffe37..9319bbc8538b 100644 --- a/examples/next-openai/app/use-chat-tools/page.tsx +++ b/examples/next-openai/app/use-chat-tools/page.tsx @@ -28,71 +28,76 @@ export default function Chat() { {messages?.map((m: Message) => (
{`${m.role}: `} - {m.toolInvocations?.map((toolInvocation: ToolInvocation) => { - const toolCallId = toolInvocation.toolCallId; + {m.parts.map(p => { + switch (p.type) { + case 'text': + return p.text; + case 'tool-invocation': { + const toolInvocation = p.toolInvocation; + const toolCallId = toolInvocation.toolCallId; - // example of pre-rendering streaming tool calls - if (toolInvocation.state === 'partial-call') { - return ( - 
-                  {JSON.stringify(toolInvocation, null, 2)}
-
- ); - } + // example of pre-rendering streaming tool calls + if (toolInvocation.state === 'partial-call') { + return ( + 
+                      {JSON.stringify(toolInvocation, null, 2)}
+
+ ); + } + + // render confirmation tool (client-side tool with user interaction) + if (toolInvocation.toolName === 'askForConfirmation') { + return ( +
+ {toolInvocation.args.message} +
+ {'result' in toolInvocation ? ( + {toolInvocation.result} + ) : ( + <> + + + + )} +
+
+ ); + } - // render confirmation tool (client-side tool with user interaction) - if (toolInvocation.toolName === 'askForConfirmation') { - return ( -
- {toolInvocation.args.message} -
- {'result' in toolInvocation ? ( - {toolInvocation.result} - ) : ( - <> - - - - )} + // other tools: + return 'result' in toolInvocation ? ( +
+ Tool call {`${toolInvocation.toolName}: `} + {toolInvocation.result} +
+ ) : ( +
+ Calling {toolInvocation.toolName}...
-
- ); + ); + } } - - // other tools: - return 'result' in toolInvocation ? ( -
- Tool call {`${toolInvocation.toolName}: `} - {toolInvocation.result} -
- ) : ( -
- Calling {toolInvocation.toolName}... -
- ); })} - {m.content} -

))} diff --git a/packages/react/src/use-assistant.ts b/packages/react/src/use-assistant.ts index 0af3802635e1..0436adedc69f 100644 --- a/packages/react/src/use-assistant.ts +++ b/packages/react/src/use-assistant.ts @@ -185,6 +185,7 @@ export function useAssistant({ id: value.id, role: value.role, content: value.content[0].text.value, + parts: [], }, ]); }, @@ -198,6 +199,7 @@ export function useAssistant({ id: lastMessage.id, role: lastMessage.role, content: lastMessage.content + value, + parts: lastMessage.parts, }, ]; }); @@ -220,6 +222,7 @@ export function useAssistant({ role: 'data', content: '', data: value.data, + parts: [], }, ]); }, @@ -257,7 +260,7 @@ export function useAssistant({ return; } - append({ role: 'user', content: input }, requestOptions); + append({ role: 'user', content: input, parts: [] }, requestOptions); }; const setThreadId = (threadId: string | undefined) => { diff --git a/packages/react/src/use-chat.ts b/packages/react/src/use-chat.ts index 311f16e4f120..c966a545fe0f 100644 --- a/packages/react/src/use-chat.ts +++ b/packages/react/src/use-chat.ts @@ -4,6 +4,7 @@ import type { CreateMessage, JSONValue, Message, + ToolInvocationUIPart, UseChatOptions, } from '@ai-sdk/ui-utils'; import { @@ -497,6 +498,7 @@ By default, it's set to 1, which means that only a single LLM call is made. content: input, experimental_attachments: attachmentsForRequest.length > 0 ? attachmentsForRequest : undefined, + parts: [{ type: 'text', text: input }], }); const chatRequest: ChatRequest = { @@ -519,32 +521,38 @@ By default, it's set to 1, which means that only a single LLM call is made. const addToolResult = useCallback( ({ toolCallId, result }: { toolCallId: string; result: any }) => { - const updatedMessages = messagesRef.current.map((message, index, arr) => - // update the tool calls in the last assistant message: - index === arr.length - 1 && - message.role === 'assistant' && - message.toolInvocations - ? { - ...message, - toolInvocations: message.toolInvocations.map(toolInvocation => - toolInvocation.toolCallId === toolCallId - ? { - ...toolInvocation, - result, - state: 'result' as const, - } - : toolInvocation, - ), - } - : message, + const lastMessage = messagesRef.current[messagesRef.current.length - 1]; + + const invocationPart = lastMessage.parts.find( + (part): part is ToolInvocationUIPart => + part.type === 'tool-invocation' && + part.toolInvocation.toolCallId === toolCallId, + ); + + if (invocationPart == null) { + return; + } + + const toolResult = { + ...invocationPart.toolInvocation, + state: 'result' as const, + result, + }; + + invocationPart.toolInvocation = toolResult; + + lastMessage.toolInvocations = lastMessage.toolInvocations?.map( + toolInvocation => + toolInvocation.toolCallId === toolCallId + ? toolResult + : toolInvocation, ); - mutate(updatedMessages, false); + mutate(messagesRef.current, false); // auto-submit when all tool calls in the last assistant message have results: - const lastMessage = updatedMessages[updatedMessages.length - 1]; if (isAssistantMessageWithCompletedToolCalls(lastMessage)) { - triggerRequest({ messages: updatedMessages }); + triggerRequest({ messages: messagesRef.current }); } }, [mutate, triggerRequest], diff --git a/packages/ui-utils/src/call-chat-api.ts b/packages/ui-utils/src/call-chat-api.ts index 56610e892238..bd3a7f91418e 100644 --- a/packages/ui-utils/src/call-chat-api.ts +++ b/packages/ui-utils/src/call-chat-api.ts @@ -80,6 +80,7 @@ export async function callChatApi({ createdAt: new Date(), role: 'assistant' as const, content: '', + parts: [], }; await processTextStream({ diff --git a/packages/ui-utils/src/process-chat-response.ts b/packages/ui-utils/src/process-chat-response.ts index e1ff8bb533bf..71c34b029e99 100644 --- a/packages/ui-utils/src/process-chat-response.ts +++ b/packages/ui-utils/src/process-chat-response.ts @@ -1,7 +1,15 @@ import { generateId as generateIdFunction } from '@ai-sdk/provider-utils'; import { parsePartialJson } from './parse-partial-json'; import { processDataStream } from './process-data-stream'; -import type { JSONValue, Message, UseChatOptions } from './types'; +import type { + JSONValue, + Message, + ReasoningUIPart, + TextUIPart, + ToolInvocation, + ToolInvocationUIPart, + UseChatOptions, +} from './types'; import { LanguageModelV1FinishReason } from '@ai-sdk/provider'; import { calculateLanguageModelUsage, @@ -49,8 +57,32 @@ export async function processChatResponse({ createdAt: getCurrentDate(), role: 'assistant', content: '', + parts: [], }; + let currentTextPart: TextUIPart | undefined = undefined; + let currentReasoningPart: ReasoningUIPart | undefined = undefined; + + function updateToolInvocationPart( + toolCallId: string, + invocation: ToolInvocation, + ) { + const part = message.parts.find( + part => + part.type === 'tool-invocation' && + part.toolInvocation.toolCallId === toolCallId, + ) as ToolInvocationUIPart | undefined; + + if (part != null) { + part.toolInvocation = invocation; + } else { + message.parts.push({ + type: 'tool-invocation', + toolInvocation: invocation, + }); + } + } + const data: JSONValue[] = []; // keep list of current message annotations for message @@ -103,10 +135,30 @@ export async function processChatResponse({ await processDataStream({ stream, onTextPart(value) { + if (currentTextPart == null) { + currentTextPart = { + type: 'text', + text: value, + }; + message.parts.push(currentTextPart); + } else { + currentTextPart.text += value; + } + message.content += value; execUpdate(); }, onReasoningPart(value) { + if (currentReasoningPart == null) { + currentReasoningPart = { + type: 'reasoning', + reasoning: value, + }; + message.parts.push(currentReasoningPart); + } else { + currentReasoningPart.reasoning += value; + } + message.reasoning = (message.reasoning ?? '') + value; execUpdate(); }, @@ -123,13 +175,17 @@ export async function processChatResponse({ index: message.toolInvocations.length, }; - message.toolInvocations.push({ + const invocation = { state: 'partial-call', step, toolCallId: value.toolCallId, toolName: value.toolName, args: undefined, - }); + } as const; + + message.toolInvocations.push(invocation); + + updateToolInvocationPart(value.toolCallId, invocation); execUpdate(); }, @@ -140,49 +196,59 @@ export async function processChatResponse({ const { value: partialArgs } = parsePartialJson(partialToolCall.text); - message.toolInvocations![partialToolCall.index] = { + const invocation = { state: 'partial-call', step: partialToolCall.step, toolCallId: value.toolCallId, toolName: partialToolCall.toolName, args: partialArgs, - }; + } as const; + + message.toolInvocations![partialToolCall.index] = invocation; + + updateToolInvocationPart(value.toolCallId, invocation); execUpdate(); }, async onToolCallPart(value) { + const invocation = { + state: 'call', + step, + ...value, + } as const; + if (partialToolCalls[value.toolCallId] != null) { // change the partial tool call to a full tool call - message.toolInvocations![partialToolCalls[value.toolCallId].index] = { - state: 'call', - step, - ...value, - }; + message.toolInvocations![partialToolCalls[value.toolCallId].index] = + invocation; } else { if (message.toolInvocations == null) { message.toolInvocations = []; } - message.toolInvocations.push({ - state: 'call', - step, - ...value, - }); + message.toolInvocations.push(invocation); } + updateToolInvocationPart(value.toolCallId, invocation); + // invoke the onToolCall callback if it exists. This is blocking. // In the future we should make this non-blocking, which // requires additional state management for error handling etc. if (onToolCall) { const result = await onToolCall({ toolCall: value }); if (result != null) { - // store the result in the tool invocation - message.toolInvocations![message.toolInvocations!.length - 1] = { + const invocation = { state: 'result', step, ...value, result, - }; + } as const; + + // store the result in the tool invocation + message.toolInvocations![message.toolInvocations!.length - 1] = + invocation; + + updateToolInvocationPart(value.toolCallId, invocation); } } @@ -207,11 +273,15 @@ export async function processChatResponse({ ); } - toolInvocations[toolInvocationIndex] = { + const invocation = { ...toolInvocations[toolInvocationIndex], state: 'result' as const, ...value, - }; + } as const; + + toolInvocations[toolInvocationIndex] = invocation; + + updateToolInvocationPart(value.toolCallId, invocation); execUpdate(); }, diff --git a/packages/vue/src/use-assistant.ts b/packages/vue/src/use-assistant.ts index c79af9b1b0f2..d1b513c1a641 100644 --- a/packages/vue/src/use-assistant.ts +++ b/packages/vue/src/use-assistant.ts @@ -189,6 +189,7 @@ export function useAssistant({ id: value.id, content: value.content[0].text.value, role: value.role, + parts: [], }, ]; }, @@ -220,6 +221,7 @@ export function useAssistant({ role: 'data', content: '', data: value.data, + parts: [], }, ]); }, @@ -261,6 +263,7 @@ export function useAssistant({ { role: 'user', content: input.value, + parts: [], }, requestOptions, ); From 0ee042a9495031973992bbd2877cfc7ae92ba2ca Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 10:51:13 +0100 Subject: [PATCH 03/61] tweak --- .../next-openai/app/use-chat-tools/page.tsx | 163 +++++++++++------- 1 file changed, 96 insertions(+), 67 deletions(-) diff --git a/examples/next-openai/app/use-chat-tools/page.tsx b/examples/next-openai/app/use-chat-tools/page.tsx index 9319bbc8538b..6dd4e7b2f725 100644 --- a/examples/next-openai/app/use-chat-tools/page.tsx +++ b/examples/next-openai/app/use-chat-tools/page.tsx @@ -1,7 +1,6 @@ 'use client'; -import { ToolInvocation } from 'ai'; -import { Message, useChat } from 'ai/react'; +import { useChat } from 'ai/react'; export default function Chat() { const { messages, input, handleInputChange, handleSubmit, addToolResult } = @@ -25,76 +24,106 @@ export default function Chat() { return (
- {messages?.map((m: Message) => ( -
- {`${m.role}: `} - {m.parts.map(p => { - switch (p.type) { + {messages?.map(message => ( +
+ {`${message.role}: `} + {message.parts.map(part => { + switch (part.type) { case 'text': - return p.text; + return part.text; case 'tool-invocation': { - const toolInvocation = p.toolInvocation; - const toolCallId = toolInvocation.toolCallId; + const invocation = part.toolInvocation; + const callId = invocation.toolCallId; - // example of pre-rendering streaming tool calls - if (toolInvocation.state === 'partial-call') { - return ( - 
-                      {JSON.stringify(toolInvocation, null, 2)}
-
- ); - } + switch (invocation.toolName) { + case 'askForConfirmation': { + switch (invocation.state) { + case 'partial-call': + return undefined; + case 'call': + return ( +
+ {invocation.args.message} +
+ + +
+
+ ); + case 'result': + return ( +
+ Location access allowed: {invocation.result} +
+ ); + } + } - // render confirmation tool (client-side tool with user interaction) - if (toolInvocation.toolName === 'askForConfirmation') { - return ( -
- {toolInvocation.args.message} -
- {'result' in toolInvocation ? ( - {toolInvocation.result} - ) : ( - <> - - - - )} -
-
- ); - } + case 'getLocation': { + switch (invocation.state) { + case 'partial-call': + return undefined; + case 'call': + return ( +
+ Getting location... +
+ ); + case 'result': + return ( +
+ Location: {invocation.result} +
+ ); + } + } - // other tools: - return 'result' in toolInvocation ? ( -
- Tool call {`${toolInvocation.toolName}: `} - {toolInvocation.result} -
- ) : ( -
- Calling {toolInvocation.toolName}... -
- ); + case 'getWeatherInformation': { + switch (invocation.state) { + // example of pre-rendering streaming tool calls: + case 'partial-call': + return ( + 
+                            {JSON.stringify(invocation, null, 2)}
+
+ ); + case 'call': + return ( +
+ Getting weather information for{' '} + {invocation.args.city}... +
+ ); + case 'result': + return ( +
+ Weather in {invocation.args.city}:{' '} + {invocation.result} +
+ ); + } + } + } } } })} From 793d89d65429603ea4f2b6ef03b2899242f3d2ca Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 11:07:50 +0100 Subject: [PATCH 04/61] fx --- examples/next-openai/app/api/use-chat-tools/route.ts | 4 +++- examples/next-openai/package.json | 1 + packages/react/src/use-chat.ts | 2 +- pnpm-lock.yaml | 3 +++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/next-openai/app/api/use-chat-tools/route.ts b/examples/next-openai/app/api/use-chat-tools/route.ts index 51f5f40e933e..1acf9753c1e3 100644 --- a/examples/next-openai/app/api/use-chat-tools/route.ts +++ b/examples/next-openai/app/api/use-chat-tools/route.ts @@ -1,3 +1,4 @@ +import { anthropic } from '@ai-sdk/anthropic'; import { openai } from '@ai-sdk/openai'; import { streamText, tool } from 'ai'; import { z } from 'zod'; @@ -9,7 +10,8 @@ export async function POST(req: Request) { const { messages } = await req.json(); const result = streamText({ - model: openai('gpt-4o'), + // model: openai('gpt-4o'), + model: anthropic('claude-3-5-sonnet-latest'), messages, toolCallStreaming: true, maxSteps: 5, // multi-steps for server-side tools diff --git a/examples/next-openai/package.json b/examples/next-openai/package.json index ec26d2e2abea..706bc24d06d6 100644 --- a/examples/next-openai/package.json +++ b/examples/next-openai/package.json @@ -9,6 +9,7 @@ "lint": "next lint" }, "dependencies": { + "@ai-sdk/anthropic": "1.1.6", "@ai-sdk/deepseek": "0.1.8", "@ai-sdk/openai": "1.1.9", "@ai-sdk/ui-utils": "1.1.8", diff --git a/packages/react/src/use-chat.ts b/packages/react/src/use-chat.ts index c966a545fe0f..63aa377020de 100644 --- a/packages/react/src/use-chat.ts +++ b/packages/react/src/use-chat.ts @@ -349,7 +349,7 @@ By default, it's set to 1, which means that only a single LLM call is made. // check that next step is possible: isAssistantMessageWithCompletedToolCalls(lastMessage) && // check that assistant has not answered yet: - !lastMessage.content && // empty string or undefined + lastMessage.parts[lastMessage.parts.length - 1].type !== 'text' && // TODO brittle, should check for text after last tool invocation // limit the number of automatic steps: (extractMaxToolInvocationStep(lastMessage.toolInvocations) ?? 0) < maxSteps diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2014241164d6..43789c963495 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -481,6 +481,9 @@ importers: examples/next-openai: dependencies: + '@ai-sdk/anthropic': + specifier: 1.1.6 + version: link:../../packages/anthropic '@ai-sdk/deepseek': specifier: 0.1.8 version: link:../../packages/deepseek From 63911a456a19cf72b3535bc29d9a3e8f38ab915b Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 12:28:53 +0100 Subject: [PATCH 05/61] tests --- .../process-chat-response.test.ts.snap | 1189 +++++++++++++++++ .../src/process-chat-response.test.ts | 669 +--------- 2 files changed, 1218 insertions(+), 640 deletions(-) diff --git a/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap b/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap index e0d8e57732cf..0f421e7a72ff 100644 --- a/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap +++ b/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap @@ -1,5 +1,325 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html +exports[`scenario: delayed message annotations in onFinish > should call the onFinish function with the correct arguments 1`] = ` +[ + { + "finishReason": "stop", + "message": { + "annotations": [ + { + "example": "annotation", + }, + ], + "content": "text", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "text", + "type": "text", + }, + ], + "role": "assistant", + }, + "usage": { + "completionTokens": 5, + "promptTokens": 10, + "totalTokens": 15, + }, + }, +] +`; + +exports[`scenario: delayed message annotations in onFinish > should call the update function with the correct arguments 1`] = ` +[ + { + "data": [], + "message": { + "content": "text", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "text", + "type": "text", + }, + ], + "revisionId": "id-1", + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "annotations": [ + { + "example": "annotation", + }, + ], + "content": "text", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "text", + "type": "text", + }, + ], + "revisionId": "id-2", + "role": "assistant", + }, + "replaceLastMessage": false, + }, +] +`; + +exports[`scenario: message annotations in onChunk > should call the onFinish function with the correct arguments 1`] = ` +[ + { + "finishReason": "stop", + "message": { + "annotations": [ + "annotation1", + "annotation2", + ], + "content": "t1t2", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "t1t2", + "type": "text", + }, + ], + "role": "assistant", + }, + "usage": { + "completionTokens": 5, + "promptTokens": 10, + "totalTokens": 15, + }, + }, +] +`; + +exports[`scenario: message annotations in onChunk > should call the update function with the correct arguments 1`] = ` +[ + { + "data": [], + "message": { + "annotations": [ + "annotation1", + ], + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [], + "revisionId": "id-1", + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "annotations": [ + "annotation1", + ], + "content": "t1", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "t1", + "type": "text", + }, + ], + "revisionId": "id-2", + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "annotations": [ + "annotation1", + "annotation2", + ], + "content": "t1", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "t1", + "type": "text", + }, + ], + "revisionId": "id-3", + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "annotations": [ + "annotation1", + "annotation2", + ], + "content": "t1t2", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "t1t2", + "type": "text", + }, + ], + "revisionId": "id-4", + "role": "assistant", + }, + "replaceLastMessage": false, + }, +] +`; + +exports[`scenario: message annotations with existing assistant lastMessage > should call the onFinish function with the correct arguments 1`] = ` +[ + { + "finishReason": "stop", + "message": { + "annotations": [ + "annotation0", + "annotation1", + ], + "content": "t1", + "createdAt": 2023-01-02T00:00:00.000Z, + "id": "original-id", + "parts": [ + { + "text": "t1", + "type": "text", + }, + ], + "role": "assistant", + }, + "usage": { + "completionTokens": 5, + "promptTokens": 10, + "totalTokens": 15, + }, + }, +] +`; + +exports[`scenario: message annotations with existing assistant lastMessage > should call the update function with the correct arguments 1`] = ` +[ + { + "data": [], + "message": { + "annotations": [ + "annotation0", + "annotation1", + ], + "content": "", + "createdAt": 2023-01-02T00:00:00.000Z, + "id": "original-id", + "parts": [], + "revisionId": "id-0", + "role": "assistant", + }, + "replaceLastMessage": true, + }, + { + "data": [], + "message": { + "annotations": [ + "annotation0", + "annotation1", + ], + "content": "t1", + "createdAt": 2023-01-02T00:00:00.000Z, + "id": "original-id", + "parts": [ + { + "text": "t1", + "type": "text", + }, + ], + "revisionId": "id-1", + "role": "assistant", + }, + "replaceLastMessage": true, + }, +] +`; + +exports[`scenario: server provides message ids > should call the onFinish function with the correct arguments 1`] = ` +[ + { + "finishReason": "stop", + "message": { + "content": "Hello, world!", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "step_123", + "parts": [ + { + "text": "Hello, world!", + "type": "text", + }, + ], + "role": "assistant", + }, + "usage": { + "completionTokens": 5, + "promptTokens": 10, + "totalTokens": 15, + }, + }, +] +`; + +exports[`scenario: server provides message ids > should call the update function with the correct arguments 1`] = ` +[ + { + "data": [], + "message": { + "content": "Hello, ", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "step_123", + "parts": [ + { + "text": "Hello, ", + "type": "text", + }, + ], + "revisionId": "id-1", + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "Hello, world!", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "step_123", + "parts": [ + { + "text": "Hello, world!", + "type": "text", + }, + ], + "revisionId": "id-2", + "role": "assistant", + }, + "replaceLastMessage": false, + }, +] +`; + exports[`scenario: server provides reasoning > should call the onFinish function with the correct arguments 1`] = ` [ { @@ -8,6 +328,16 @@ exports[`scenario: server provides reasoning > should call the onFinish function "content": "Hello, world!", "createdAt": 2023-01-01T00:00:00.000Z, "id": "step_123", + "parts": [ + { + "reasoning": "I will open the conversation with witty banter. Once the user has relaxed, I will pry for valuable information.", + "type": "reasoning", + }, + { + "text": "Hello, world!", + "type": "text", + }, + ], "reasoning": "I will open the conversation with witty banter. Once the user has relaxed, I will pry for valuable information.", "role": "assistant", }, @@ -28,6 +358,12 @@ exports[`scenario: server provides reasoning > should call the update function w "content": "", "createdAt": 2023-01-01T00:00:00.000Z, "id": "step_123", + "parts": [ + { + "reasoning": "I will open the conversation", + "type": "reasoning", + }, + ], "reasoning": "I will open the conversation", "revisionId": "id-1", "role": "assistant", @@ -40,6 +376,12 @@ exports[`scenario: server provides reasoning > should call the update function w "content": "", "createdAt": 2023-01-01T00:00:00.000Z, "id": "step_123", + "parts": [ + { + "reasoning": "I will open the conversation with witty banter. ", + "type": "reasoning", + }, + ], "reasoning": "I will open the conversation with witty banter. ", "revisionId": "id-2", "role": "assistant", @@ -52,6 +394,12 @@ exports[`scenario: server provides reasoning > should call the update function w "content": "", "createdAt": 2023-01-01T00:00:00.000Z, "id": "step_123", + "parts": [ + { + "reasoning": "I will open the conversation with witty banter. Once the user has relaxed,", + "type": "reasoning", + }, + ], "reasoning": "I will open the conversation with witty banter. Once the user has relaxed,", "revisionId": "id-3", "role": "assistant", @@ -64,6 +412,12 @@ exports[`scenario: server provides reasoning > should call the update function w "content": "", "createdAt": 2023-01-01T00:00:00.000Z, "id": "step_123", + "parts": [ + { + "reasoning": "I will open the conversation with witty banter. Once the user has relaxed, I will pry for valuable information.", + "type": "reasoning", + }, + ], "reasoning": "I will open the conversation with witty banter. Once the user has relaxed, I will pry for valuable information.", "revisionId": "id-4", "role": "assistant", @@ -76,6 +430,16 @@ exports[`scenario: server provides reasoning > should call the update function w "content": "Hello, ", "createdAt": 2023-01-01T00:00:00.000Z, "id": "step_123", + "parts": [ + { + "reasoning": "I will open the conversation with witty banter. Once the user has relaxed, I will pry for valuable information.", + "type": "reasoning", + }, + { + "text": "Hello, ", + "type": "text", + }, + ], "reasoning": "I will open the conversation with witty banter. Once the user has relaxed, I will pry for valuable information.", "revisionId": "id-5", "role": "assistant", @@ -88,6 +452,16 @@ exports[`scenario: server provides reasoning > should call the update function w "content": "Hello, world!", "createdAt": 2023-01-01T00:00:00.000Z, "id": "step_123", + "parts": [ + { + "reasoning": "I will open the conversation with witty banter. Once the user has relaxed, I will pry for valuable information.", + "type": "reasoning", + }, + { + "text": "Hello, world!", + "type": "text", + }, + ], "reasoning": "I will open the conversation with witty banter. Once the user has relaxed, I will pry for valuable information.", "revisionId": "id-6", "role": "assistant", @@ -96,3 +470,818 @@ exports[`scenario: server provides reasoning > should call the update function w }, ] `; + +exports[`scenario: server-side continue roundtrip > should call the onFinish function with the correct arguments 1`] = ` +[ + { + "finishReason": "stop", + "message": { + "content": "The weather in London is sunny.", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "The weather in London is sunny.", + "type": "text", + }, + ], + "role": "assistant", + }, + "usage": { + "completionTokens": 7, + "promptTokens": 14, + "totalTokens": 21, + }, + }, +] +`; + +exports[`scenario: server-side continue roundtrip > should call the update function with the correct arguments 1`] = ` +[ + { + "data": [], + "message": { + "content": "The weather in London ", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "The weather in London ", + "type": "text", + }, + ], + "revisionId": "id-1", + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "The weather in London is sunny.", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "The weather in London is sunny.", + "type": "text", + }, + ], + "revisionId": "id-2", + "role": "assistant", + }, + "replaceLastMessage": false, + }, +] +`; + +exports[`scenario: server-side tool roundtrip > should call the onFinish function with the correct arguments 1`] = ` +[ + { + "finishReason": "stop", + "message": { + "content": "The weather in London is sunny.", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + { + "text": "The weather in London is sunny.", + "type": "text", + }, + ], + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "usage": { + "completionTokens": 7, + "promptTokens": 14, + "totalTokens": 21, + }, + }, +] +`; + +exports[`scenario: server-side tool roundtrip > should call the update function with the correct arguments 1`] = ` +[ + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "toolInvocation": { + "args": { + "city": "London", + }, + "state": "call", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + ], + "revisionId": "id-1", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "state": "call", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + ], + "revisionId": "id-2", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "The weather in London is sunny.", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + { + "text": "The weather in London is sunny.", + "type": "text", + }, + ], + "revisionId": "id-3", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": false, + }, +] +`; + +exports[`scenario: server-side tool roundtrip with existing assistant message > should call the onFinish function with the correct arguments 1`] = ` +[ + { + "finishReason": "stop", + "message": { + "content": "The weather in London is sunny.", + "createdAt": 2023-01-02T00:00:00.000Z, + "id": "original-id", + "parts": [ + { + "toolInvocation": { + "args": {}, + "result": { + "location": "Berlin", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id-original", + "toolName": "tool-name-original", + }, + "type": "tool-invocation", + }, + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 1, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + { + "text": "The weather in London is sunny.", + "type": "text", + }, + ], + "role": "assistant", + "toolInvocations": [ + { + "args": {}, + "result": { + "location": "Berlin", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id-original", + "toolName": "tool-name-original", + }, + { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 1, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "usage": { + "completionTokens": 7, + "promptTokens": 14, + "totalTokens": 21, + }, + }, +] +`; + +exports[`scenario: server-side tool roundtrip with existing assistant message > should call the update function with the correct arguments 1`] = ` +[ + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-02T00:00:00.000Z, + "id": "original-id", + "parts": [ + { + "toolInvocation": { + "args": {}, + "result": { + "location": "Berlin", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id-original", + "toolName": "tool-name-original", + }, + "type": "tool-invocation", + }, + { + "toolInvocation": { + "args": { + "city": "London", + }, + "state": "call", + "step": 1, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + ], + "revisionId": "id-0", + "role": "assistant", + "toolInvocations": [ + { + "args": {}, + "result": { + "location": "Berlin", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id-original", + "toolName": "tool-name-original", + }, + { + "args": { + "city": "London", + }, + "state": "call", + "step": 1, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": true, + }, + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-02T00:00:00.000Z, + "id": "original-id", + "parts": [ + { + "toolInvocation": { + "args": {}, + "result": { + "location": "Berlin", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id-original", + "toolName": "tool-name-original", + }, + "type": "tool-invocation", + }, + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 1, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + ], + "revisionId": "id-1", + "role": "assistant", + "toolInvocations": [ + { + "args": {}, + "result": { + "location": "Berlin", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id-original", + "toolName": "tool-name-original", + }, + { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 1, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": true, + }, + { + "data": [], + "message": { + "content": "The weather in London is sunny.", + "createdAt": 2023-01-02T00:00:00.000Z, + "id": "original-id", + "parts": [ + { + "toolInvocation": { + "args": {}, + "result": { + "location": "Berlin", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id-original", + "toolName": "tool-name-original", + }, + "type": "tool-invocation", + }, + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 1, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + { + "text": "The weather in London is sunny.", + "type": "text", + }, + ], + "revisionId": "id-2", + "role": "assistant", + "toolInvocations": [ + { + "args": {}, + "result": { + "location": "Berlin", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id-original", + "toolName": "tool-name-original", + }, + { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 1, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": true, + }, +] +`; + +exports[`scenario: simple text response > should call the onFinish function with the correct arguments 1`] = ` +[ + { + "finishReason": "stop", + "message": { + "content": "Hello, world!", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "Hello, world!", + "type": "text", + }, + ], + "role": "assistant", + }, + "usage": { + "completionTokens": 5, + "promptTokens": 10, + "totalTokens": 15, + }, + }, +] +`; + +exports[`scenario: simple text response > should call the update function with the correct arguments 1`] = ` +[ + { + "data": [], + "message": { + "content": "Hello, ", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "Hello, ", + "type": "text", + }, + ], + "revisionId": "id-1", + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "Hello, world!", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "Hello, world!", + "type": "text", + }, + ], + "revisionId": "id-2", + "role": "assistant", + }, + "replaceLastMessage": false, + }, +] +`; + +exports[`scenario: tool call streaming > should call the onFinish function with the correct arguments 1`] = ` +[ + { + "finishReason": "stop", + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "toolInvocation": { + "args": { + "testArg": "test-value", + }, + "result": "test-result", + "state": "result", + "step": 0, + "toolCallId": "tool-call-0", + "toolName": "test-tool", + }, + "type": "tool-invocation", + }, + ], + "role": "assistant", + "toolInvocations": [ + { + "args": { + "testArg": "test-value", + }, + "result": "test-result", + "state": "result", + "step": 0, + "toolCallId": "tool-call-0", + "toolName": "test-tool", + }, + ], + }, + "usage": { + "completionTokens": 5, + "promptTokens": 10, + "totalTokens": 15, + }, + }, +] +`; + +exports[`scenario: tool call streaming > should call the update function with the correct arguments 1`] = ` +[ + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "toolInvocation": { + "args": undefined, + "state": "partial-call", + "step": 0, + "toolCallId": "tool-call-0", + "toolName": "test-tool", + }, + "type": "tool-invocation", + }, + ], + "revisionId": "id-1", + "role": "assistant", + "toolInvocations": [ + { + "args": undefined, + "state": "partial-call", + "step": 0, + "toolCallId": "tool-call-0", + "toolName": "test-tool", + }, + ], + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "toolInvocation": { + "args": { + "testArg": "t", + }, + "state": "partial-call", + "step": 0, + "toolCallId": "tool-call-0", + "toolName": "test-tool", + }, + "type": "tool-invocation", + }, + ], + "revisionId": "id-2", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "testArg": "t", + }, + "state": "partial-call", + "step": 0, + "toolCallId": "tool-call-0", + "toolName": "test-tool", + }, + ], + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "toolInvocation": { + "args": { + "testArg": "test-value", + }, + "state": "partial-call", + "step": 0, + "toolCallId": "tool-call-0", + "toolName": "test-tool", + }, + "type": "tool-invocation", + }, + ], + "revisionId": "id-3", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "testArg": "test-value", + }, + "state": "partial-call", + "step": 0, + "toolCallId": "tool-call-0", + "toolName": "test-tool", + }, + ], + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "toolInvocation": { + "args": { + "testArg": "test-value", + }, + "state": "call", + "step": 0, + "toolCallId": "tool-call-0", + "toolName": "test-tool", + }, + "type": "tool-invocation", + }, + ], + "revisionId": "id-4", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "testArg": "test-value", + }, + "state": "call", + "step": 0, + "toolCallId": "tool-call-0", + "toolName": "test-tool", + }, + ], + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "toolInvocation": { + "args": { + "testArg": "test-value", + }, + "result": "test-result", + "state": "result", + "step": 0, + "toolCallId": "tool-call-0", + "toolName": "test-tool", + }, + "type": "tool-invocation", + }, + ], + "revisionId": "id-5", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "testArg": "test-value", + }, + "result": "test-result", + "state": "result", + "step": 0, + "toolCallId": "tool-call-0", + "toolName": "test-tool", + }, + ], + }, + "replaceLastMessage": false, + }, +] +`; diff --git a/packages/ui-utils/src/process-chat-response.test.ts b/packages/ui-utils/src/process-chat-response.test.ts index 1755506ef5f3..186cf2adffbb 100644 --- a/packages/ui-utils/src/process-chat-response.test.ts +++ b/packages/ui-utils/src/process-chat-response.test.ts @@ -71,49 +71,11 @@ describe('scenario: simple text response', () => { }); it('should call the update function with the correct arguments', async () => { - expect(updateCalls).toStrictEqual([ - { - message: { - content: 'Hello, ', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - revisionId: 'id-1', - role: 'assistant', - }, - data: [], - replaceLastMessage: false, - }, - { - message: { - content: 'Hello, world!', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - revisionId: 'id-2', - role: 'assistant', - }, - data: [], - replaceLastMessage: false, - }, - ]); + expect(updateCalls).toMatchSnapshot(); }); it('should call the onFinish function with the correct arguments', async () => { - expect(finishCalls).toStrictEqual([ - { - message: { - content: 'Hello, world!', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - role: 'assistant', - }, - finishReason: 'stop', - usage: { - completionTokens: 5, - promptTokens: 10, - totalTokens: 15, - }, - }, - ]); + expect(finishCalls).toMatchSnapshot(); }); }); @@ -157,109 +119,11 @@ describe('scenario: server-side tool roundtrip', () => { }); it('should call the update function with the correct arguments', async () => { - expect(updateCalls).toStrictEqual([ - { - message: { - id: 'id-0', - revisionId: 'id-1', - role: 'assistant', - content: '', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - toolInvocations: [ - { - args: { - city: 'London', - }, - state: 'call', - toolCallId: 'tool-call-id', - toolName: 'tool-name', - step: 0, - }, - ], - }, - data: [], - replaceLastMessage: false, - }, - { - message: { - id: 'id-0', - revisionId: 'id-2', - role: 'assistant', - content: '', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - toolInvocations: [ - { - args: { - city: 'London', - }, - result: { - weather: 'sunny', - }, - state: 'result', - toolCallId: 'tool-call-id', - toolName: 'tool-name', - step: 0, - }, - ], - }, - data: [], - replaceLastMessage: false, - }, - { - message: { - id: 'id-0', - revisionId: 'id-3', - role: 'assistant', - content: 'The weather in London is sunny.', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - toolInvocations: [ - { - args: { - city: 'London', - }, - result: { - weather: 'sunny', - }, - state: 'result', - toolCallId: 'tool-call-id', - toolName: 'tool-name', - step: 0, - }, - ], - }, - data: [], - replaceLastMessage: false, - }, - ]); + expect(updateCalls).toMatchSnapshot(); }); it('should call the onFinish function with the correct arguments', async () => { - expect(finishCalls).toStrictEqual([ - { - message: { - id: 'id-0', - role: 'assistant', - content: 'The weather in London is sunny.', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - toolInvocations: [ - { - args: { city: 'London' }, - result: { weather: 'sunny' }, - state: 'result', - step: 0, - toolCallId: 'tool-call-id', - toolName: 'tool-name', - }, - ], - }, - finishReason: 'stop', - usage: { - completionTokens: 7, - promptTokens: 14, - totalTokens: 21, - }, - }, - ]); + expect(finishCalls).toMatchSnapshot(); }); }); @@ -314,84 +178,10 @@ describe('scenario: server-side tool roundtrip with existing assistant message', toolName: 'tool-name-original', }, ], - }, - }); - }); - - it('should call the update function with the correct arguments', async () => { - expect(updateCalls).toStrictEqual([ - { - message: { - id: 'original-id', - revisionId: 'id-0', - role: 'assistant', - content: '', - createdAt: new Date('2023-01-02T00:00:00.000Z'), - toolInvocations: [ - { - args: {}, - result: { location: 'Berlin' }, - state: 'result', - step: 0, - toolCallId: 'tool-call-id-original', - toolName: 'tool-name-original', - }, - { - args: { - city: 'London', - }, - state: 'call', - toolCallId: 'tool-call-id', - toolName: 'tool-name', - step: 1, - }, - ], - }, - data: [], - replaceLastMessage: true, - }, - { - message: { - id: 'original-id', - revisionId: 'id-1', - role: 'assistant', - content: '', - createdAt: new Date('2023-01-02T00:00:00.000Z'), - toolInvocations: [ - { - args: {}, - result: { location: 'Berlin' }, - state: 'result', - step: 0, - toolCallId: 'tool-call-id-original', - toolName: 'tool-name-original', - }, - { - args: { - city: 'London', - }, - result: { - weather: 'sunny', - }, - state: 'result', - toolCallId: 'tool-call-id', - toolName: 'tool-name', - step: 1, - }, - ], - }, - data: [], - replaceLastMessage: true, - }, - { - message: { - id: 'original-id', - revisionId: 'id-2', - role: 'assistant', - content: 'The weather in London is sunny.', - createdAt: new Date('2023-01-02T00:00:00.000Z'), - toolInvocations: [ - { + parts: [ + { + type: 'tool-invocation', + toolInvocation: { args: {}, result: { location: 'Berlin' }, state: 'result', @@ -399,61 +189,18 @@ describe('scenario: server-side tool roundtrip with existing assistant message', toolCallId: 'tool-call-id-original', toolName: 'tool-name-original', }, - { - args: { - city: 'London', - }, - result: { - weather: 'sunny', - }, - state: 'result', - toolCallId: 'tool-call-id', - toolName: 'tool-name', - step: 1, - }, - ], - }, - data: [], - replaceLastMessage: true, + }, + ], }, - ]); + }); + }); + + it('should call the update function with the correct arguments', async () => { + expect(updateCalls).toMatchSnapshot(); }); it('should call the onFinish function with the correct arguments', async () => { - expect(finishCalls).toStrictEqual([ - { - message: { - id: 'original-id', - role: 'assistant', - content: 'The weather in London is sunny.', - createdAt: new Date('2023-01-02T00:00:00.000Z'), - toolInvocations: [ - { - args: {}, - result: { location: 'Berlin' }, - state: 'result', - step: 0, - toolCallId: 'tool-call-id-original', - toolName: 'tool-name-original', - }, - { - args: { city: 'London' }, - result: { weather: 'sunny' }, - state: 'result', - step: 1, - toolCallId: 'tool-call-id', - toolName: 'tool-name', - }, - ], - }, - finishReason: 'stop', - usage: { - completionTokens: 7, - promptTokens: 14, - totalTokens: 21, - }, - }, - ]); + expect(finishCalls).toMatchSnapshot(); }); }); @@ -489,49 +236,11 @@ describe('scenario: server-side continue roundtrip', () => { }); it('should call the update function with the correct arguments', async () => { - expect(updateCalls).toStrictEqual([ - { - message: { - id: 'id-0', - revisionId: 'id-1', - role: 'assistant', - content: 'The weather in London ', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - }, - data: [], - replaceLastMessage: false, - }, - { - message: { - id: 'id-0', - revisionId: 'id-2', - role: 'assistant', - content: 'The weather in London is sunny.', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - }, - data: [], - replaceLastMessage: false, - }, - ]); + expect(updateCalls).toMatchSnapshot(); }); it('should call the onFinish function with the correct arguments', async () => { - expect(finishCalls).toStrictEqual([ - { - message: { - id: 'id-0', - role: 'assistant', - content: 'The weather in London is sunny.', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - }, - finishReason: 'stop', - usage: { - completionTokens: 7, - promptTokens: 14, - totalTokens: 21, - }, - }, - ]); + expect(finishCalls).toMatchSnapshot(); }); }); @@ -567,51 +276,11 @@ describe('scenario: delayed message annotations in onFinish', () => { }); it('should call the update function with the correct arguments', async () => { - expect(updateCalls).toStrictEqual([ - { - message: { - content: 'text', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - revisionId: 'id-1', - role: 'assistant', - }, - data: [], - replaceLastMessage: false, - }, - { - message: { - content: 'text', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - revisionId: 'id-2', - role: 'assistant', - annotations: [{ example: 'annotation' }], - }, - data: [], - replaceLastMessage: false, - }, - ]); + expect(updateCalls).toMatchSnapshot(); }); it('should call the onFinish function with the correct arguments', async () => { - expect(finishCalls).toStrictEqual([ - { - message: { - content: 'text', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - role: 'assistant', - annotations: [{ example: 'annotation' }], - }, - finishReason: 'stop', - usage: { - completionTokens: 5, - promptTokens: 10, - totalTokens: 15, - }, - }, - ]); + expect(finishCalls).toMatchSnapshot(); }); }); @@ -644,76 +313,11 @@ describe('scenario: message annotations in onChunk', () => { }); it('should call the update function with the correct arguments', async () => { - expect(updateCalls).toStrictEqual([ - { - message: { - content: '', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - revisionId: 'id-1', - role: 'assistant', - annotations: ['annotation1'], - }, - data: [], - replaceLastMessage: false, - }, - { - message: { - content: 't1', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - revisionId: 'id-2', - role: 'assistant', - annotations: ['annotation1'], - }, - data: [], - replaceLastMessage: false, - }, - { - message: { - content: 't1', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - revisionId: 'id-3', - role: 'assistant', - annotations: ['annotation1', 'annotation2'], - }, - data: [], - replaceLastMessage: false, - }, - { - message: { - content: 't1t2', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - revisionId: 'id-4', - role: 'assistant', - annotations: ['annotation1', 'annotation2'], - }, - data: [], - replaceLastMessage: false, - }, - ]); + expect(updateCalls).toMatchSnapshot(); }); it('should call the onFinish function with the correct arguments', async () => { - expect(finishCalls).toStrictEqual([ - { - message: { - content: 't1t2', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - role: 'assistant', - annotations: ['annotation1', 'annotation2'], - }, - finishReason: 'stop', - usage: { - completionTokens: 5, - promptTokens: 10, - totalTokens: 15, - }, - }, - ]); + expect(finishCalls).toMatchSnapshot(); }); }); @@ -745,57 +349,17 @@ describe('scenario: message annotations with existing assistant lastMessage', () createdAt: new Date('2023-01-02T00:00:00.000Z'), content: '', annotations: ['annotation0'], + parts: [], }, }); }); it('should call the update function with the correct arguments', async () => { - expect(updateCalls).toStrictEqual([ - { - message: { - content: '', - createdAt: new Date('2023-01-02T00:00:00.000Z'), - id: 'original-id', - revisionId: 'id-0', - role: 'assistant', - annotations: ['annotation0', 'annotation1'], - }, - data: [], - replaceLastMessage: true, - }, - { - message: { - content: 't1', - createdAt: new Date('2023-01-02T00:00:00.000Z'), - id: 'original-id', - revisionId: 'id-1', - role: 'assistant', - annotations: ['annotation0', 'annotation1'], - }, - data: [], - replaceLastMessage: true, - }, - ]); + expect(updateCalls).toMatchSnapshot(); }); it('should call the onFinish function with the correct arguments', async () => { - expect(finishCalls).toStrictEqual([ - { - message: { - content: 't1', - createdAt: new Date('2023-01-02T00:00:00.000Z'), - id: 'original-id', - role: 'assistant', - annotations: ['annotation0', 'annotation1'], - }, - finishReason: 'stop', - usage: { - completionTokens: 5, - promptTokens: 10, - totalTokens: 15, - }, - }, - ]); + expect(finishCalls).toMatchSnapshot(); }); }); @@ -845,148 +409,11 @@ describe('scenario: tool call streaming', () => { }); it('should call the update function with the correct arguments', async () => { - expect(updateCalls).toStrictEqual([ - { - message: { - content: '', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - revisionId: 'id-1', - role: 'assistant', - toolInvocations: [ - { - state: 'partial-call', - step: 0, - toolCallId: 'tool-call-0', - toolName: 'test-tool', - args: undefined, - }, - ], - }, - data: [], - replaceLastMessage: false, - }, - { - message: { - content: '', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - revisionId: 'id-2', - role: 'assistant', - toolInvocations: [ - { - args: { - testArg: 't', - }, - state: 'partial-call', - step: 0, - toolCallId: 'tool-call-0', - toolName: 'test-tool', - }, - ], - }, - data: [], - replaceLastMessage: false, - }, - { - message: { - content: '', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - revisionId: 'id-3', - role: 'assistant', - toolInvocations: [ - { - args: { - testArg: 'test-value', - }, - state: 'partial-call', - step: 0, - toolCallId: 'tool-call-0', - toolName: 'test-tool', - }, - ], - }, - data: [], - replaceLastMessage: false, - }, - { - message: { - content: '', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - revisionId: 'id-4', - role: 'assistant', - toolInvocations: [ - { - args: { - testArg: 'test-value', - }, - state: 'call', - toolCallId: 'tool-call-0', - toolName: 'test-tool', - step: 0, - }, - ], - }, - data: [], - replaceLastMessage: false, - }, - { - message: { - content: '', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - revisionId: 'id-5', - role: 'assistant', - toolInvocations: [ - { - args: { - testArg: 'test-value', - }, - result: 'test-result', - state: 'result', - toolCallId: 'tool-call-0', - toolName: 'test-tool', - step: 0, - }, - ], - }, - data: [], - replaceLastMessage: false, - }, - ]); + expect(updateCalls).toMatchSnapshot(); }); it('should call the onFinish function with the correct arguments', async () => { - expect(finishCalls).toStrictEqual([ - { - message: { - content: '', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'id-0', - role: 'assistant', - toolInvocations: [ - { - args: { - testArg: 'test-value', - }, - result: 'test-result', - state: 'result', - toolCallId: 'tool-call-0', - toolName: 'test-tool', - step: 0, - }, - ], - }, - finishReason: 'stop', - usage: { - completionTokens: 5, - promptTokens: 10, - totalTokens: 15, - }, - }, - ]); + expect(finishCalls).toMatchSnapshot(); }); }); @@ -1018,49 +445,11 @@ describe('scenario: server provides message ids', () => { }); it('should call the update function with the correct arguments', async () => { - expect(updateCalls).toStrictEqual([ - { - message: { - content: 'Hello, ', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'step_123', - revisionId: 'id-1', - role: 'assistant', - }, - data: [], - replaceLastMessage: false, - }, - { - message: { - content: 'Hello, world!', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'step_123', - revisionId: 'id-2', - role: 'assistant', - }, - data: [], - replaceLastMessage: false, - }, - ]); + expect(updateCalls).toMatchSnapshot(); }); it('should call the onFinish function with the correct arguments', async () => { - expect(finishCalls).toStrictEqual([ - { - message: { - content: 'Hello, world!', - createdAt: new Date('2023-01-01T00:00:00.000Z'), - id: 'step_123', - role: 'assistant', - }, - finishReason: 'stop', - usage: { - completionTokens: 5, - promptTokens: 10, - totalTokens: 15, - }, - }, - ]); + expect(finishCalls).toMatchSnapshot(); }); }); From 59df9ccbfdd7f095a94d4a00f785b49ebcce2e5b Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 12:45:09 +0100 Subject: [PATCH 06/61] 3 --- .../process-chat-text-response.test.ts.snap | 126 ++++++++++++++++++ packages/ui-utils/src/call-chat-api.ts | 31 +---- .../src/process-chat-text-response.test.ts | 110 +++++++++++++++ .../src/process-chat-text-response.ts | 50 +++++++ .../src/test/create-data-protocol-stream.ts | 5 +- 5 files changed, 293 insertions(+), 29 deletions(-) create mode 100644 packages/ui-utils/src/__snapshots__/process-chat-text-response.test.ts.snap create mode 100644 packages/ui-utils/src/process-chat-text-response.test.ts create mode 100644 packages/ui-utils/src/process-chat-text-response.ts diff --git a/packages/ui-utils/src/__snapshots__/process-chat-text-response.test.ts.snap b/packages/ui-utils/src/__snapshots__/process-chat-text-response.test.ts.snap new file mode 100644 index 000000000000..52fa31f2ef31 --- /dev/null +++ b/packages/ui-utils/src/__snapshots__/process-chat-text-response.test.ts.snap @@ -0,0 +1,126 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`processChatTextResponse > scenario: multiple short chunks > should call the onFinish function after the stream ends 1`] = ` +[ + { + "content": "ABCDE", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "test-id", + "parts": [], + "role": "assistant", + }, +] +`; + +exports[`processChatTextResponse > scenario: multiple short chunks > should call the update function with correct arguments for each chunk 1`] = ` +[ + { + "data": [], + "message": { + "content": "A", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "test-id", + "parts": [], + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "AB", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "test-id", + "parts": [], + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "ABC", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "test-id", + "parts": [], + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "ABCD", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "test-id", + "parts": [], + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "ABCDE", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "test-id", + "parts": [], + "role": "assistant", + }, + "replaceLastMessage": false, + }, +] +`; + +exports[`processChatTextResponse > scenario: no text chunks > should call the onFinish function after the stream ends 1`] = ` +[ + { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "test-id", + "parts": [], + "role": "assistant", + }, +] +`; + +exports[`processChatTextResponse > scenario: no text chunks > should call the update function with correct arguments for each chunk 1`] = `[]`; + +exports[`processChatTextResponse > scenario: simple text response > should call the onFinish function after the stream ends 1`] = ` +[ + { + "content": "Hello, world!", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "test-id", + "parts": [], + "role": "assistant", + }, +] +`; + +exports[`processChatTextResponse > scenario: simple text response > should call the update function with correct arguments for each chunk 1`] = ` +[ + { + "data": [], + "message": { + "content": "Hello, ", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "test-id", + "parts": [], + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "Hello, world!", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "test-id", + "parts": [], + "role": "assistant", + }, + "replaceLastMessage": false, + }, +] +`; diff --git a/packages/ui-utils/src/call-chat-api.ts b/packages/ui-utils/src/call-chat-api.ts index bd3a7f91418e..1ed6d40b708f 100644 --- a/packages/ui-utils/src/call-chat-api.ts +++ b/packages/ui-utils/src/call-chat-api.ts @@ -1,5 +1,5 @@ import { processChatResponse } from './process-chat-response'; -import { processTextStream } from './process-text-stream'; +import { processChatTextResponse } from './process-chat-text-response'; import { IdGenerator, JSONValue, Message, UseChatOptions } from './types'; // use function to allow for mocking in tests: @@ -75,32 +75,11 @@ export async function callChatApi({ switch (streamProtocol) { case 'text': { - const resultMessage: Message = { - id: generateId(), - createdAt: new Date(), - role: 'assistant' as const, - content: '', - parts: [], - }; - - await processTextStream({ + await processChatTextResponse({ stream: response.body, - onTextPart: chunk => { - resultMessage.content += chunk; - - // note: creating a new message object is required for Solid.js streaming - onUpdate({ - message: { ...resultMessage }, - data: [], - replaceLastMessage: false, - }); - }, - }); - - // in text mode, we don't have usage information or finish reason: - onFinish?.(resultMessage, { - usage: { completionTokens: NaN, promptTokens: NaN, totalTokens: NaN }, - finishReason: 'unknown', + update: onUpdate, + onFinish, + generateId, }); return; } diff --git a/packages/ui-utils/src/process-chat-text-response.test.ts b/packages/ui-utils/src/process-chat-text-response.test.ts new file mode 100644 index 000000000000..8a13ae201a88 --- /dev/null +++ b/packages/ui-utils/src/process-chat-text-response.test.ts @@ -0,0 +1,110 @@ +import { convertArrayToReadableStream } from '@ai-sdk/provider-utils/test'; +import { beforeEach, describe, expect, it } from 'vitest'; +import { processChatTextResponse } from './process-chat-text-response'; +import { Message } from './types'; + +function createTextStream(chunks: string[]): ReadableStream { + return convertArrayToReadableStream(chunks).pipeThrough( + new TextEncoderStream(), + ); +} + +let updateCalls: Array<{ + message: Message; + data: any[] | undefined; + replaceLastMessage: boolean; +}> = []; + +let finishCallMessages: Message[] = []; + +const update = (options: { + message: Message; + data: any[] | undefined; + replaceLastMessage: boolean; +}) => { + // clone to preserve the original object + updateCalls.push(structuredClone(options)); +}; + +const onFinish = (message: Message) => { + // store the final message + finishCallMessages.push(structuredClone(message)); +}; + +function mockId(): string { + // a simple predictable ID generator + return 'test-id'; +} + +beforeEach(() => { + updateCalls = []; + finishCallMessages = []; +}); + +describe('processChatTextResponse', () => { + describe('scenario: simple text response', () => { + beforeEach(async () => { + const stream = createTextStream(['Hello, ', 'world!']); + + await processChatTextResponse({ + stream, + update, + onFinish, + generateId: () => mockId(), + getCurrentDate: vi.fn().mockReturnValue(new Date('2023-01-01')), + }); + }); + + it('should call the update function with correct arguments for each chunk', () => { + expect(updateCalls).toMatchSnapshot(); + }); + + it('should call the onFinish function after the stream ends', () => { + expect(finishCallMessages).toMatchSnapshot(); + }); + }); + + describe('scenario: no text chunks', () => { + beforeEach(async () => { + const stream = createTextStream([]); + + await processChatTextResponse({ + stream, + update, + onFinish, + generateId: () => mockId(), + getCurrentDate: vi.fn().mockReturnValue(new Date('2023-01-01')), + }); + }); + + it('should call the update function with correct arguments for each chunk', () => { + expect(updateCalls).toMatchSnapshot(); + }); + + it('should call the onFinish function after the stream ends', () => { + expect(finishCallMessages).toMatchSnapshot(); + }); + }); + + describe('scenario: multiple short chunks', () => { + beforeEach(async () => { + const stream = createTextStream(['A', 'B', 'C', 'D', 'E']); + + await processChatTextResponse({ + stream, + update, + onFinish, + generateId: () => mockId(), + getCurrentDate: vi.fn().mockReturnValue(new Date('2023-01-01')), + }); + }); + + it('should call the update function with correct arguments for each chunk', () => { + expect(updateCalls).toMatchSnapshot(); + }); + + it('should call the onFinish function after the stream ends', () => { + expect(finishCallMessages).toMatchSnapshot(); + }); + }); +}); diff --git a/packages/ui-utils/src/process-chat-text-response.ts b/packages/ui-utils/src/process-chat-text-response.ts new file mode 100644 index 000000000000..d2c6fd58e9af --- /dev/null +++ b/packages/ui-utils/src/process-chat-text-response.ts @@ -0,0 +1,50 @@ +import { JSONValue } from '@ai-sdk/provider'; +import { generateId as generateIdFunction } from '@ai-sdk/provider-utils'; +import { processTextStream } from './process-text-stream'; +import { Message, UseChatOptions } from './types'; + +export async function processChatTextResponse({ + stream, + update, + onFinish, + getCurrentDate = () => new Date(), + generateId = generateIdFunction, +}: { + stream: ReadableStream; + update: (options: { + message: Message; + data: JSONValue[] | undefined; + replaceLastMessage: boolean; + }) => void; + onFinish: UseChatOptions['onFinish']; + getCurrentDate?: () => Date; + generateId?: () => string; +}) { + const resultMessage: Message = { + id: generateId(), + createdAt: getCurrentDate(), + role: 'assistant' as const, + content: '', + parts: [], + }; + + await processTextStream({ + stream, + onTextPart: chunk => { + resultMessage.content += chunk; + + // note: creating a new message object is required for Solid.js streaming + update({ + message: { ...resultMessage }, + data: [], + replaceLastMessage: false, + }); + }, + }); + + // in text mode, we don't have usage information or finish reason: + onFinish?.(resultMessage, { + usage: { completionTokens: NaN, promptTokens: NaN, totalTokens: NaN }, + finishReason: 'unknown', + }); +} diff --git a/packages/ui-utils/src/test/create-data-protocol-stream.ts b/packages/ui-utils/src/test/create-data-protocol-stream.ts index 54ea4ce7d06a..f58aaec05906 100644 --- a/packages/ui-utils/src/test/create-data-protocol-stream.ts +++ b/packages/ui-utils/src/test/create-data-protocol-stream.ts @@ -4,8 +4,7 @@ import { DataStreamString } from '../data-stream-parts'; export function createDataProtocolStream( dataPartTexts: DataStreamString[], ): ReadableStream { - const encoder = new TextEncoder(); - return convertArrayToReadableStream( - dataPartTexts.map(part => encoder.encode(part)), + return convertArrayToReadableStream(dataPartTexts).pipeThrough( + new TextEncoderStream(), ); } From b29d26fb31aaed15bc9e05867adafe339190132b Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 12:46:12 +0100 Subject: [PATCH 07/61] text part --- .../process-chat-text-response.test.ts.snap | 70 ++++++++++++++++--- .../src/process-chat-text-response.ts | 7 +- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/packages/ui-utils/src/__snapshots__/process-chat-text-response.test.ts.snap b/packages/ui-utils/src/__snapshots__/process-chat-text-response.test.ts.snap index 52fa31f2ef31..74f33e7f5b1f 100644 --- a/packages/ui-utils/src/__snapshots__/process-chat-text-response.test.ts.snap +++ b/packages/ui-utils/src/__snapshots__/process-chat-text-response.test.ts.snap @@ -6,7 +6,12 @@ exports[`processChatTextResponse > scenario: multiple short chunks > should call "content": "ABCDE", "createdAt": 2023-01-01T00:00:00.000Z, "id": "test-id", - "parts": [], + "parts": [ + { + "text": "ABCDE", + "type": "text", + }, + ], "role": "assistant", }, ] @@ -20,7 +25,12 @@ exports[`processChatTextResponse > scenario: multiple short chunks > should call "content": "A", "createdAt": 2023-01-01T00:00:00.000Z, "id": "test-id", - "parts": [], + "parts": [ + { + "text": "A", + "type": "text", + }, + ], "role": "assistant", }, "replaceLastMessage": false, @@ -31,7 +41,12 @@ exports[`processChatTextResponse > scenario: multiple short chunks > should call "content": "AB", "createdAt": 2023-01-01T00:00:00.000Z, "id": "test-id", - "parts": [], + "parts": [ + { + "text": "AB", + "type": "text", + }, + ], "role": "assistant", }, "replaceLastMessage": false, @@ -42,7 +57,12 @@ exports[`processChatTextResponse > scenario: multiple short chunks > should call "content": "ABC", "createdAt": 2023-01-01T00:00:00.000Z, "id": "test-id", - "parts": [], + "parts": [ + { + "text": "ABC", + "type": "text", + }, + ], "role": "assistant", }, "replaceLastMessage": false, @@ -53,7 +73,12 @@ exports[`processChatTextResponse > scenario: multiple short chunks > should call "content": "ABCD", "createdAt": 2023-01-01T00:00:00.000Z, "id": "test-id", - "parts": [], + "parts": [ + { + "text": "ABCD", + "type": "text", + }, + ], "role": "assistant", }, "replaceLastMessage": false, @@ -64,7 +89,12 @@ exports[`processChatTextResponse > scenario: multiple short chunks > should call "content": "ABCDE", "createdAt": 2023-01-01T00:00:00.000Z, "id": "test-id", - "parts": [], + "parts": [ + { + "text": "ABCDE", + "type": "text", + }, + ], "role": "assistant", }, "replaceLastMessage": false, @@ -78,7 +108,12 @@ exports[`processChatTextResponse > scenario: no text chunks > should call the on "content": "", "createdAt": 2023-01-01T00:00:00.000Z, "id": "test-id", - "parts": [], + "parts": [ + { + "text": "", + "type": "text", + }, + ], "role": "assistant", }, ] @@ -92,7 +127,12 @@ exports[`processChatTextResponse > scenario: simple text response > should call "content": "Hello, world!", "createdAt": 2023-01-01T00:00:00.000Z, "id": "test-id", - "parts": [], + "parts": [ + { + "text": "Hello, world!", + "type": "text", + }, + ], "role": "assistant", }, ] @@ -106,7 +146,12 @@ exports[`processChatTextResponse > scenario: simple text response > should call "content": "Hello, ", "createdAt": 2023-01-01T00:00:00.000Z, "id": "test-id", - "parts": [], + "parts": [ + { + "text": "Hello, ", + "type": "text", + }, + ], "role": "assistant", }, "replaceLastMessage": false, @@ -117,7 +162,12 @@ exports[`processChatTextResponse > scenario: simple text response > should call "content": "Hello, world!", "createdAt": 2023-01-01T00:00:00.000Z, "id": "test-id", - "parts": [], + "parts": [ + { + "text": "Hello, world!", + "type": "text", + }, + ], "role": "assistant", }, "replaceLastMessage": false, diff --git a/packages/ui-utils/src/process-chat-text-response.ts b/packages/ui-utils/src/process-chat-text-response.ts index d2c6fd58e9af..991aa38b3915 100644 --- a/packages/ui-utils/src/process-chat-text-response.ts +++ b/packages/ui-utils/src/process-chat-text-response.ts @@ -1,7 +1,7 @@ import { JSONValue } from '@ai-sdk/provider'; import { generateId as generateIdFunction } from '@ai-sdk/provider-utils'; import { processTextStream } from './process-text-stream'; -import { Message, UseChatOptions } from './types'; +import { Message, TextUIPart, UseChatOptions } from './types'; export async function processChatTextResponse({ stream, @@ -20,18 +20,21 @@ export async function processChatTextResponse({ getCurrentDate?: () => Date; generateId?: () => string; }) { + const textPart: TextUIPart = { type: 'text', text: '' }; + const resultMessage: Message = { id: generateId(), createdAt: getCurrentDate(), role: 'assistant' as const, content: '', - parts: [], + parts: [textPart], }; await processTextStream({ stream, onTextPart: chunk => { resultMessage.content += chunk; + textPart.text += chunk; // note: creating a new message object is required for Solid.js streaming update({ From c0118be11f62f5a3b610daea7f7eb0bc4d13cda9 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 13:22:51 +0100 Subject: [PATCH 08/61] append to response msgs --- .../append-response-messages.test.ts.snap | 355 ++++++++++++++++++ .../prompt/append-response-messages.test.ts | 218 ++++------- .../core/prompt/append-response-messages.ts | 63 +++- 3 files changed, 480 insertions(+), 156 deletions(-) create mode 100644 packages/ai/core/prompt/__snapshots__/append-response-messages.test.ts.snap diff --git a/packages/ai/core/prompt/__snapshots__/append-response-messages.test.ts.snap b/packages/ai/core/prompt/__snapshots__/append-response-messages.test.ts.snap new file mode 100644 index 000000000000..4a2603afcb72 --- /dev/null +++ b/packages/ai/core/prompt/__snapshots__/append-response-messages.test.ts.snap @@ -0,0 +1,355 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`appendResponseMessages > adds assistant text response to previous assistant message 1`] = ` +[ + { + "content": "User wants a tool invocation", + "createdAt": 1970-01-01T00:00:00.123Z, + "id": "1", + "parts": [ + { + "text": "User wants a tool invocation", + "type": "text", + }, + ], + "role": "user", + }, + { + "content": "This is a response from the assistant.", + "createdAt": 1970-01-01T00:00:00.456Z, + "id": "2", + "parts": [ + { + "toolInvocation": { + "args": { + "query": "some query", + }, + "result": { + "answer": "Tool result data", + }, + "state": "result", + "step": 0, + "toolCallId": "call-1", + "toolName": "some-tool", + }, + "type": "tool-invocation", + }, + { + "text": "This is a response from the assistant.", + "type": "text", + }, + ], + "role": "assistant", + "toolInvocations": [ + { + "args": { + "query": "some query", + }, + "result": { + "answer": "Tool result data", + }, + "state": "result", + "step": 0, + "toolCallId": "call-1", + "toolName": "some-tool", + }, + ], + }, +] +`; + +exports[`appendResponseMessages > adds assistant tool call response to previous assistant message 1`] = ` +[ + { + "content": "User wants a tool invocation", + "createdAt": 1970-01-01T00:00:00.123Z, + "id": "1", + "parts": [ + { + "text": "User wants a tool invocation", + "type": "text", + }, + ], + "role": "user", + }, + { + "content": "", + "createdAt": 1970-01-01T00:00:00.456Z, + "id": "2", + "parts": [ + { + "toolInvocation": { + "args": { + "query": "some query", + }, + "result": { + "answer": "Tool result data", + }, + "state": "result", + "step": 0, + "toolCallId": "call-1", + "toolName": "some-tool", + }, + "type": "tool-invocation", + }, + { + "toolInvocation": { + "args": { + "query": "another query", + }, + "state": "call", + "step": 1, + "toolCallId": "call-2", + "toolName": "some-tool", + }, + "type": "tool-invocation", + }, + ], + "role": "assistant", + "toolInvocations": [ + { + "args": { + "query": "some query", + }, + "result": { + "answer": "Tool result data", + }, + "state": "result", + "step": 0, + "toolCallId": "call-1", + "toolName": "some-tool", + }, + { + "args": { + "query": "another query", + }, + "state": "call", + "step": 1, + "toolCallId": "call-2", + "toolName": "some-tool", + }, + ], + }, +] +`; + +exports[`appendResponseMessages > adds chain of assistant messages and tool results 1`] = ` +[ + { + "content": "User wants a tool invocation", + "createdAt": 1970-01-01T00:00:00.123Z, + "id": "1", + "parts": [ + { + "text": "User wants a tool invocation", + "type": "text", + }, + ], + "role": "user", + }, + { + "content": "response", + "createdAt": 1970-01-01T00:00:00.789Z, + "id": "2", + "parts": [ + { + "toolInvocation": { + "args": { + "query": "some query", + }, + "result": { + "answer": "Tool result data", + }, + "state": "result", + "step": 0, + "toolCallId": "call-1", + "toolName": "some-tool", + }, + "type": "tool-invocation", + }, + { + "toolInvocation": { + "args": { + "query": "another query", + }, + "result": { + "answer": "another result", + }, + "state": "result", + "step": 1, + "toolCallId": "call-2", + "toolName": "some-tool", + }, + "type": "tool-invocation", + }, + { + "text": "response", + "type": "text", + }, + ], + "role": "assistant", + "toolInvocations": [ + { + "args": { + "query": "some query", + }, + "result": { + "answer": "Tool result data", + }, + "state": "result", + "step": 0, + "toolCallId": "call-1", + "toolName": "some-tool", + }, + { + "args": { + "query": "another query", + }, + "result": { + "answer": "another result", + }, + "state": "result", + "step": 1, + "toolCallId": "call-2", + "toolName": "some-tool", + }, + ], + }, +] +`; + +exports[`appendResponseMessages > adds tool results to the previously invoked tool calls (assistant message) 1`] = ` +[ + { + "content": "User wants a tool invocation", + "createdAt": 1970-01-01T00:00:00.123Z, + "id": "1", + "parts": [ + { + "text": "User wants a tool invocation", + "type": "text", + }, + ], + "role": "user", + }, + { + "content": "Placeholder text", + "createdAt": 1970-01-01T00:00:00.456Z, + "id": "2", + "parts": [ + { + "toolInvocation": { + "args": { + "query": "some query", + }, + "result": { + "answer": "Tool result data", + }, + "state": "result", + "step": 0, + "toolCallId": "call-1", + "toolName": "some-tool", + }, + "type": "tool-invocation", + }, + ], + "role": "assistant", + "toolInvocations": [ + { + "args": { + "query": "some query", + }, + "result": { + "answer": "Tool result data", + }, + "state": "result", + "step": 0, + "toolCallId": "call-1", + "toolName": "some-tool", + }, + ], + }, +] +`; + +exports[`appendResponseMessages > appends assistant messages with text content 1`] = ` +[ + { + "content": "Hello!", + "createdAt": 1970-01-01T00:00:00.123Z, + "id": "1", + "parts": [ + { + "text": "Hello!", + "type": "text", + }, + ], + "role": "user", + }, + { + "content": "This is a response from the assistant.", + "createdAt": 1970-01-01T00:00:00.789Z, + "id": "123", + "parts": [ + { + "text": "This is a response from the assistant.", + "type": "text", + }, + ], + "role": "assistant", + "toolInvocations": [], + }, +] +`; + +exports[`appendResponseMessages > handles tool calls and marks them as "call" initially 1`] = ` +[ + { + "content": "User wants a tool invocation", + "createdAt": 1970-01-01T00:00:00.123Z, + "id": "1", + "parts": [ + { + "text": "User wants a tool invocation", + "type": "text", + }, + ], + "role": "user", + }, + { + "content": "Processing tool call...", + "createdAt": 1970-01-01T00:00:00.789Z, + "id": "123", + "parts": [ + { + "text": "Processing tool call...", + "type": "text", + }, + { + "toolInvocation": { + "args": { + "query": "some query", + }, + "state": "call", + "step": 0, + "toolCallId": "call-1", + "toolName": "some-tool", + }, + "type": "tool-invocation", + }, + ], + "role": "assistant", + "toolInvocations": [ + { + "args": { + "query": "some query", + }, + "state": "call", + "step": 0, + "toolCallId": "call-1", + "toolName": "some-tool", + }, + ], + }, +] +`; diff --git a/packages/ai/core/prompt/append-response-messages.test.ts b/packages/ai/core/prompt/append-response-messages.test.ts index b1143fd60ee1..b2fc17b478c9 100644 --- a/packages/ai/core/prompt/append-response-messages.test.ts +++ b/packages/ai/core/prompt/append-response-messages.test.ts @@ -10,6 +10,7 @@ describe('appendResponseMessages', () => { id: '1', content: 'Hello!', createdAt: new Date(123), + parts: [{ type: 'text', text: 'Hello!' }], }, ], responseMessages: [ @@ -19,23 +20,12 @@ describe('appendResponseMessages', () => { id: '123', }, ], + _internal: { + currentDate: () => new Date(789), + }, }); - expect(result).toStrictEqual([ - { - role: 'user', - id: '1', - content: 'Hello!', - createdAt: new Date(123), - }, - { - role: 'assistant', - content: 'This is a response from the assistant.', - id: '123', - createdAt: expect.any(Date), - toolInvocations: [], - }, - ]); + expect(result).toMatchSnapshot(); }); it('adds assistant text response to previous assistant message', () => { @@ -46,6 +36,7 @@ describe('appendResponseMessages', () => { id: '1', content: 'User wants a tool invocation', createdAt: new Date(123), + parts: [{ type: 'text', text: 'User wants a tool invocation' }], }, { role: 'assistant', @@ -62,6 +53,19 @@ describe('appendResponseMessages', () => { step: 0, }, ], + parts: [ + { + type: 'tool-invocation', + toolInvocation: { + toolCallId: 'call-1', + toolName: 'some-tool', + state: 'result', + args: { query: 'some query' }, + result: { answer: 'Tool result data' }, + step: 0, + }, + }, + ], }, ], responseMessages: [ @@ -71,32 +75,12 @@ describe('appendResponseMessages', () => { id: '123', }, ], + _internal: { + currentDate: () => new Date(789), + }, }); - expect(result).toStrictEqual([ - { - role: 'user', - id: '1', - content: 'User wants a tool invocation', - createdAt: new Date(123), - }, - { - role: 'assistant', - content: 'This is a response from the assistant.', - id: '2', - createdAt: new Date(456), - toolInvocations: [ - { - state: 'result', - toolCallId: 'call-1', - toolName: 'some-tool', - args: { query: 'some query' }, - result: { answer: 'Tool result data' }, - step: 0, - }, - ], - }, - ]); + expect(result).toMatchSnapshot(); }); it('adds assistant tool call response to previous assistant message', () => { @@ -107,6 +91,7 @@ describe('appendResponseMessages', () => { id: '1', content: 'User wants a tool invocation', createdAt: new Date(123), + parts: [{ type: 'text', text: 'User wants a tool invocation' }], }, { role: 'assistant', @@ -123,6 +108,19 @@ describe('appendResponseMessages', () => { step: 0, }, ], + parts: [ + { + type: 'tool-invocation', + toolInvocation: { + toolCallId: 'call-1', + toolName: 'some-tool', + state: 'result', + args: { query: 'some query' }, + result: { answer: 'Tool result data' }, + step: 0, + }, + }, + ], }, ], responseMessages: [ @@ -139,39 +137,12 @@ describe('appendResponseMessages', () => { id: '123', }, ], + _internal: { + currentDate: () => new Date(789), + }, }); - expect(result).toStrictEqual([ - { - role: 'user', - id: '1', - content: 'User wants a tool invocation', - createdAt: new Date(123), - }, - { - role: 'assistant', - content: '', - id: '2', - createdAt: new Date(456), - toolInvocations: [ - { - state: 'result', - toolCallId: 'call-1', - toolName: 'some-tool', - args: { query: 'some query' }, - result: { answer: 'Tool result data' }, - step: 0, - }, - { - state: 'call', - toolCallId: 'call-2', - toolName: 'some-tool', - args: { query: 'another query' }, - step: 1, - }, - ], - }, - ]); + expect(result).toMatchSnapshot(); }); it('handles tool calls and marks them as "call" initially', () => { @@ -182,6 +153,7 @@ describe('appendResponseMessages', () => { id: '1', content: 'User wants a tool invocation', createdAt: new Date(123), + parts: [{ type: 'text', text: 'User wants a tool invocation' }], }, ], responseMessages: [ @@ -199,31 +171,12 @@ describe('appendResponseMessages', () => { id: '123', }, ], + _internal: { + currentDate: () => new Date(789), + }, }); - expect(result).toStrictEqual([ - { - role: 'user', - id: '1', - content: 'User wants a tool invocation', - createdAt: new Date(123), - }, - { - role: 'assistant', - content: 'Processing tool call...', - id: '123', - createdAt: expect.any(Date), - toolInvocations: [ - { - state: 'call', - toolCallId: 'call-1', - toolName: 'some-tool', - args: { query: 'some query' }, - step: 0, - }, - ], - }, - ]); + expect(result).toMatchSnapshot(); }); it('adds tool results to the previously invoked tool calls (assistant message)', () => { @@ -234,6 +187,7 @@ describe('appendResponseMessages', () => { id: '1', content: 'User wants a tool invocation', createdAt: new Date(123), + parts: [{ type: 'text', text: 'User wants a tool invocation' }], }, { role: 'assistant', @@ -249,6 +203,18 @@ describe('appendResponseMessages', () => { step: 0, }, ], + parts: [ + { + type: 'tool-invocation', + toolInvocation: { + toolCallId: 'call-1', + toolName: 'some-tool', + state: 'call', + args: { query: 'some query' }, + step: 0, + }, + }, + ], }, ], responseMessages: [ @@ -265,32 +231,12 @@ describe('appendResponseMessages', () => { ], }, ], + _internal: { + currentDate: () => new Date(789), + }, }); - expect(result).toStrictEqual([ - { - role: 'user', - id: '1', - content: 'User wants a tool invocation', - createdAt: new Date(123), - }, - { - role: 'assistant', - content: 'Placeholder text', - id: '2', - createdAt: new Date(456), - toolInvocations: [ - { - state: 'result', - toolCallId: 'call-1', - toolName: 'some-tool', - args: { query: 'some query' }, - result: { answer: 'Tool result data' }, - step: 0, - }, - ], - }, - ]); + expect(result).toMatchSnapshot(); }); it('adds chain of assistant messages and tool results', () => { @@ -301,6 +247,7 @@ describe('appendResponseMessages', () => { id: '1', content: 'User wants a tool invocation', createdAt: new Date(123), + parts: [{ type: 'text', text: 'User wants a tool invocation' }], }, ], responseMessages: [ @@ -358,40 +305,12 @@ describe('appendResponseMessages', () => { id: '6', }, ], + _internal: { + currentDate: () => new Date(789), + }, }); - expect(result).toStrictEqual([ - { - role: 'user', - id: '1', - content: 'User wants a tool invocation', - createdAt: new Date(123), - }, - { - role: 'assistant', - content: 'response', - id: '2', - createdAt: expect.any(Date), - toolInvocations: [ - { - state: 'result', - toolCallId: 'call-1', - toolName: 'some-tool', - args: { query: 'some query' }, - result: { answer: 'Tool result data' }, - step: 0, - }, - { - state: 'result', - toolCallId: 'call-2', - toolName: 'some-tool', - args: { query: 'another query' }, - result: { answer: 'another result' }, - step: 1, - }, - ], - }, - ]); + expect(result).toMatchSnapshot(); }); it('throws an error if a tool result follows a non-assistant message', () => { @@ -403,6 +322,7 @@ describe('appendResponseMessages', () => { id: '1', content: 'User message', createdAt: new Date(), + parts: [{ type: 'text', text: 'User message' }], }, ], responseMessages: [ diff --git a/packages/ai/core/prompt/append-response-messages.ts b/packages/ai/core/prompt/append-response-messages.ts index 9f4bda63a9a4..7ac26abf099c 100644 --- a/packages/ai/core/prompt/append-response-messages.ts +++ b/packages/ai/core/prompt/append-response-messages.ts @@ -2,6 +2,7 @@ import { extractMaxToolInvocationStep, Message, ToolInvocation, + ToolInvocationUIPart, } from '@ai-sdk/ui-utils'; import { ResponseMessage } from '../generate-text/step-result'; @@ -15,9 +16,17 @@ import { ResponseMessage } from '../generate-text/step-result'; export function appendResponseMessages({ messages, responseMessages, + _internal: { currentDate = () => new Date() } = {}, }: { messages: Message[]; responseMessages: ResponseMessage[]; + + /** +@internal For test use only. May change without notice. + */ + _internal?: { + currentDate?: () => Date; + }; }): Message[] { const clonedMessages = structuredClone(messages); @@ -29,8 +38,7 @@ export function appendResponseMessages({ const isLastMessageAssistant = lastMessage.role === 'assistant'; switch (role) { - case 'assistant': { - // only include text in the content: + case 'assistant': // only include text in the content: const textContent = typeof message.content === 'string' ? message.content @@ -58,24 +66,50 @@ export function appendResponseMessages({ lastMessage.toolInvocations, ); + lastMessage.parts ??= []; + lastMessage.content = textContent; + if (textContent.length > 0) { + lastMessage.parts.push({ + type: 'text' as const, + text: textContent, + }); + } + lastMessage.toolInvocations = [ ...(lastMessage.toolInvocations ?? []), ...getToolInvocations(maxStep === undefined ? 0 : maxStep + 1), ]; + + getToolInvocations(maxStep === undefined ? 0 : maxStep + 1) + .map(call => ({ + type: 'tool-invocation' as const, + toolInvocation: call, + })) + .forEach(part => { + lastMessage.parts.push(part); + }); } else { // last message was a user message, add the assistant message: clonedMessages.push({ role: 'assistant', id: message.id, - createdAt: new Date(), // generate a createdAt date for the message, will be overridden by the client + createdAt: currentDate(), // generate a createdAt date for the message, will be overridden by the client content: textContent, toolInvocations: getToolInvocations(0), + parts: [ + ...(textContent.length > 0 + ? [{ type: 'text' as const, text: textContent }] + : []), + ...getToolInvocations(0).map(call => ({ + type: 'tool-invocation' as const, + toolInvocation: call, + })), + ], }); } break; - } case 'tool': { // for tool call results, add the result to previous message: @@ -87,11 +121,17 @@ export function appendResponseMessages({ ); } - for (const part of message.content) { + for (const contentPart of message.content) { // find the tool call in the previous message: const toolCall = lastMessage.toolInvocations.find( - call => call.toolCallId === part.toolCallId, + call => call.toolCallId === contentPart.toolCallId, ); + const toolCallPart: ToolInvocationUIPart | undefined = + lastMessage.parts.find( + (part): part is ToolInvocationUIPart => + part.type === 'tool-invocation' && + part.toolInvocation.toolCallId === contentPart.toolCallId, + ); if (!toolCall) { throw new Error('Tool call not found in previous message'); @@ -100,7 +140,16 @@ export function appendResponseMessages({ // add the result to the tool call: toolCall.state = 'result'; const toolResult = toolCall as ToolInvocation & { state: 'result' }; - toolResult.result = part.result; + toolResult.result = contentPart.result; + + if (toolCallPart) { + toolCallPart.toolInvocation = toolResult; + } else { + lastMessage.parts.push({ + type: 'tool-invocation' as const, + toolInvocation: toolResult, + }); + } } break; From de16955e817f611f996e4f18fe98a651b2d90bbf Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 13:48:35 +0100 Subject: [PATCH 09/61] test --- .../process-chat-response.test.ts.snap | 89 +++++++++++++++++++ .../src/process-chat-response.test.ts | 39 ++++++++ 2 files changed, 128 insertions(+) diff --git a/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap b/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap index 0f421e7a72ff..12027670c51e 100644 --- a/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap +++ b/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap @@ -256,6 +256,95 @@ exports[`scenario: message annotations with existing assistant lastMessage > sho ] `; +exports[`scenario: onToolCall is executed > should call the onFinish function with the correct arguments 1`] = ` +[ + { + "finishReason": "stop", + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": "test-result", + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + ], + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "result": "test-result", + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "usage": { + "completionTokens": 5, + "promptTokens": 10, + "totalTokens": 15, + }, + }, +] +`; + +exports[`scenario: onToolCall is executed > should call the update function with the correct arguments 1`] = ` +[ + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": "test-result", + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + ], + "revisionId": "id-1", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "result": "test-result", + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": false, + }, +] +`; + exports[`scenario: server provides message ids > should call the onFinish function with the correct arguments 1`] = ` [ { diff --git a/packages/ui-utils/src/process-chat-response.test.ts b/packages/ui-utils/src/process-chat-response.test.ts index 186cf2adffbb..24077acdc588 100644 --- a/packages/ui-utils/src/process-chat-response.test.ts +++ b/packages/ui-utils/src/process-chat-response.test.ts @@ -495,3 +495,42 @@ describe('scenario: server provides reasoning', () => { expect(finishCalls).toMatchSnapshot(); }); }); + +describe('scenario: onToolCall is executed', () => { + beforeEach(async () => { + const stream = createDataProtocolStream([ + formatDataStreamPart('tool_call', { + toolCallId: 'tool-call-id', + toolName: 'tool-name', + args: { city: 'London' }, + }), + formatDataStreamPart('finish_step', { + finishReason: 'tool-calls', + usage: { completionTokens: 5, promptTokens: 10 }, + isContinued: false, + }), + formatDataStreamPart('finish_message', { + finishReason: 'stop', + usage: { completionTokens: 5, promptTokens: 10 }, + }), + ]); + + await processChatResponse({ + stream, + update, + onFinish, + generateId: mockId(), + getCurrentDate: vi.fn().mockReturnValue(new Date('2023-01-01')), + lastMessage: undefined, + onToolCall: vi.fn().mockResolvedValue('test-result'), + }); + }); + + it('should call the update function with the correct arguments', async () => { + expect(updateCalls).toMatchSnapshot(); + }); + + it('should call the onFinish function with the correct arguments', async () => { + expect(finishCalls).toMatchSnapshot(); + }); +}); From 94289d71094bb3f78ca7d8309ae9e36bf6e5a037 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 15:01:56 +0100 Subject: [PATCH 10/61] fix --- packages/react/src/use-chat.ui.test.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/react/src/use-chat.ui.test.tsx b/packages/react/src/use-chat.ui.test.tsx index 4c85389fd073..a5c581c28083 100644 --- a/packages/react/src/use-chat.ui.test.tsx +++ b/packages/react/src/use-chat.ui.test.tsx @@ -271,6 +271,7 @@ describe('data protocol stream', () => { createdAt: expect.any(Date), role: 'assistant', content: 'Hello, world.', + parts: [{ text: 'Hello, world.', type: 'text' }], }, options: { finishReason: 'stop', @@ -453,6 +454,7 @@ describe('text stream', () => { createdAt: expect.any(Date), role: 'assistant', content: 'Hello, world.', + parts: [{ text: 'Hello, world.', type: 'text' }], }, options: { finishReason: 'unknown', From 819e6733bbb8352f4b63f7fec1e02d2ce2465685 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 15:10:09 +0100 Subject: [PATCH 11/61] update-tool-call-result --- packages/react/src/use-chat.ts | 34 +++----------- packages/ui-utils/src/index.ts | 1 + .../ui-utils/src/update-tool-call-result.ts | 45 +++++++++++++++++++ 3 files changed, 53 insertions(+), 27 deletions(-) create mode 100644 packages/ui-utils/src/update-tool-call-result.ts diff --git a/packages/react/src/use-chat.ts b/packages/react/src/use-chat.ts index 63aa377020de..1045725c3d3f 100644 --- a/packages/react/src/use-chat.ts +++ b/packages/react/src/use-chat.ts @@ -4,7 +4,6 @@ import type { CreateMessage, JSONValue, Message, - ToolInvocationUIPart, UseChatOptions, } from '@ai-sdk/ui-utils'; import { @@ -12,6 +11,7 @@ import { extractMaxToolInvocationStep, generateId as generateIdFunc, prepareAttachmentsForRequest, + updateToolCallResult, } from '@ai-sdk/ui-utils'; import { useCallback, useEffect, useRef, useState } from 'react'; import useSWR from 'swr'; @@ -521,36 +521,16 @@ By default, it's set to 1, which means that only a single LLM call is made. const addToolResult = useCallback( ({ toolCallId, result }: { toolCallId: string; result: any }) => { - const lastMessage = messagesRef.current[messagesRef.current.length - 1]; - - const invocationPart = lastMessage.parts.find( - (part): part is ToolInvocationUIPart => - part.type === 'tool-invocation' && - part.toolInvocation.toolCallId === toolCallId, - ); - - if (invocationPart == null) { - return; - } - - const toolResult = { - ...invocationPart.toolInvocation, - state: 'result' as const, - result, - }; - - invocationPart.toolInvocation = toolResult; - - lastMessage.toolInvocations = lastMessage.toolInvocations?.map( - toolInvocation => - toolInvocation.toolCallId === toolCallId - ? toolResult - : toolInvocation, - ); + updateToolCallResult({ + messages: messagesRef.current, + toolCallId, + toolResult: result, + }); mutate(messagesRef.current, false); // auto-submit when all tool calls in the last assistant message have results: + const lastMessage = messagesRef.current[messagesRef.current.length - 1]; if (isAssistantMessageWithCompletedToolCalls(lastMessage)) { triggerRequest({ messages: messagesRef.current }); } diff --git a/packages/ui-utils/src/index.ts b/packages/ui-utils/src/index.ts index 8ca86a2bc4c4..a6277a407144 100644 --- a/packages/ui-utils/src/index.ts +++ b/packages/ui-utils/src/index.ts @@ -28,4 +28,5 @@ export { processDataStream } from './process-data-stream'; export { processTextStream } from './process-text-stream'; export { asSchema, jsonSchema } from './schema'; export type { Schema } from './schema'; +export { updateToolCallResult } from './update-tool-call-result'; export { zodSchema } from './zod-schema'; diff --git a/packages/ui-utils/src/update-tool-call-result.ts b/packages/ui-utils/src/update-tool-call-result.ts new file mode 100644 index 000000000000..5ff7d1b587eb --- /dev/null +++ b/packages/ui-utils/src/update-tool-call-result.ts @@ -0,0 +1,45 @@ +import { Message, ToolInvocationUIPart } from './types'; + +/** + * Updates the result of a specific tool invocation in the last message of the given messages array. + * + * @param {object} params - The parameters object. + * @param {Message[]} params.messages - An array of messages, from which the last one is updated. + * @param {string} params.toolCallId - The unique identifier for the tool invocation to update. + * @param {unknown} params.toolResult - The result object to attach to the tool invocation. + * @returns {void} This function does not return anything. + */ +export function updateToolCallResult({ + messages, + toolCallId, + toolResult: result, +}: { + messages: Message[]; + toolCallId: string; + toolResult: unknown; +}) { + const lastMessage = messages[messages.length - 1]; + + const invocationPart = lastMessage.parts.find( + (part): part is ToolInvocationUIPart => + part.type === 'tool-invocation' && + part.toolInvocation.toolCallId === toolCallId, + ); + + if (invocationPart == null) { + return; + } + + const toolResult = { + ...invocationPart.toolInvocation, + state: 'result' as const, + result, + }; + + invocationPart.toolInvocation = toolResult; + + lastMessage.toolInvocations = lastMessage.toolInvocations?.map( + toolInvocation => + toolInvocation.toolCallId === toolCallId ? toolResult : toolInvocation, + ); +} From 003780718e3da3ff98fbaff68abbd4af655d5428 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 15:13:21 +0100 Subject: [PATCH 12/61] fix --- packages/solid/src/use-assistant.ts | 16 +++++++--------- packages/svelte/src/use-assistant.ts | 7 ++++++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/solid/src/use-assistant.ts b/packages/solid/src/use-assistant.ts index 829e56880bd3..9705dcdf99a8 100644 --- a/packages/solid/src/use-assistant.ts +++ b/packages/solid/src/use-assistant.ts @@ -2,19 +2,14 @@ import { isAbortError } from '@ai-sdk/provider-utils'; import { AssistantStatus, CreateMessage, - Message, - UseAssistantOptions, generateId, + Message, processAssistantStream, + UseAssistantOptions, } from '@ai-sdk/ui-utils'; import { Accessor, createMemo, createSignal, JSX, Setter } from 'solid-js'; +import { createStore, SetStoreFunction, Store } from 'solid-js/store'; import { convertToAccessorOptions } from './utils/convert-to-accessor-options'; -import { - createStore, - SetStoreFunction, - Store, - StoreSetter, -} from 'solid-js/store'; // use function to allow for mocking in tests: const getOriginalFetch = () => fetch; @@ -189,6 +184,7 @@ export function useAssistant( id: value.id, role: value.role, content: value.content[0].text.value, + parts: [], }, ]); }, @@ -202,6 +198,7 @@ export function useAssistant( id: lastMessage.id, role: lastMessage.role, content: lastMessage.content + value, + parts: lastMessage.parts, }, ]; }); @@ -224,6 +221,7 @@ export function useAssistant( role: 'data', content: '', data: value.data, + parts: [], }, ]); }, @@ -262,7 +260,7 @@ export function useAssistant( return; } - append({ role: 'user', content: input() }, requestOptions); + append({ role: 'user', content: input(), parts: [] }, requestOptions); }; const setThreadId = (threadId: string | undefined) => { diff --git a/packages/svelte/src/use-assistant.ts b/packages/svelte/src/use-assistant.ts index 86c74a9a9bfc..d518f99c5f3a 100644 --- a/packages/svelte/src/use-assistant.ts +++ b/packages/svelte/src/use-assistant.ts @@ -152,6 +152,7 @@ export function useAssistant({ id: value.id, role: value.role, content: value.content[0].text.value, + parts: [], }, ]); }, @@ -186,6 +187,7 @@ export function useAssistant({ role: 'data', content: '', data: value.data, + parts: [], }, ]); }, @@ -231,7 +233,10 @@ export function useAssistant({ const inputValue = get(input); if (!inputValue) return; - await append({ role: 'user', content: inputValue }, requestOptions); + await append( + { role: 'user', content: inputValue, parts: [] }, + requestOptions, + ); } return { From 72abc1094f998742c431a001bd0125ecd24fb9fd Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 15:43:12 +0100 Subject: [PATCH 13/61] svelte --- packages/svelte/src/use-chat.ts | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/packages/svelte/src/use-chat.ts b/packages/svelte/src/use-chat.ts index 64a4d2cdef30..df99a21ae2fe 100644 --- a/packages/svelte/src/use-chat.ts +++ b/packages/svelte/src/use-chat.ts @@ -13,6 +13,7 @@ import { extractMaxToolInvocationStep, generateId as generateIdFunc, prepareAttachmentsForRequest, + updateToolCallResult, } from '@ai-sdk/ui-utils'; import { useSWR } from 'sswr'; import { Readable, Writable, derived, get, writable } from 'svelte/store'; @@ -463,33 +464,20 @@ export function useChat({ result: any; }) => { const messagesSnapshot = get(messages) ?? []; - const updatedMessages = messagesSnapshot.map((message, index, arr) => - // update the tool calls in the last assistant message: - index === arr.length - 1 && - message.role === 'assistant' && - message.toolInvocations - ? { - ...message, - toolInvocations: message.toolInvocations.map(toolInvocation => - toolInvocation.toolCallId === toolCallId - ? { - ...toolInvocation, - result, - state: 'result' as const, - } - : toolInvocation, - ), - } - : message, - ); - messages.set(updatedMessages); + updateToolCallResult({ + messages: messagesSnapshot, + toolCallId, + toolResult: result, + }); + + messages.set(messagesSnapshot); // auto-submit when all tool calls in the last assistant message have results: - const lastMessage = updatedMessages[updatedMessages.length - 1]; + const lastMessage = messagesSnapshot[messagesSnapshot.length - 1]; if (isAssistantMessageWithCompletedToolCalls(lastMessage)) { - triggerRequest({ messages: updatedMessages }); + triggerRequest({ messages: messagesSnapshot }); } }; From 9da69dd869a5c520b383fa355e9f80abf198fa02 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 15:47:06 +0100 Subject: [PATCH 14/61] solid --- packages/solid/src/use-chat.ts | 32 ++++++++++---------------------- packages/svelte/src/use-chat.ts | 3 ++- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/packages/solid/src/use-chat.ts b/packages/solid/src/use-chat.ts index 86c29ee56eb8..a54c4bf0c36f 100644 --- a/packages/solid/src/use-chat.ts +++ b/packages/solid/src/use-chat.ts @@ -13,6 +13,7 @@ import { extractMaxToolInvocationStep, generateId as generateIdFunc, prepareAttachmentsForRequest, + updateToolCallResult, } from '@ai-sdk/ui-utils'; import { Accessor, @@ -454,6 +455,7 @@ export function useChat( createdAt: new Date(), experimental_attachments: attachmentsForRequest.length > 0 ? attachmentsForRequest : undefined, + parts: [{ type: 'text', text: inputValue }], }), headers: options.headers, body: options.body, @@ -476,32 +478,18 @@ export function useChat( }) => { const messagesSnapshot = _messages() ?? []; - const updatedMessages = messagesSnapshot.map((message, index, arr) => - // update the tool calls in the last assistant message: - index === arr.length - 1 && - message.role === 'assistant' && - message.toolInvocations - ? { - ...message, - toolInvocations: message.toolInvocations.map(toolInvocation => - toolInvocation.toolCallId === toolCallId - ? { - ...toolInvocation, - result, - state: 'result' as const, - } - : toolInvocation, - ), - } - : message, - ); + updateToolCallResult({ + messages: messagesSnapshot, + toolCallId, + toolResult: result, + }); - mutate(updatedMessages); + mutate(messagesSnapshot); // auto-submit when all tool calls in the last assistant message have results: - const lastMessage = updatedMessages[updatedMessages.length - 1]; + const lastMessage = messagesSnapshot[messagesSnapshot.length - 1]; if (isAssistantMessageWithCompletedToolCalls(lastMessage)) { - triggerRequest({ messages: updatedMessages }); + triggerRequest({ messages: messagesSnapshot }); } }; diff --git a/packages/svelte/src/use-chat.ts b/packages/svelte/src/use-chat.ts index df99a21ae2fe..47c6287ee254 100644 --- a/packages/svelte/src/use-chat.ts +++ b/packages/svelte/src/use-chat.ts @@ -440,7 +440,8 @@ export function useChat({ createdAt: new Date(), experimental_attachments: attachmentsForRequest.length > 0 ? attachmentsForRequest : undefined, - } as Message), + parts: [{ type: 'text', text: inputValue }], + }), body: options.body, headers: options.headers, data: options.data, From f03ac6dc8d28808b56838a8e8cf20403e294f9e8 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 16:10:27 +0100 Subject: [PATCH 15/61] adjust --- packages/react/src/use-chat.ts | 74 ++++++++++++++----- packages/react/src/use-chat.ui.test.tsx | 1 + packages/solid/src/use-chat.ui.test.tsx | 10 +-- packages/ui-utils/src/call-chat-api.ts | 6 +- .../ui-utils/src/process-chat-response.ts | 12 +-- .../src/process-chat-text-response.ts | 6 +- packages/ui-utils/src/types.ts | 7 +- .../ui-utils/src/update-tool-call-result.ts | 6 +- 8 files changed, 80 insertions(+), 42 deletions(-) diff --git a/packages/react/src/use-chat.ts b/packages/react/src/use-chat.ts index 1045725c3d3f..a85dd0d0d820 100644 --- a/packages/react/src/use-chat.ts +++ b/packages/react/src/use-chat.ts @@ -4,6 +4,10 @@ import type { CreateMessage, JSONValue, Message, + ReasoningUIPart, + TextUIPart, + ToolInvocationUIPart, + UIMessage, UseChatOptions, } from '@ai-sdk/ui-utils'; import { @@ -21,7 +25,7 @@ export type { CreateMessage, Message, UseChatOptions }; export type UseChatHelpers = { /** Current messages in the chat */ - messages: Message[]; + messages: UIMessage[]; /** The error object of the API request */ error: undefined | Error; /** @@ -163,14 +167,19 @@ By default, it's set to 1, which means that only a single LLM call is made. const [initialMessagesFallback] = useState([]); // Store the chat state in SWR, using the chatId as the key to share states. - const { data: messages, mutate } = useSWR( + const { data: messages, mutate } = useSWR( [chatKey, 'messages'], null, - { fallbackData: initialMessages ?? initialMessagesFallback }, + { + fallbackData: + initialMessages != null + ? fillInUIMessageParts(initialMessages) + : initialMessagesFallback, + }, ); // Keep the latest messages in a ref. - const messagesRef = useRef(messages || []); + const messagesRef = useRef(messages || []); useEffect(() => { messagesRef.current = messages || []; }, [messages]); @@ -215,9 +224,11 @@ By default, it's set to 1, which means that only a single LLM call is made. const triggerRequest = useCallback( async (chatRequest: ChatRequest) => { - const messageCount = chatRequest.messages.length; + const chatMessages = fillInUIMessageParts(chatRequest.messages); + + const messageCount = chatMessages.length; const maxStep = extractMaxToolInvocationStep( - chatRequest.messages[chatRequest.messages.length - 1]?.toolInvocations, + chatMessages[chatMessages.length - 1]?.toolInvocations, ); try { @@ -235,11 +246,11 @@ By default, it's set to 1, which means that only a single LLM call is made. // Do an optimistic update to the chat state to show the updated messages immediately: const previousMessages = messagesRef.current; - throttledMutate(chatRequest.messages, false); + throttledMutate(chatMessages, false); const constructedMessagesPayload = sendExtraMessageFields - ? chatRequest.messages - : chatRequest.messages.map( + ? chatMessages + : chatMessages.map( ({ role, content, @@ -265,7 +276,7 @@ By default, it's set to 1, which means that only a single LLM call is made. api, body: experimental_prepareRequestBody?.({ id: chatId, - messages: chatRequest.messages, + messages: chatMessages, requestData: chatRequest.data, requestBody: chatRequest.body, }) ?? { @@ -292,11 +303,8 @@ By default, it's set to 1, which means that only a single LLM call is made. throttledMutate( [ ...(replaceLastMessage - ? chatRequest.messages.slice( - 0, - chatRequest.messages.length - 1, - ) - : chatRequest.messages), + ? chatMessages.slice(0, chatMessages.length - 1) + : chatMessages), message, ], false, @@ -313,7 +321,7 @@ By default, it's set to 1, which means that only a single LLM call is made. onFinish, generateId, fetch, - lastMessage: chatRequest.messages[chatRequest.messages.length - 1], + lastMessage: chatMessages[chatMessages.length - 1], }); abortControllerRef.current = null; @@ -403,6 +411,7 @@ By default, it's set to 1, which means that only a single LLM call is made. createdAt: message.createdAt ?? new Date(), experimental_attachments: attachmentsForRequest.length > 0 ? attachmentsForRequest : undefined, + parts: getMessageParts(message), }); return triggerRequest({ messages, headers, body, data }); @@ -444,8 +453,9 @@ By default, it's set to 1, which means that only a single LLM call is made. messages = messages(messagesRef.current); } - mutate(messages, false); - messagesRef.current = messages; + const messagesWithParts = fillInUIMessageParts(messages); + mutate(messagesWithParts, false); + messagesRef.current = messagesWithParts; }, [mutate], ); @@ -574,3 +584,31 @@ function isAssistantMessageWithCompletedToolCalls( message.toolInvocations.every(toolInvocation => 'result' in toolInvocation) ); } + +function fillInUIMessageParts(messages: Message[]): UIMessage[] { + return messages.map(message => ({ + ...message, + parts: getMessageParts(message), + })); +} + +function getMessageParts( + message: Message | CreateMessage | UIMessage, +): (TextUIPart | ReasoningUIPart | ToolInvocationUIPart)[] { + return ( + message.parts ?? [ + ...(message.reasoning + ? [{ type: 'reasoning' as const, reasoning: message.reasoning }] + : []), + ...(message.content + ? [{ type: 'text' as const, text: message.content }] + : []), + ...(message.toolInvocations + ? message.toolInvocations.map(toolInvocation => ({ + type: 'tool-invocation' as const, + toolInvocation, + })) + : []), + ] + ); +} diff --git a/packages/react/src/use-chat.ui.test.tsx b/packages/react/src/use-chat.ui.test.tsx index a5c581c28083..85bc3b6223a0 100644 --- a/packages/react/src/use-chat.ui.test.tsx +++ b/packages/react/src/use-chat.ui.test.tsx @@ -716,6 +716,7 @@ describe('prepareRequestBody', () => { id: expect.any(String), experimental_attachments: undefined, createdAt: expect.any(Date), + parts: [{ type: 'text', text: 'hi' }], }, ], requestData: { 'test-data-key': 'test-data-value' }, diff --git a/packages/solid/src/use-chat.ui.test.tsx b/packages/solid/src/use-chat.ui.test.tsx index ab6686eaabf7..2a0cdc185cf9 100644 --- a/packages/solid/src/use-chat.ui.test.tsx +++ b/packages/solid/src/use-chat.ui.test.tsx @@ -16,14 +16,8 @@ import { useChat } from './use-chat'; describe('file attachments with data url', () => { const TestComponent = () => { - const { - messages, - handleSubmit, - handleInputChange, - isLoading, - input, - setInput, - } = useChat(); + const { messages, handleSubmit, handleInputChange, isLoading, input } = + useChat(); const [attachments, setAttachments] = createSignal(); let fileInputRef: HTMLInputElement | undefined; diff --git a/packages/ui-utils/src/call-chat-api.ts b/packages/ui-utils/src/call-chat-api.ts index 1ed6d40b708f..80fc1636f8eb 100644 --- a/packages/ui-utils/src/call-chat-api.ts +++ b/packages/ui-utils/src/call-chat-api.ts @@ -1,6 +1,6 @@ import { processChatResponse } from './process-chat-response'; import { processChatTextResponse } from './process-chat-text-response'; -import { IdGenerator, JSONValue, Message, UseChatOptions } from './types'; +import { IdGenerator, JSONValue, UIMessage, UseChatOptions } from './types'; // use function to allow for mocking in tests: const getOriginalFetch = () => fetch; @@ -30,7 +30,7 @@ export async function callChatApi({ restoreMessagesOnFailure: () => void; onResponse: ((response: Response) => void | Promise) | undefined; onUpdate: (options: { - message: Message; + message: UIMessage; data: JSONValue[] | undefined; replaceLastMessage: boolean; }) => void; @@ -38,7 +38,7 @@ export async function callChatApi({ onToolCall: UseChatOptions['onToolCall']; generateId: IdGenerator; fetch: ReturnType | undefined; - lastMessage: Message | undefined; + lastMessage: UIMessage | undefined; }) { const response = await fetch(api, { method: 'POST', diff --git a/packages/ui-utils/src/process-chat-response.ts b/packages/ui-utils/src/process-chat-response.ts index 71c34b029e99..7c5f38fe10fd 100644 --- a/packages/ui-utils/src/process-chat-response.ts +++ b/packages/ui-utils/src/process-chat-response.ts @@ -3,11 +3,11 @@ import { parsePartialJson } from './parse-partial-json'; import { processDataStream } from './process-data-stream'; import type { JSONValue, - Message, ReasoningUIPart, TextUIPart, ToolInvocation, ToolInvocationUIPart, + UIMessage, UseChatOptions, } from './types'; import { LanguageModelV1FinishReason } from '@ai-sdk/provider'; @@ -27,19 +27,19 @@ export async function processChatResponse({ }: { stream: ReadableStream; update: (options: { - message: Message; + message: UIMessage; data: JSONValue[] | undefined; replaceLastMessage: boolean; }) => void; onToolCall?: UseChatOptions['onToolCall']; onFinish?: (options: { - message: Message | undefined; + message: UIMessage | undefined; finishReason: LanguageModelV1FinishReason; usage: LanguageModelUsage; }) => void; generateId?: () => string; getCurrentDate?: () => Date; - lastMessage: Message | undefined; + lastMessage: UIMessage | undefined; }) { const replaceLastMessage = lastMessage?.role === 'assistant'; let step = replaceLastMessage @@ -50,7 +50,7 @@ export async function processChatResponse({ }, 0) ?? 0) : 0; - const message: Message = replaceLastMessage + const message: UIMessage = replaceLastMessage ? structuredClone(lastMessage) : { id: generateId(), @@ -123,7 +123,7 @@ export async function processChatResponse({ // is updated with SWR (without it, the changes get stuck in SWR and are not // forwarded to rendering): revisionId: generateId(), - } as Message; + } as UIMessage; update({ message: copiedMessage, diff --git a/packages/ui-utils/src/process-chat-text-response.ts b/packages/ui-utils/src/process-chat-text-response.ts index 991aa38b3915..d673f46d8e1b 100644 --- a/packages/ui-utils/src/process-chat-text-response.ts +++ b/packages/ui-utils/src/process-chat-text-response.ts @@ -1,7 +1,7 @@ import { JSONValue } from '@ai-sdk/provider'; import { generateId as generateIdFunction } from '@ai-sdk/provider-utils'; import { processTextStream } from './process-text-stream'; -import { Message, TextUIPart, UseChatOptions } from './types'; +import { TextUIPart, UIMessage, UseChatOptions } from './types'; export async function processChatTextResponse({ stream, @@ -12,7 +12,7 @@ export async function processChatTextResponse({ }: { stream: ReadableStream; update: (options: { - message: Message; + message: UIMessage; data: JSONValue[] | undefined; replaceLastMessage: boolean; }) => void; @@ -22,7 +22,7 @@ export async function processChatTextResponse({ }) { const textPart: TextUIPart = { type: 'text', text: '' }; - const resultMessage: Message = { + const resultMessage: UIMessage = { id: generateId(), createdAt: getCurrentDate(), role: 'assistant' as const, diff --git a/packages/ui-utils/src/types.ts b/packages/ui-utils/src/types.ts index ef14c295843c..e502a7a2acdf 100644 --- a/packages/ui-utils/src/types.ts +++ b/packages/ui-utils/src/types.ts @@ -85,9 +85,14 @@ that the assistant made as part of this message. */ toolInvocations?: Array; - parts: Array; + // note: optional on the Message type (which serves as input) + parts?: Array; } +export type UIMessage = Message & { + parts: Array; +}; + export type TextUIPart = { type: 'text'; text: string; diff --git a/packages/ui-utils/src/update-tool-call-result.ts b/packages/ui-utils/src/update-tool-call-result.ts index 5ff7d1b587eb..3421b52015ec 100644 --- a/packages/ui-utils/src/update-tool-call-result.ts +++ b/packages/ui-utils/src/update-tool-call-result.ts @@ -1,10 +1,10 @@ -import { Message, ToolInvocationUIPart } from './types'; +import { ToolInvocationUIPart, UIMessage } from './types'; /** * Updates the result of a specific tool invocation in the last message of the given messages array. * * @param {object} params - The parameters object. - * @param {Message[]} params.messages - An array of messages, from which the last one is updated. + * @param {UIMessage[]} params.messages - An array of messages, from which the last one is updated. * @param {string} params.toolCallId - The unique identifier for the tool invocation to update. * @param {unknown} params.toolResult - The result object to attach to the tool invocation. * @returns {void} This function does not return anything. @@ -14,7 +14,7 @@ export function updateToolCallResult({ toolCallId, toolResult: result, }: { - messages: Message[]; + messages: UIMessage[]; toolCallId: string; toolResult: unknown; }) { From 0525428eb13ffc9ba95d4e5fbb8ae1a09aa4e48a Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 16:25:27 +0100 Subject: [PATCH 16/61] extract --- packages/react/src/use-chat.ts | 41 +++------------------ packages/ui-utils/src/fill-message-parts.ts | 9 +++++ packages/ui-utils/src/get-message-parts.ts | 29 +++++++++++++++ packages/ui-utils/src/index.ts | 2 + 4 files changed, 46 insertions(+), 35 deletions(-) create mode 100644 packages/ui-utils/src/fill-message-parts.ts create mode 100644 packages/ui-utils/src/get-message-parts.ts diff --git a/packages/react/src/use-chat.ts b/packages/react/src/use-chat.ts index a85dd0d0d820..d0079942601f 100644 --- a/packages/react/src/use-chat.ts +++ b/packages/react/src/use-chat.ts @@ -4,16 +4,15 @@ import type { CreateMessage, JSONValue, Message, - ReasoningUIPart, - TextUIPart, - ToolInvocationUIPart, UIMessage, UseChatOptions, } from '@ai-sdk/ui-utils'; import { callChatApi, extractMaxToolInvocationStep, + fillMessageParts, generateId as generateIdFunc, + getMessageParts, prepareAttachmentsForRequest, updateToolCallResult, } from '@ai-sdk/ui-utils'; @@ -173,7 +172,7 @@ By default, it's set to 1, which means that only a single LLM call is made. { fallbackData: initialMessages != null - ? fillInUIMessageParts(initialMessages) + ? fillMessageParts(initialMessages) : initialMessagesFallback, }, ); @@ -224,7 +223,7 @@ By default, it's set to 1, which means that only a single LLM call is made. const triggerRequest = useCallback( async (chatRequest: ChatRequest) => { - const chatMessages = fillInUIMessageParts(chatRequest.messages); + const chatMessages = fillMessageParts(chatRequest.messages); const messageCount = chatMessages.length; const maxStep = extractMaxToolInvocationStep( @@ -453,7 +452,7 @@ By default, it's set to 1, which means that only a single LLM call is made. messages = messages(messagesRef.current); } - const messagesWithParts = fillInUIMessageParts(messages); + const messagesWithParts = fillMessageParts(messages); mutate(messagesWithParts, false); messagesRef.current = messagesWithParts; }, @@ -549,7 +548,7 @@ By default, it's set to 1, which means that only a single LLM call is made. ); return { - messages: messages || [], + messages: messages ?? [], id: chatId, setMessages, data: streamData, @@ -584,31 +583,3 @@ function isAssistantMessageWithCompletedToolCalls( message.toolInvocations.every(toolInvocation => 'result' in toolInvocation) ); } - -function fillInUIMessageParts(messages: Message[]): UIMessage[] { - return messages.map(message => ({ - ...message, - parts: getMessageParts(message), - })); -} - -function getMessageParts( - message: Message | CreateMessage | UIMessage, -): (TextUIPart | ReasoningUIPart | ToolInvocationUIPart)[] { - return ( - message.parts ?? [ - ...(message.reasoning - ? [{ type: 'reasoning' as const, reasoning: message.reasoning }] - : []), - ...(message.content - ? [{ type: 'text' as const, text: message.content }] - : []), - ...(message.toolInvocations - ? message.toolInvocations.map(toolInvocation => ({ - type: 'tool-invocation' as const, - toolInvocation, - })) - : []), - ] - ); -} diff --git a/packages/ui-utils/src/fill-message-parts.ts b/packages/ui-utils/src/fill-message-parts.ts new file mode 100644 index 000000000000..84e9d3b36273 --- /dev/null +++ b/packages/ui-utils/src/fill-message-parts.ts @@ -0,0 +1,9 @@ +import { getMessageParts } from './get-message-parts'; +import { Message, UIMessage } from './types'; + +export function fillMessageParts(messages: Message[]): UIMessage[] { + return messages.map(message => ({ + ...message, + parts: getMessageParts(message), + })); +} diff --git a/packages/ui-utils/src/get-message-parts.ts b/packages/ui-utils/src/get-message-parts.ts new file mode 100644 index 000000000000..6f71996ba871 --- /dev/null +++ b/packages/ui-utils/src/get-message-parts.ts @@ -0,0 +1,29 @@ +import { + CreateMessage, + Message, + ReasoningUIPart, + TextUIPart, + ToolInvocationUIPart, + UIMessage, +} from './types'; + +export function getMessageParts( + message: Message | CreateMessage | UIMessage, +): (TextUIPart | ReasoningUIPart | ToolInvocationUIPart)[] { + return ( + message.parts ?? [ + ...(message.reasoning + ? [{ type: 'reasoning' as const, reasoning: message.reasoning }] + : []), + ...(message.content + ? [{ type: 'text' as const, text: message.content }] + : []), + ...(message.toolInvocations + ? message.toolInvocations.map(toolInvocation => ({ + type: 'tool-invocation' as const, + toolInvocation, + })) + : []), + ] + ); +} diff --git a/packages/ui-utils/src/index.ts b/packages/ui-utils/src/index.ts index a6277a407144..4b679945071a 100644 --- a/packages/ui-utils/src/index.ts +++ b/packages/ui-utils/src/index.ts @@ -20,6 +20,8 @@ export type { DataStreamPart, DataStreamString } from './data-stream-parts'; export { getTextFromDataUrl } from './data-url'; export type { DeepPartial } from './deep-partial'; export { extractMaxToolInvocationStep } from './extract-max-tool-invocation-step'; +export { fillMessageParts } from './fill-message-parts'; +export { getMessageParts } from './get-message-parts'; export { isDeepEqualData } from './is-deep-equal-data'; export { parsePartialJson } from './parse-partial-json'; export { prepareAttachmentsForRequest } from './prepare-attachments-for-request'; From eec611155ea97f4c5b21b562c2ea7ab65381721a Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Mon, 3 Feb 2025 16:42:43 +0100 Subject: [PATCH 17/61] solid --- packages/react/src/use-chat.ts | 10 +++-- packages/solid/src/use-chat.ts | 57 ++++++++++++++----------- packages/solid/src/use-chat.ui.test.tsx | 2 + 3 files changed, 41 insertions(+), 28 deletions(-) diff --git a/packages/react/src/use-chat.ts b/packages/react/src/use-chat.ts index d0079942601f..e7759f1de6ae 100644 --- a/packages/react/src/use-chat.ts +++ b/packages/react/src/use-chat.ts @@ -530,18 +530,20 @@ By default, it's set to 1, which means that only a single LLM call is made. const addToolResult = useCallback( ({ toolCallId, result }: { toolCallId: string; result: any }) => { + const currentMessages = messagesRef.current; + updateToolCallResult({ - messages: messagesRef.current, + messages: currentMessages, toolCallId, toolResult: result, }); - mutate(messagesRef.current, false); + mutate(currentMessages, false); // auto-submit when all tool calls in the last assistant message have results: - const lastMessage = messagesRef.current[messagesRef.current.length - 1]; + const lastMessage = currentMessages[currentMessages.length - 1]; if (isAssistantMessageWithCompletedToolCalls(lastMessage)) { - triggerRequest({ messages: messagesRef.current }); + triggerRequest({ messages: currentMessages }); } }, [mutate, triggerRequest], diff --git a/packages/solid/src/use-chat.ts b/packages/solid/src/use-chat.ts index a54c4bf0c36f..a53db7f309af 100644 --- a/packages/solid/src/use-chat.ts +++ b/packages/solid/src/use-chat.ts @@ -7,11 +7,14 @@ import type { JSONValue, Message, UseChatOptions as SharedUseChatOptions, + UIMessage, } from '@ai-sdk/ui-utils'; import { callChatApi, extractMaxToolInvocationStep, + fillMessageParts, generateId as generateIdFunc, + getMessageParts, prepareAttachmentsForRequest, updateToolCallResult, } from '@ai-sdk/ui-utils'; @@ -33,7 +36,7 @@ export type UseChatHelpers = { /** * Current messages in the chat as a SolidJS store. */ - messages: () => Store; + messages: () => Store; /** The error object of the API request */ error: Accessor; @@ -115,11 +118,11 @@ or to provide a custom fetch implementation for e.g. testing. const processStreamedResponse = async ( api: string, chatRequest: ChatRequest, - mutate: (data: Message[]) => void, + mutate: (data: UIMessage[]) => void, setStreamData: Setter, streamData: Accessor, extraMetadata: any, - messagesRef: Message[], + messagesRef: UIMessage[], abortController: AbortController | null, generateId: IdGenerator, streamProtocol: UseChatOptions['streamProtocol'] = 'data', @@ -134,14 +137,15 @@ const processStreamedResponse = async ( // Do an optimistic update to the chat state to show the updated messages // immediately. const previousMessages = messagesRef; + const chatMessages = fillMessageParts(chatRequest.messages); - mutate(chatRequest.messages); + mutate(chatMessages); const existingStreamData = streamData() ?? []; const constructedMessagesPayload = sendExtraMessageFields - ? chatRequest.messages - : chatRequest.messages.map( + ? chatMessages + : chatMessages.map( ({ role, content, @@ -186,8 +190,8 @@ const processStreamedResponse = async ( onUpdate({ message, data, replaceLastMessage }) { mutate([ ...(replaceLastMessage - ? chatRequest.messages.slice(0, chatRequest.messages.length - 1) - : chatRequest.messages), + ? chatMessages.slice(0, chatMessages.length - 1) + : chatMessages), message, ]); @@ -199,7 +203,7 @@ const processStreamedResponse = async ( onFinish, generateId, fetch, - lastMessage: chatRequest.messages[chatRequest.messages.length - 1], + lastMessage: chatMessages[chatMessages.length - 1], }); }; @@ -235,12 +239,14 @@ export function useChat( chatCache.get(chatKey()) ?? useChatOptions().initialMessages?.() ?? [], ); - const [messagesStore, setMessagesStore] = createStore(_messages()); + const [messagesStore, setMessagesStore] = createStore( + fillMessageParts(_messages()), + ); createEffect(() => { - setMessagesStore(reconcile(_messages(), { merge: true })); + setMessagesStore(reconcile(fillMessageParts(_messages()), { merge: true })); }); - const mutate = (messages: Message[]) => { + const mutate = (messages: UIMessage[]) => { chatCache.set(chatKey(), messages); }; @@ -250,9 +256,9 @@ export function useChat( ); const [isLoading, setIsLoading] = createSignal(false); - let messagesRef: Message[] = _messages() || []; + let messagesRef: UIMessage[] = fillMessageParts(_messages()) || []; createEffect(() => { - messagesRef = _messages() || []; + messagesRef = fillMessageParts(_messages()) || []; }); let abortController: AbortController | null = null; @@ -354,15 +360,17 @@ export function useChat( experimental_attachments, ); - const newMessage = { + const messages = messagesRef.concat({ ...message, id: message.id ?? generateId()(), + createdAt: message.createdAt ?? new Date(), experimental_attachments: attachmentsForRequest.length > 0 ? attachmentsForRequest : undefined, - }; + parts: getMessageParts(message), + }); return triggerRequest({ - messages: messagesRef.concat(newMessage as Message), + messages, headers, body, data, @@ -405,8 +413,9 @@ export function useChat( messagesArg = messagesArg(messagesRef); } - mutate(messagesArg); - messagesRef = messagesArg; + const messagesWithParts = fillMessageParts(messagesArg); + mutate(messagesWithParts); + messagesRef = messagesWithParts; }; const setData = ( @@ -476,20 +485,20 @@ export function useChat( toolCallId: string; result: any; }) => { - const messagesSnapshot = _messages() ?? []; + const currentMessages = messagesRef ?? []; updateToolCallResult({ - messages: messagesSnapshot, + messages: currentMessages, toolCallId, toolResult: result, }); - mutate(messagesSnapshot); + mutate(currentMessages); // auto-submit when all tool calls in the last assistant message have results: - const lastMessage = messagesSnapshot[messagesSnapshot.length - 1]; + const lastMessage = currentMessages[currentMessages.length - 1]; if (isAssistantMessageWithCompletedToolCalls(lastMessage)) { - triggerRequest({ messages: messagesSnapshot }); + triggerRequest({ messages: currentMessages }); } }; diff --git a/packages/solid/src/use-chat.ui.test.tsx b/packages/solid/src/use-chat.ui.test.tsx index 2a0cdc185cf9..97880ed3f049 100644 --- a/packages/solid/src/use-chat.ui.test.tsx +++ b/packages/solid/src/use-chat.ui.test.tsx @@ -373,6 +373,7 @@ describe('data protocol stream', () => { createdAt: expect.any(Date), role: 'assistant', content: 'Hello, world.', + parts: [{ text: 'Hello, world.', type: 'text' }], }, options: { finishReason: 'stop', @@ -511,6 +512,7 @@ describe('text stream', () => { createdAt: expect.any(Date), role: 'assistant', content: 'Hello, world.', + parts: [{ text: 'Hello, world.', type: 'text' }], }, options: { finishReason: 'unknown', From 5413a32c49fcd8fb0fb43a59bc93415ea184347e Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 09:05:08 +0100 Subject: [PATCH 18/61] svekte fille --- packages/svelte/src/use-chat.ts | 46 ++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/packages/svelte/src/use-chat.ts b/packages/svelte/src/use-chat.ts index 47c6287ee254..1bbd0ad9f092 100644 --- a/packages/svelte/src/use-chat.ts +++ b/packages/svelte/src/use-chat.ts @@ -7,11 +7,14 @@ import type { JSONValue, Message, UseChatOptions as SharedUseChatOptions, + UIMessage, } from '@ai-sdk/ui-utils'; import { callChatApi, extractMaxToolInvocationStep, + fillMessageParts, generateId as generateIdFunc, + getMessageParts, prepareAttachmentsForRequest, updateToolCallResult, } from '@ai-sdk/ui-utils'; @@ -94,7 +97,7 @@ export type UseChatHelpers = { const getStreamedResponse = async ( api: string, chatRequest: ChatRequest, - mutate: (messages: Message[]) => void, + mutate: (messages: UIMessage[]) => void, mutateStreamData: (data: JSONValue[] | undefined) => void, existingData: JSONValue[] | undefined, extraMetadata: { @@ -102,7 +105,7 @@ const getStreamedResponse = async ( headers?: Record | Headers; body?: any; }, - previousMessages: Message[], + previousMessages: UIMessage[], abortControllerRef: AbortController | null, generateId: IdGenerator, streamProtocol: UseChatOptions['streamProtocol'], @@ -116,11 +119,13 @@ const getStreamedResponse = async ( ) => { // Do an optimistic update to the chat state to show the updated messages // immediately. - mutate(chatRequest.messages); + const chatMessages = fillMessageParts(chatRequest.messages); + + mutate(chatMessages); const constructedMessagesPayload = sendExtraMessageFields - ? chatRequest.messages - : chatRequest.messages.map( + ? chatMessages + : chatMessages.map( ({ role, content, @@ -165,8 +170,8 @@ const getStreamedResponse = async ( onUpdate({ message, data, replaceLastMessage }) { mutate([ ...(replaceLastMessage - ? chatRequest.messages.slice(0, chatRequest.messages.length - 1) - : chatRequest.messages), + ? chatMessages.slice(0, chatMessages.length - 1) + : chatMessages), message, ]); if (data?.length) { @@ -177,11 +182,11 @@ const getStreamedResponse = async ( generateId, onToolCall, fetch, - lastMessage: chatRequest.messages[chatRequest.messages.length - 1], + lastMessage: chatMessages[chatMessages.length - 1], }); }; -const store: Record = {}; +const store: Record = {}; /** Check if the message is an assistant message with completed tool calls. @@ -232,9 +237,9 @@ export function useChat({ data, mutate: originalMutate, isLoading: isSWRLoading, - } = useSWR(key, { - fetcher: () => store[key] || initialMessages, - fallbackData: initialMessages, + } = useSWR(key, { + fetcher: () => store[key] ?? fillMessageParts(initialMessages), + fallbackData: fillMessageParts(initialMessages), }); const streamData = writable(undefined); @@ -242,15 +247,15 @@ export function useChat({ const loading = writable(false); // Force the `data` to be `initialMessages` if it's `undefined`. - data.set(initialMessages); + data.set(fillMessageParts(initialMessages)); - const mutate = (data: Message[]) => { + const mutate = (data: UIMessage[]) => { store[key] = data; return originalMutate(data); }; // Because of the `fallbackData` option, the `data` will never be `undefined`. - const messages = data as Writable; + const messages = data as Writable; // Abort controller to cancel the current API call. let abortController: AbortController | null = null; @@ -344,10 +349,6 @@ export function useChat({ message: Message | CreateMessage, { data, headers, body, experimental_attachments }: ChatRequestOptions = {}, ) => { - if (!message.id) { - message.id = generateId(); - } - const attachmentsForRequest = await prepareAttachmentsForRequest( experimental_attachments, ); @@ -355,9 +356,12 @@ export function useChat({ return triggerRequest({ messages: get(messages).concat({ ...message, + id: message.id ?? generateId(), + createdAt: message.createdAt ?? new Date(), experimental_attachments: attachmentsForRequest.length > 0 ? attachmentsForRequest : undefined, - } as Message), + parts: getMessageParts(message), + } as UIMessage), headers, body, data, @@ -401,7 +405,7 @@ export function useChat({ messagesArg = messagesArg(get(messages)); } - mutate(messagesArg); + mutate(fillMessageParts(messagesArg)); }; const setData = ( From ca8815d43992b9d06a36303028b6d45a49061d76 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 09:26:38 +0100 Subject: [PATCH 19/61] resubmit react --- packages/react/src/use-chat.ts | 42 +++--------- packages/ui-utils/src/index.ts | 4 ++ .../ui-utils/src/should-resubmit-messages.ts | 64 +++++++++++++++++++ 3 files changed, 76 insertions(+), 34 deletions(-) create mode 100644 packages/ui-utils/src/should-resubmit-messages.ts diff --git a/packages/react/src/use-chat.ts b/packages/react/src/use-chat.ts index e7759f1de6ae..4c4118ee3c97 100644 --- a/packages/react/src/use-chat.ts +++ b/packages/react/src/use-chat.ts @@ -13,7 +13,9 @@ import { fillMessageParts, generateId as generateIdFunc, getMessageParts, + isAssistantMessageWithCompletedToolCalls, prepareAttachmentsForRequest, + shouldResubmitMessages, updateToolCallResult, } from '@ai-sdk/ui-utils'; import { useCallback, useEffect, useRef, useState } from 'react'; @@ -343,23 +345,13 @@ By default, it's set to 1, which means that only a single LLM call is made. // auto-submit when all tool calls in the last assistant message have results // and assistant has not answered yet const messages = messagesRef.current; - const lastMessage = messages[messages.length - 1]; if ( - // ensure there is a last message: - lastMessage != null && - // ensure we actually have new steps (to prevent infinite loops in case of errors): - (messages.length > messageCount || - extractMaxToolInvocationStep(lastMessage.toolInvocations) !== - maxStep) && - // check if the feature is enabled: - maxSteps > 1 && - // check that next step is possible: - isAssistantMessageWithCompletedToolCalls(lastMessage) && - // check that assistant has not answered yet: - lastMessage.parts[lastMessage.parts.length - 1].type !== 'text' && // TODO brittle, should check for text after last tool invocation - // limit the number of automatic steps: - (extractMaxToolInvocationStep(lastMessage.toolInvocations) ?? 0) < - maxSteps + shouldResubmitMessages({ + originalMaxToolInvocationStep: maxStep, + originalMessageCount: messageCount, + maxSteps, + messages, + }) ) { await triggerRequest({ messages }); } @@ -567,21 +559,3 @@ By default, it's set to 1, which means that only a single LLM call is made. addToolResult, }; } - -/** -Check if the message is an assistant message with completed tool calls. -The message must have at least one tool invocation and all tool invocations -must have a result. - */ -function isAssistantMessageWithCompletedToolCalls( - message: Message, -): message is Message & { - role: 'assistant'; -} { - return ( - message.role === 'assistant' && - message.toolInvocations != null && - message.toolInvocations.length > 0 && - message.toolInvocations.every(toolInvocation => 'result' in toolInvocation) - ); -} diff --git a/packages/ui-utils/src/index.ts b/packages/ui-utils/src/index.ts index 4b679945071a..12ba2071ccb6 100644 --- a/packages/ui-utils/src/index.ts +++ b/packages/ui-utils/src/index.ts @@ -30,5 +30,9 @@ export { processDataStream } from './process-data-stream'; export { processTextStream } from './process-text-stream'; export { asSchema, jsonSchema } from './schema'; export type { Schema } from './schema'; +export { + isAssistantMessageWithCompletedToolCalls, + shouldResubmitMessages, +} from './should-resubmit-messages'; export { updateToolCallResult } from './update-tool-call-result'; export { zodSchema } from './zod-schema'; diff --git a/packages/ui-utils/src/should-resubmit-messages.ts b/packages/ui-utils/src/should-resubmit-messages.ts new file mode 100644 index 000000000000..fddf05f38432 --- /dev/null +++ b/packages/ui-utils/src/should-resubmit-messages.ts @@ -0,0 +1,64 @@ +import { extractMaxToolInvocationStep } from './extract-max-tool-invocation-step'; +import { UIMessage } from './types'; + +export function shouldResubmitMessages({ + originalMaxToolInvocationStep, + originalMessageCount, + maxSteps, + messages, +}: { + originalMaxToolInvocationStep: number | undefined; + originalMessageCount: number; + maxSteps: number; + messages: UIMessage[]; +}) { + const lastMessage = messages[messages.length - 1]; + return ( + // check if the feature is enabled: + maxSteps > 1 && + // ensure there is a last message: + lastMessage != null && + // ensure we actually have new steps (to prevent infinite loops in case of errors): + (messages.length > originalMessageCount || + extractMaxToolInvocationStep(lastMessage.toolInvocations) !== + originalMaxToolInvocationStep) && + // check that next step is possible: + isAssistantMessageWithCompletedToolCalls(lastMessage) && + // check that assistant has not answered yet: + !isLastToolInvocationFollowedByText(lastMessage) && + // limit the number of automatic steps: + (extractMaxToolInvocationStep(lastMessage.toolInvocations) ?? 0) < maxSteps + ); +} + +function isLastToolInvocationFollowedByText(message: UIMessage) { + let isLastToolInvocationFollowedByText = false; + + message.parts.forEach(part => { + if (part.type === 'text') { + isLastToolInvocationFollowedByText = true; + } + if (part.type === 'tool-invocation') { + isLastToolInvocationFollowedByText = false; + } + }); + return isLastToolInvocationFollowedByText; +} + +/** +Check if the message is an assistant message with completed tool calls. +The message must have at least one tool invocation and all tool invocations +must have a result. + */ +export function isAssistantMessageWithCompletedToolCalls( + message: UIMessage, +): message is UIMessage & { + role: 'assistant'; +} { + return ( + message.role === 'assistant' && + message.parts + .filter(part => part.type === 'tool-invocation') + .every(part => 'result' in part.toolInvocation) + ); +} From b136b30a24807a08498d24b344dcc6603ec2ac16 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 09:32:28 +0100 Subject: [PATCH 20/61] svelte --- packages/svelte/src/use-chat.ts | 39 +++++++-------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/packages/svelte/src/use-chat.ts b/packages/svelte/src/use-chat.ts index 1bbd0ad9f092..80660ebf002b 100644 --- a/packages/svelte/src/use-chat.ts +++ b/packages/svelte/src/use-chat.ts @@ -15,7 +15,9 @@ import { fillMessageParts, generateId as generateIdFunc, getMessageParts, + isAssistantMessageWithCompletedToolCalls, prepareAttachmentsForRequest, + shouldResubmitMessages, updateToolCallResult, } from '@ai-sdk/ui-utils'; import { useSWR } from 'sswr'; @@ -188,20 +190,6 @@ const getStreamedResponse = async ( const store: Record = {}; -/** -Check if the message is an assistant message with completed tool calls. -The message must have at least one tool invocation and all tool invocations -must have a result. - */ -function isAssistantMessageWithCompletedToolCalls(message: Message) { - return ( - message.role === 'assistant' && - message.toolInvocations && - message.toolInvocations.length > 0 && - message.toolInvocations.every(toolInvocation => 'result' in toolInvocation) - ); -} - export function useChat({ api = '/api/chat', id, @@ -322,24 +310,13 @@ export function useChat({ // auto-submit when all tool calls in the last assistant message have results: const newMessagesSnapshot = get(messages); - - const lastMessage = newMessagesSnapshot[newMessagesSnapshot.length - 1]; if ( - // ensure there is a last message: - lastMessage != null && - // ensure we actually have new messages (to prevent infinite loops in case of errors): - (newMessagesSnapshot.length > messageCount || - extractMaxToolInvocationStep(lastMessage.toolInvocations) !== - maxStep) && - // check if the feature is enabled: - maxSteps > 1 && - // check that next step is possible: - isAssistantMessageWithCompletedToolCalls(lastMessage) && - // check that assistant has not answered yet: - !lastMessage.content && // empty string or undefined - // limit the number of automatic steps: - (extractMaxToolInvocationStep(lastMessage.toolInvocations) ?? 0) < - maxSteps + shouldResubmitMessages({ + originalMaxToolInvocationStep: maxStep, + originalMessageCount: messageCount, + maxSteps, + messages: newMessagesSnapshot, + }) ) { await triggerRequest({ messages: newMessagesSnapshot }); } From 4ffbd80ced27172f9f8787f15fdf462cc2c38905 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 09:33:50 +0100 Subject: [PATCH 21/61] solid --- packages/solid/src/use-chat.ts | 38 +++++++--------------------------- 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/packages/solid/src/use-chat.ts b/packages/solid/src/use-chat.ts index a53db7f309af..a7fa889fdaec 100644 --- a/packages/solid/src/use-chat.ts +++ b/packages/solid/src/use-chat.ts @@ -15,7 +15,9 @@ import { fillMessageParts, generateId as generateIdFunc, getMessageParts, + isAssistantMessageWithCompletedToolCalls, prepareAttachmentsForRequest, + shouldResubmitMessages, updateToolCallResult, } from '@ai-sdk/ui-utils'; import { @@ -330,23 +332,13 @@ export function useChat( // auto-submit when all tool calls in the last assistant message have results: const messages = messagesRef; - const lastMessage = messages[messages.length - 1]; if ( - // ensure there is a last message: - lastMessage != null && - // ensure we actually have new steps (to prevent infinite loops in case of errors): - (messages.length > messageCount || - extractMaxToolInvocationStep(lastMessage.toolInvocations) !== - maxStep) && - // check if the feature is enabled: - maxSteps > 1 && - // check that next step is possible: - isAssistantMessageWithCompletedToolCalls(lastMessage) && - // check that assistant has not answered yet: - !lastMessage.content && // empty string or undefined - // limit the number of automatic steps: - (extractMaxToolInvocationStep(lastMessage.toolInvocations) ?? 0) < - maxSteps + shouldResubmitMessages({ + originalMaxToolInvocationStep: maxStep, + originalMessageCount: messageCount, + maxSteps, + messages, + }) ) { await triggerRequest({ messages }); } @@ -521,17 +513,3 @@ export function useChat( addToolResult, }; } - -/** -Check if the message is an assistant message with completed tool calls. -The message must have at least one tool invocation and all tool invocations -must have a result. - */ -function isAssistantMessageWithCompletedToolCalls(message: Message) { - return ( - message.role === 'assistant' && - message.toolInvocations && - message.toolInvocations.length > 0 && - message.toolInvocations.every(toolInvocation => 'result' in toolInvocation) - ); -} From eb35787e83baacc576f0c6e24fd9178618968121 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 09:42:01 +0100 Subject: [PATCH 22/61] vue --- packages/vue/src/use-chat.ts | 70 +++++++++++---------------- packages/vue/src/use-chat.ui.test.tsx | 2 + 2 files changed, 31 insertions(+), 41 deletions(-) diff --git a/packages/vue/src/use-chat.ts b/packages/vue/src/use-chat.ts index a71d8c9fda08..443673deb079 100644 --- a/packages/vue/src/use-chat.ts +++ b/packages/vue/src/use-chat.ts @@ -1,16 +1,19 @@ import type { - ChatRequest, ChatRequestOptions, CreateMessage, JSONValue, Message, + UIMessage, UseChatOptions, } from '@ai-sdk/ui-utils'; import { callChatApi, extractMaxToolInvocationStep, + fillMessageParts, generateId as generateIdFunc, + getMessageParts, prepareAttachmentsForRequest, + shouldResubmitMessages, } from '@ai-sdk/ui-utils'; import swrv from 'swrv'; import type { Ref } from 'vue'; @@ -85,7 +88,7 @@ export type UseChatHelpers = { // @ts-expect-error - some issues with the default export of useSWRV const useSWRV = (swrv.default as typeof import('swrv')['default']) || swrv; -const store: Record = {}; +const store: Record = {}; export function useChat( { @@ -105,7 +108,7 @@ export function useChat( onToolCall, fetch, keepLastMessageOnError = true, - maxSteps, + maxSteps = 1, }: UseChatOptions & { /** * Maximum number of sequential LLM calls (steps), e.g. when you use tool calls. Must be at least 1. @@ -121,9 +124,9 @@ export function useChat( const chatId = id ?? generateId(); const key = `${api}|${chatId}`; - const { data: messagesData, mutate: originalMutate } = useSWRV( + const { data: messagesData, mutate: originalMutate } = useSWRV( key, - () => store[key] || initialMessages, + () => store[key] ?? fillMessageParts(initialMessages), ); const { data: isLoading, mutate: mutateLoading } = useSWRV( @@ -134,15 +137,15 @@ export function useChat( isLoading.value ??= false; // Force the `data` to be `initialMessages` if it's `undefined`. - messagesData.value ??= initialMessages; + messagesData.value ??= fillMessageParts(initialMessages); - const mutate = (data?: Message[]) => { + const mutate = (data?: UIMessage[]) => { store[key] = data; return originalMutate(); }; // Because of the `initialData` option, the `data` will never be `undefined`. - const messages = messagesData as Ref; + const messages = messagesData as Ref; const error = ref(undefined); // cannot use JSONValue[] in ref because of infinite Typescript recursion: @@ -167,21 +170,15 @@ export function useChat( // Do an optimistic update to the chat state to show the updated messages // immediately. - const previousMessages = messagesSnapshot; - mutate(messagesSnapshot); - - const chatRequest: ChatRequest = { - messages: messagesSnapshot, - body, - headers, - data, - }; + const previousMessages = fillMessageParts(messagesSnapshot); + const chatMessages = fillMessageParts(messagesSnapshot); + mutate(chatMessages); const existingData = (streamData.value ?? []) as JSONValue[]; const constructedMessagesPayload = sendExtraMessageFields - ? chatRequest.messages - : chatRequest.messages.map( + ? chatMessages + : chatMessages.map( ({ role, content, @@ -206,7 +203,7 @@ export function useChat( body: { id: chatId, messages: constructedMessagesPayload, - data: chatRequest.data, + data, ...unref(metadataBody), // Use unref to unwrap the ref value ...body, }, @@ -221,8 +218,8 @@ export function useChat( onUpdate({ message, data, replaceLastMessage }) { mutate([ ...(replaceLastMessage - ? chatRequest.messages.slice(0, chatRequest.messages.length - 1) - : chatRequest.messages), + ? chatMessages.slice(0, chatMessages.length - 1) + : chatMessages), message, ]); if (data?.length) { @@ -239,7 +236,7 @@ export function useChat( generateId, onToolCall, fetch, - lastMessage: chatRequest.messages[chatRequest.messages.length - 1], + lastMessage: chatMessages[chatMessages.length - 1], }); } catch (err) { // Ignore abort errors as they are expected. @@ -259,24 +256,13 @@ export function useChat( } // auto-submit when all tool calls in the last assistant message have results: - const lastMessage = messages.value[messages.value.length - 1]; if ( - // ensure there is a last message: - lastMessage != null && - // ensure we actually have new messages (to prevent infinite loops in case of errors): - (messages.value.length > messageCount || - extractMaxToolInvocationStep(lastMessage.toolInvocations) !== - maxStep) && - // check if the feature is enabled: - maxSteps && - maxSteps > 1 && - // check that next step is possible: - isAssistantMessageWithCompletedToolCalls(lastMessage) && - // check that assistant has not answered yet: - !lastMessage.content && // empty string or undefined - // limit the number of automatic steps: - (extractMaxToolInvocationStep(lastMessage.toolInvocations) ?? 0) < - maxSteps + shouldResubmitMessages({ + originalMaxToolInvocationStep: maxStep, + originalMessageCount: messageCount, + maxSteps, + messages: messages.value, + }) ) { await triggerRequest(messages.value); } @@ -294,6 +280,7 @@ export function useChat( createdAt: message.createdAt ?? new Date(), experimental_attachments: attachmentsForRequest.length > 0 ? attachmentsForRequest : undefined, + parts: getMessageParts(message), }), options, ); @@ -325,7 +312,7 @@ export function useChat( messagesArg = messagesArg(messages.value); } - mutate(messagesArg); + mutate(fillMessageParts(messagesArg)); }; const setData = ( @@ -365,6 +352,7 @@ export function useChat( role: 'user', experimental_attachments: attachmentsForRequest.length > 0 ? attachmentsForRequest : undefined, + parts: [{ type: 'text', text: inputValue }], }), options, ); diff --git a/packages/vue/src/use-chat.ui.test.tsx b/packages/vue/src/use-chat.ui.test.tsx index 5c56ae91b98e..f0df66d401b2 100644 --- a/packages/vue/src/use-chat.ui.test.tsx +++ b/packages/vue/src/use-chat.ui.test.tsx @@ -185,6 +185,7 @@ describe('data protocol stream', () => { createdAt: expect.any(String), role: 'assistant', content: 'Hello, world.', + parts: [{ text: 'Hello, world.', type: 'text' }], }, options: { finishReason: 'stop', @@ -255,6 +256,7 @@ describe('text stream', () => { createdAt: expect.any(String), role: 'assistant', content: 'Hello, world.', + parts: [{ text: 'Hello, world.', type: 'text' }], }, options: { finishReason: 'unknown', From 3cb1c3701531c4d238aaf2a6ef9917bdc5b61973 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 09:44:45 +0100 Subject: [PATCH 23/61] tool call --- packages/vue/src/use-chat.ts | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/packages/vue/src/use-chat.ts b/packages/vue/src/use-chat.ts index 443673deb079..ea645661f9d9 100644 --- a/packages/vue/src/use-chat.ts +++ b/packages/vue/src/use-chat.ts @@ -14,6 +14,7 @@ import { getMessageParts, prepareAttachmentsForRequest, shouldResubmitMessages, + updateToolCallResult, } from '@ai-sdk/ui-utils'; import swrv from 'swrv'; import type { Ref } from 'vue'; @@ -367,33 +368,20 @@ export function useChat( toolCallId: string; result: any; }) => { - const updatedMessages = messages.value.map((message, index, arr) => - // update the tool calls in the last assistant message: - index === arr.length - 1 && - message.role === 'assistant' && - message.toolInvocations - ? { - ...message, - toolInvocations: message.toolInvocations.map(toolInvocation => - toolInvocation.toolCallId === toolCallId - ? { - ...toolInvocation, - result, - state: 'result' as const, - } - : toolInvocation, - ), - } - : message, - ); + const currentMessages = messages.value; - mutate(updatedMessages); + updateToolCallResult({ + messages: currentMessages, + toolCallId, + toolResult: result, + }); - // auto-submit when all tool calls in the last assistant message have results: - const lastMessage = updatedMessages[updatedMessages.length - 1]; + mutate(currentMessages); + // auto-submit when all tool calls in the last assistant message have results: + const lastMessage = currentMessages[currentMessages.length - 1]; if (isAssistantMessageWithCompletedToolCalls(lastMessage)) { - triggerRequest(updatedMessages); + triggerRequest(currentMessages); } }; From be86bce10164e4a6b74b60cea5ac6beeec20a890 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 09:46:42 +0100 Subject: [PATCH 24/61] deprecate --- packages/ui-utils/src/types.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/ui-utils/src/types.ts b/packages/ui-utils/src/types.ts index e502a7a2acdf..0d4e220dc433 100644 --- a/packages/ui-utils/src/types.ts +++ b/packages/ui-utils/src/types.ts @@ -57,11 +57,15 @@ The timestamp of the message. /** Text content of the message. + +@deprecated Use `parts` instead. */ content: string; /** Reasoning for the message. + +@deprecated Use `parts` instead. */ reasoning?: string; @@ -70,6 +74,9 @@ Reasoning for the message. */ experimental_attachments?: Attachment[]; + /** +The 'data' role is deprecated. + */ role: 'system' | 'user' | 'assistant' | 'data'; data?: JSONValue; @@ -82,6 +89,8 @@ Reasoning for the message. /** Tool invocations (that can be tool calls or tool results, depending on whether or not the invocation has finished) that the assistant made as part of this message. + +@deprecated Use `parts` instead. */ toolInvocations?: Array; @@ -90,6 +99,14 @@ that the assistant made as part of this message. } export type UIMessage = Message & { + /** + * The timestamp of the message. + */ + createdAt: Date; + + /** + * The parts of the message. Use this for rendering the message in the UI. + */ parts: Array; }; From 980083da3d998ef1970e2ae78b3cf3854e98b37e Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 09:49:05 +0100 Subject: [PATCH 25/61] x --- packages/ui-utils/src/fill-message-parts.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ui-utils/src/fill-message-parts.ts b/packages/ui-utils/src/fill-message-parts.ts index 84e9d3b36273..11de88514678 100644 --- a/packages/ui-utils/src/fill-message-parts.ts +++ b/packages/ui-utils/src/fill-message-parts.ts @@ -4,6 +4,7 @@ import { Message, UIMessage } from './types'; export function fillMessageParts(messages: Message[]): UIMessage[] { return messages.map(message => ({ ...message, + createdAt: message.createdAt!, parts: getMessageParts(message), })); } From ddec81791e2beb3f76321b874321ddf73a58f77c Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 09:49:18 +0100 Subject: [PATCH 26/61] revert --- packages/ui-utils/src/fill-message-parts.ts | 1 - packages/ui-utils/src/types.ts | 5 ----- 2 files changed, 6 deletions(-) diff --git a/packages/ui-utils/src/fill-message-parts.ts b/packages/ui-utils/src/fill-message-parts.ts index 11de88514678..84e9d3b36273 100644 --- a/packages/ui-utils/src/fill-message-parts.ts +++ b/packages/ui-utils/src/fill-message-parts.ts @@ -4,7 +4,6 @@ import { Message, UIMessage } from './types'; export function fillMessageParts(messages: Message[]): UIMessage[] { return messages.map(message => ({ ...message, - createdAt: message.createdAt!, parts: getMessageParts(message), })); } diff --git a/packages/ui-utils/src/types.ts b/packages/ui-utils/src/types.ts index 0d4e220dc433..b1456085466a 100644 --- a/packages/ui-utils/src/types.ts +++ b/packages/ui-utils/src/types.ts @@ -99,11 +99,6 @@ that the assistant made as part of this message. } export type UIMessage = Message & { - /** - * The timestamp of the message. - */ - createdAt: Date; - /** * The parts of the message. Use this for rendering the message in the UI. */ From d1a7e9fda658d0eeda7232f8faa81a5358e9b81c Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 10:01:55 +0100 Subject: [PATCH 27/61] fix types --- packages/ai/core/prompt/append-response-messages.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/ai/core/prompt/append-response-messages.ts b/packages/ai/core/prompt/append-response-messages.ts index 7ac26abf099c..1843c3c25652 100644 --- a/packages/ai/core/prompt/append-response-messages.ts +++ b/packages/ai/core/prompt/append-response-messages.ts @@ -87,7 +87,7 @@ export function appendResponseMessages({ toolInvocation: call, })) .forEach(part => { - lastMessage.parts.push(part); + lastMessage.parts!.push(part); }); } else { // last message was a user message, add the assistant message: @@ -121,6 +121,8 @@ export function appendResponseMessages({ ); } + lastMessage.parts ??= []; + for (const contentPart of message.content) { // find the tool call in the previous message: const toolCall = lastMessage.toolInvocations.find( From f2d98167bb2c19a5cd4d10cfb0f6b6a0948560d4 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 10:07:29 +0100 Subject: [PATCH 28/61] 1 --- .../process-chat-response.test.ts.snap | 557 ++++++++++++++++++ .../src/process-chat-response.test.ts | 105 ++++ 2 files changed, 662 insertions(+) diff --git a/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap b/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap index 12027670c51e..c1bc951a7c7e 100644 --- a/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap +++ b/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap @@ -1082,6 +1082,563 @@ exports[`scenario: server-side tool roundtrip with existing assistant message > ] `; +exports[`scenario: server-side tool roundtrip with multiple assistant reasoning > should call the onFinish function with the correct arguments 1`] = ` +[ + { + "finishReason": "stop", + "message": { + "content": "The weather in London is sunny.", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "reasoning": "I willuse a tool to get the weather in London.I know know the weather in London.", + "type": "reasoning", + }, + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + { + "text": "The weather in London is sunny.", + "type": "text", + }, + ], + "reasoning": "I willuse a tool to get the weather in London.I know know the weather in London.", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "usage": { + "completionTokens": 7, + "promptTokens": 14, + "totalTokens": 21, + }, + }, +] +`; + +exports[`scenario: server-side tool roundtrip with multiple assistant reasoning > should call the update function with the correct arguments 1`] = ` +[ + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "reasoning": "I will", + "type": "reasoning", + }, + ], + "reasoning": "I will", + "revisionId": "id-1", + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "reasoning": "I willuse a tool to get the weather in London.", + "type": "reasoning", + }, + ], + "reasoning": "I willuse a tool to get the weather in London.", + "revisionId": "id-2", + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "reasoning": "I willuse a tool to get the weather in London.", + "type": "reasoning", + }, + { + "toolInvocation": { + "args": { + "city": "London", + }, + "state": "call", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + ], + "reasoning": "I willuse a tool to get the weather in London.", + "revisionId": "id-3", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "state": "call", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "reasoning": "I willuse a tool to get the weather in London.", + "type": "reasoning", + }, + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + ], + "reasoning": "I willuse a tool to get the weather in London.", + "revisionId": "id-4", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "reasoning": "I willuse a tool to get the weather in London.I know know the weather in London.", + "type": "reasoning", + }, + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + ], + "reasoning": "I willuse a tool to get the weather in London.I know know the weather in London.", + "revisionId": "id-5", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "The weather in London is sunny.", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "reasoning": "I willuse a tool to get the weather in London.I know know the weather in London.", + "type": "reasoning", + }, + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + { + "text": "The weather in London is sunny.", + "type": "text", + }, + ], + "reasoning": "I willuse a tool to get the weather in London.I know know the weather in London.", + "revisionId": "id-6", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": false, + }, +] +`; + +exports[`scenario: server-side tool roundtrip with multiple assistant texts > should call the onFinish function with the correct arguments 1`] = ` +[ + { + "finishReason": "stop", + "message": { + "content": "I willuse a tool to get the weather in London.The weather in Londonis sunny.", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "I willuse a tool to get the weather in London.The weather in Londonis sunny.", + "type": "text", + }, + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + ], + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "usage": { + "completionTokens": 7, + "promptTokens": 14, + "totalTokens": 21, + }, + }, +] +`; + +exports[`scenario: server-side tool roundtrip with multiple assistant texts > should call the update function with the correct arguments 1`] = ` +[ + { + "data": [], + "message": { + "content": "I will", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "I will", + "type": "text", + }, + ], + "revisionId": "id-1", + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "I willuse a tool to get the weather in London.", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "I willuse a tool to get the weather in London.", + "type": "text", + }, + ], + "revisionId": "id-2", + "role": "assistant", + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "I willuse a tool to get the weather in London.", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "I willuse a tool to get the weather in London.", + "type": "text", + }, + { + "toolInvocation": { + "args": { + "city": "London", + }, + "state": "call", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + ], + "revisionId": "id-3", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "state": "call", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "I willuse a tool to get the weather in London.", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "I willuse a tool to get the weather in London.", + "type": "text", + }, + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + ], + "revisionId": "id-4", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "I willuse a tool to get the weather in London.The weather in London", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "I willuse a tool to get the weather in London.The weather in London", + "type": "text", + }, + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + ], + "revisionId": "id-5", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": false, + }, + { + "data": [], + "message": { + "content": "I willuse a tool to get the weather in London.The weather in Londonis sunny.", + "createdAt": 2023-01-01T00:00:00.000Z, + "id": "id-0", + "parts": [ + { + "text": "I willuse a tool to get the weather in London.The weather in Londonis sunny.", + "type": "text", + }, + { + "toolInvocation": { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + "type": "tool-invocation", + }, + ], + "revisionId": "id-6", + "role": "assistant", + "toolInvocations": [ + { + "args": { + "city": "London", + }, + "result": { + "weather": "sunny", + }, + "state": "result", + "step": 0, + "toolCallId": "tool-call-id", + "toolName": "tool-name", + }, + ], + }, + "replaceLastMessage": false, + }, +] +`; + exports[`scenario: simple text response > should call the onFinish function with the correct arguments 1`] = ` [ { diff --git a/packages/ui-utils/src/process-chat-response.test.ts b/packages/ui-utils/src/process-chat-response.test.ts index 24077acdc588..98b8ea793f6b 100644 --- a/packages/ui-utils/src/process-chat-response.test.ts +++ b/packages/ui-utils/src/process-chat-response.test.ts @@ -204,6 +204,111 @@ describe('scenario: server-side tool roundtrip with existing assistant message', }); }); +describe('scenario: server-side tool roundtrip with multiple assistant texts', () => { + beforeEach(async () => { + const stream = createDataProtocolStream([ + formatDataStreamPart('text', 'I will'), + formatDataStreamPart('text', 'use a tool to get the weather in London.'), + formatDataStreamPart('tool_call', { + toolCallId: 'tool-call-id', + toolName: 'tool-name', + args: { city: 'London' }, + }), + formatDataStreamPart('tool_result', { + toolCallId: 'tool-call-id', + result: { weather: 'sunny' }, + }), + formatDataStreamPart('finish_step', { + finishReason: 'tool-calls', + usage: { completionTokens: 5, promptTokens: 10 }, + isContinued: false, + }), + formatDataStreamPart('text', 'The weather in London'), + formatDataStreamPart('text', 'is sunny.'), + formatDataStreamPart('finish_step', { + finishReason: 'stop', + usage: { completionTokens: 2, promptTokens: 4 }, + isContinued: false, + }), + formatDataStreamPart('finish_message', { + finishReason: 'stop', + usage: { completionTokens: 7, promptTokens: 14 }, + }), + ]); + + await processChatResponse({ + stream, + update, + onFinish, + generateId: mockId(), + getCurrentDate: vi.fn().mockReturnValue(new Date('2023-01-01')), + lastMessage: undefined, + }); + }); + + it('should call the update function with the correct arguments', async () => { + expect(updateCalls).toMatchSnapshot(); + }); + + it('should call the onFinish function with the correct arguments', async () => { + expect(finishCalls).toMatchSnapshot(); + }); +}); + +describe('scenario: server-side tool roundtrip with multiple assistant reasoning', () => { + beforeEach(async () => { + const stream = createDataProtocolStream([ + formatDataStreamPart('reasoning', 'I will'), + formatDataStreamPart( + 'reasoning', + 'use a tool to get the weather in London.', + ), + formatDataStreamPart('tool_call', { + toolCallId: 'tool-call-id', + toolName: 'tool-name', + args: { city: 'London' }, + }), + formatDataStreamPart('tool_result', { + toolCallId: 'tool-call-id', + result: { weather: 'sunny' }, + }), + formatDataStreamPart('finish_step', { + finishReason: 'tool-calls', + usage: { completionTokens: 5, promptTokens: 10 }, + isContinued: false, + }), + formatDataStreamPart('reasoning', 'I know know the weather in London.'), + formatDataStreamPart('text', 'The weather in London is sunny.'), + formatDataStreamPart('finish_step', { + finishReason: 'stop', + usage: { completionTokens: 2, promptTokens: 4 }, + isContinued: false, + }), + formatDataStreamPart('finish_message', { + finishReason: 'stop', + usage: { completionTokens: 7, promptTokens: 14 }, + }), + ]); + + await processChatResponse({ + stream, + update, + onFinish, + generateId: mockId(), + getCurrentDate: vi.fn().mockReturnValue(new Date('2023-01-01')), + lastMessage: undefined, + }); + }); + + it('should call the update function with the correct arguments', async () => { + expect(updateCalls).toMatchSnapshot(); + }); + + it('should call the onFinish function with the correct arguments', async () => { + expect(finishCalls).toMatchSnapshot(); + }); +}); + describe('scenario: server-side continue roundtrip', () => { beforeEach(async () => { const stream = createDataProtocolStream([ From 5c2793e4533114a0f9052ec9918c906bb5b8b743 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 10:11:49 +0100 Subject: [PATCH 29/61] start new reasoning / text parts --- .../process-chat-response.test.ts.snap | 80 ++++++++++++------- .../src/process-chat-response.test.ts | 6 +- .../ui-utils/src/process-chat-response.ts | 4 + 3 files changed, 59 insertions(+), 31 deletions(-) diff --git a/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap b/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap index c1bc951a7c7e..2beca25cc671 100644 --- a/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap +++ b/packages/ui-utils/src/__snapshots__/process-chat-response.test.ts.snap @@ -1092,7 +1092,7 @@ exports[`scenario: server-side tool roundtrip with multiple assistant reasoning "id": "id-0", "parts": [ { - "reasoning": "I willuse a tool to get the weather in London.I know know the weather in London.", + "reasoning": "I will use a tool to get the weather in London.", "type": "reasoning", }, { @@ -1110,12 +1110,16 @@ exports[`scenario: server-side tool roundtrip with multiple assistant reasoning }, "type": "tool-invocation", }, + { + "reasoning": "I know know the weather in London.", + "type": "reasoning", + }, { "text": "The weather in London is sunny.", "type": "text", }, ], - "reasoning": "I willuse a tool to get the weather in London.I know know the weather in London.", + "reasoning": "I will use a tool to get the weather in London.I know know the weather in London.", "role": "assistant", "toolInvocations": [ { @@ -1151,11 +1155,11 @@ exports[`scenario: server-side tool roundtrip with multiple assistant reasoning "id": "id-0", "parts": [ { - "reasoning": "I will", + "reasoning": "I will ", "type": "reasoning", }, ], - "reasoning": "I will", + "reasoning": "I will ", "revisionId": "id-1", "role": "assistant", }, @@ -1169,11 +1173,11 @@ exports[`scenario: server-side tool roundtrip with multiple assistant reasoning "id": "id-0", "parts": [ { - "reasoning": "I willuse a tool to get the weather in London.", + "reasoning": "I will use a tool to get the weather in London.", "type": "reasoning", }, ], - "reasoning": "I willuse a tool to get the weather in London.", + "reasoning": "I will use a tool to get the weather in London.", "revisionId": "id-2", "role": "assistant", }, @@ -1187,7 +1191,7 @@ exports[`scenario: server-side tool roundtrip with multiple assistant reasoning "id": "id-0", "parts": [ { - "reasoning": "I willuse a tool to get the weather in London.", + "reasoning": "I will use a tool to get the weather in London.", "type": "reasoning", }, { @@ -1203,7 +1207,7 @@ exports[`scenario: server-side tool roundtrip with multiple assistant reasoning "type": "tool-invocation", }, ], - "reasoning": "I willuse a tool to get the weather in London.", + "reasoning": "I will use a tool to get the weather in London.", "revisionId": "id-3", "role": "assistant", "toolInvocations": [ @@ -1228,7 +1232,7 @@ exports[`scenario: server-side tool roundtrip with multiple assistant reasoning "id": "id-0", "parts": [ { - "reasoning": "I willuse a tool to get the weather in London.", + "reasoning": "I will use a tool to get the weather in London.", "type": "reasoning", }, { @@ -1247,7 +1251,7 @@ exports[`scenario: server-side tool roundtrip with multiple assistant reasoning "type": "tool-invocation", }, ], - "reasoning": "I willuse a tool to get the weather in London.", + "reasoning": "I will use a tool to get the weather in London.", "revisionId": "id-4", "role": "assistant", "toolInvocations": [ @@ -1275,7 +1279,7 @@ exports[`scenario: server-side tool roundtrip with multiple assistant reasoning "id": "id-0", "parts": [ { - "reasoning": "I willuse a tool to get the weather in London.I know know the weather in London.", + "reasoning": "I will use a tool to get the weather in London.", "type": "reasoning", }, { @@ -1293,8 +1297,12 @@ exports[`scenario: server-side tool roundtrip with multiple assistant reasoning }, "type": "tool-invocation", }, + { + "reasoning": "I know know the weather in London.", + "type": "reasoning", + }, ], - "reasoning": "I willuse a tool to get the weather in London.I know know the weather in London.", + "reasoning": "I will use a tool to get the weather in London.I know know the weather in London.", "revisionId": "id-5", "role": "assistant", "toolInvocations": [ @@ -1322,7 +1330,7 @@ exports[`scenario: server-side tool roundtrip with multiple assistant reasoning "id": "id-0", "parts": [ { - "reasoning": "I willuse a tool to get the weather in London.I know know the weather in London.", + "reasoning": "I will use a tool to get the weather in London.", "type": "reasoning", }, { @@ -1340,12 +1348,16 @@ exports[`scenario: server-side tool roundtrip with multiple assistant reasoning }, "type": "tool-invocation", }, + { + "reasoning": "I know know the weather in London.", + "type": "reasoning", + }, { "text": "The weather in London is sunny.", "type": "text", }, ], - "reasoning": "I willuse a tool to get the weather in London.I know know the weather in London.", + "reasoning": "I will use a tool to get the weather in London.I know know the weather in London.", "revisionId": "id-6", "role": "assistant", "toolInvocations": [ @@ -1373,12 +1385,12 @@ exports[`scenario: server-side tool roundtrip with multiple assistant texts > sh { "finishReason": "stop", "message": { - "content": "I willuse a tool to get the weather in London.The weather in Londonis sunny.", + "content": "I will use a tool to get the weather in London.The weather in London is sunny.", "createdAt": 2023-01-01T00:00:00.000Z, "id": "id-0", "parts": [ { - "text": "I willuse a tool to get the weather in London.The weather in Londonis sunny.", + "text": "I will use a tool to get the weather in London.", "type": "text", }, { @@ -1396,6 +1408,10 @@ exports[`scenario: server-side tool roundtrip with multiple assistant texts > sh }, "type": "tool-invocation", }, + { + "text": "The weather in London is sunny.", + "type": "text", + }, ], "role": "assistant", "toolInvocations": [ @@ -1427,12 +1443,12 @@ exports[`scenario: server-side tool roundtrip with multiple assistant texts > sh { "data": [], "message": { - "content": "I will", + "content": "I will ", "createdAt": 2023-01-01T00:00:00.000Z, "id": "id-0", "parts": [ { - "text": "I will", + "text": "I will ", "type": "text", }, ], @@ -1444,12 +1460,12 @@ exports[`scenario: server-side tool roundtrip with multiple assistant texts > sh { "data": [], "message": { - "content": "I willuse a tool to get the weather in London.", + "content": "I will use a tool to get the weather in London.", "createdAt": 2023-01-01T00:00:00.000Z, "id": "id-0", "parts": [ { - "text": "I willuse a tool to get the weather in London.", + "text": "I will use a tool to get the weather in London.", "type": "text", }, ], @@ -1461,12 +1477,12 @@ exports[`scenario: server-side tool roundtrip with multiple assistant texts > sh { "data": [], "message": { - "content": "I willuse a tool to get the weather in London.", + "content": "I will use a tool to get the weather in London.", "createdAt": 2023-01-01T00:00:00.000Z, "id": "id-0", "parts": [ { - "text": "I willuse a tool to get the weather in London.", + "text": "I will use a tool to get the weather in London.", "type": "text", }, { @@ -1501,12 +1517,12 @@ exports[`scenario: server-side tool roundtrip with multiple assistant texts > sh { "data": [], "message": { - "content": "I willuse a tool to get the weather in London.", + "content": "I will use a tool to get the weather in London.", "createdAt": 2023-01-01T00:00:00.000Z, "id": "id-0", "parts": [ { - "text": "I willuse a tool to get the weather in London.", + "text": "I will use a tool to get the weather in London.", "type": "text", }, { @@ -1547,12 +1563,12 @@ exports[`scenario: server-side tool roundtrip with multiple assistant texts > sh { "data": [], "message": { - "content": "I willuse a tool to get the weather in London.The weather in London", + "content": "I will use a tool to get the weather in London.The weather in London ", "createdAt": 2023-01-01T00:00:00.000Z, "id": "id-0", "parts": [ { - "text": "I willuse a tool to get the weather in London.The weather in London", + "text": "I will use a tool to get the weather in London.", "type": "text", }, { @@ -1570,6 +1586,10 @@ exports[`scenario: server-side tool roundtrip with multiple assistant texts > sh }, "type": "tool-invocation", }, + { + "text": "The weather in London ", + "type": "text", + }, ], "revisionId": "id-5", "role": "assistant", @@ -1593,12 +1613,12 @@ exports[`scenario: server-side tool roundtrip with multiple assistant texts > sh { "data": [], "message": { - "content": "I willuse a tool to get the weather in London.The weather in Londonis sunny.", + "content": "I will use a tool to get the weather in London.The weather in London is sunny.", "createdAt": 2023-01-01T00:00:00.000Z, "id": "id-0", "parts": [ { - "text": "I willuse a tool to get the weather in London.The weather in Londonis sunny.", + "text": "I will use a tool to get the weather in London.", "type": "text", }, { @@ -1616,6 +1636,10 @@ exports[`scenario: server-side tool roundtrip with multiple assistant texts > sh }, "type": "tool-invocation", }, + { + "text": "The weather in London is sunny.", + "type": "text", + }, ], "revisionId": "id-6", "role": "assistant", diff --git a/packages/ui-utils/src/process-chat-response.test.ts b/packages/ui-utils/src/process-chat-response.test.ts index 98b8ea793f6b..531394567916 100644 --- a/packages/ui-utils/src/process-chat-response.test.ts +++ b/packages/ui-utils/src/process-chat-response.test.ts @@ -207,7 +207,7 @@ describe('scenario: server-side tool roundtrip with existing assistant message', describe('scenario: server-side tool roundtrip with multiple assistant texts', () => { beforeEach(async () => { const stream = createDataProtocolStream([ - formatDataStreamPart('text', 'I will'), + formatDataStreamPart('text', 'I will '), formatDataStreamPart('text', 'use a tool to get the weather in London.'), formatDataStreamPart('tool_call', { toolCallId: 'tool-call-id', @@ -223,7 +223,7 @@ describe('scenario: server-side tool roundtrip with multiple assistant texts', ( usage: { completionTokens: 5, promptTokens: 10 }, isContinued: false, }), - formatDataStreamPart('text', 'The weather in London'), + formatDataStreamPart('text', 'The weather in London '), formatDataStreamPart('text', 'is sunny.'), formatDataStreamPart('finish_step', { finishReason: 'stop', @@ -258,7 +258,7 @@ describe('scenario: server-side tool roundtrip with multiple assistant texts', ( describe('scenario: server-side tool roundtrip with multiple assistant reasoning', () => { beforeEach(async () => { const stream = createDataProtocolStream([ - formatDataStreamPart('reasoning', 'I will'), + formatDataStreamPart('reasoning', 'I will '), formatDataStreamPart( 'reasoning', 'use a tool to get the weather in London.', diff --git a/packages/ui-utils/src/process-chat-response.ts b/packages/ui-utils/src/process-chat-response.ts index 7c5f38fe10fd..32745c4ce253 100644 --- a/packages/ui-utils/src/process-chat-response.ts +++ b/packages/ui-utils/src/process-chat-response.ts @@ -300,6 +300,10 @@ export async function processChatResponse({ }, onFinishStepPart(value) { step += 1; + + // reset the current text and reasoning parts + currentTextPart = value.isContinued ? currentTextPart : undefined; + currentReasoningPart = undefined; }, onStartStepPart(value) { // keep message id stable when we are updating an existing message: From 91f12f7a624c46a452987ead2004add50ac1ec9c Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 10:27:48 +0100 Subject: [PATCH 30/61] renane --- .../ai/core/prompt/convert-to-core-messages.ts | 10 ++++++---- .../ai/core/prompt/detect-prompt-type.test.ts | 8 ++++---- .../ai/core/prompt/message-conversion-error.ts | 6 +++--- packages/ai/core/prompt/prompt.ts | 4 ++-- packages/ai/core/prompt/standardize-prompt.ts | 4 ++-- packages/ai/core/prompt/ui-message.ts | 16 +++++++++++++--- 6 files changed, 30 insertions(+), 18 deletions(-) diff --git a/packages/ai/core/prompt/convert-to-core-messages.ts b/packages/ai/core/prompt/convert-to-core-messages.ts index 090d0447d572..55935417a407 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.ts @@ -1,15 +1,16 @@ +import { ToolInvocationUIPart } from '@ai-sdk/ui-utils'; import { ToolSet } from '../generate-text/tool-set'; import { CoreMessage, ToolCallPart, ToolResultPart } from '../prompt'; import { attachmentsToParts } from './attachments-to-parts'; import { MessageConversionError } from './message-conversion-error'; -import { UIMessage } from './ui-message'; +import { InternalUIMessage } from './ui-message'; /** Converts an array of messages from useChat into an array of CoreMessages that can be used with the AI core functions (e.g. `streamText`). */ export function convertToCoreMessages( - messages: Array, + messages: Array, options?: { tools?: TOOLS }, ) { const tools = options?.tools ?? ({} as TOOLS); @@ -18,8 +19,7 @@ export function convertToCoreMessages( for (let i = 0; i < messages.length; i++) { const message = messages[i]; const isLastMessage = i === messages.length - 1; - const { role, content, toolInvocations, experimental_attachments } = - message; + const { role, content, experimental_attachments } = message; switch (role) { case 'system': { @@ -44,6 +44,8 @@ export function convertToCoreMessages( } case 'assistant': { + const toolInvocations = message.toolInvocations; + if (toolInvocations == null || toolInvocations.length === 0) { coreMessages.push({ role: 'assistant', content }); break; diff --git a/packages/ai/core/prompt/detect-prompt-type.test.ts b/packages/ai/core/prompt/detect-prompt-type.test.ts index aa75dcccd40c..2473597bdce2 100644 --- a/packages/ai/core/prompt/detect-prompt-type.test.ts +++ b/packages/ai/core/prompt/detect-prompt-type.test.ts @@ -1,5 +1,5 @@ import { detectPromptType } from './detect-prompt-type'; -import type { UIMessage } from './ui-message'; +import type { InternalUIMessage } from './ui-message'; import type { CoreMessage } from './message'; it('should return "other" for invalid inputs', () => { @@ -13,7 +13,7 @@ it('should return "messages" for empty arrays', () => { }); it('should detect UI messages with data role', () => { - const messages: UIMessage[] = [ + const messages: InternalUIMessage[] = [ { role: 'data', content: 'some data', @@ -23,7 +23,7 @@ it('should detect UI messages with data role', () => { }); it('should detect UI messages with toolInvocations', () => { - const messages: UIMessage[] = [ + const messages: InternalUIMessage[] = [ { role: 'assistant', content: 'Hello', @@ -42,7 +42,7 @@ it('should detect UI messages with toolInvocations', () => { }); it('should detect UI messages with experimental_attachments', () => { - const messages: UIMessage[] = [ + const messages: InternalUIMessage[] = [ { role: 'user', content: 'Check this file', diff --git a/packages/ai/core/prompt/message-conversion-error.ts b/packages/ai/core/prompt/message-conversion-error.ts index 96b5fc039d7a..d00d26db0118 100644 --- a/packages/ai/core/prompt/message-conversion-error.ts +++ b/packages/ai/core/prompt/message-conversion-error.ts @@ -1,5 +1,5 @@ import { AISDKError } from '@ai-sdk/provider'; -import { UIMessage } from './ui-message'; +import { InternalUIMessage } from './ui-message'; const name = 'AI_MessageConversionError'; const marker = `vercel.ai.error.${name}`; @@ -8,13 +8,13 @@ const symbol = Symbol.for(marker); export class MessageConversionError extends AISDKError { private readonly [symbol] = true; // used in isInstance - readonly originalMessage: UIMessage; + readonly originalMessage: InternalUIMessage; constructor({ originalMessage, message, }: { - originalMessage: UIMessage; + originalMessage: InternalUIMessage; message: string; }) { super({ name, message }); diff --git a/packages/ai/core/prompt/prompt.ts b/packages/ai/core/prompt/prompt.ts index dd0bfc111aea..d4a3a4effc97 100644 --- a/packages/ai/core/prompt/prompt.ts +++ b/packages/ai/core/prompt/prompt.ts @@ -1,5 +1,5 @@ import { CoreMessage } from './message'; -import { UIMessage } from './ui-message'; +import { InternalUIMessage } from './ui-message'; /** Prompt part of the AI function options. @@ -19,5 +19,5 @@ A simple text prompt. You can either use `prompt` or `messages` but not both. /** A list of messages. You can either use `prompt` or `messages` but not both. */ - messages?: Array | Array; + messages?: Array | Array; }; diff --git a/packages/ai/core/prompt/standardize-prompt.ts b/packages/ai/core/prompt/standardize-prompt.ts index 1110b0180b4e..aaa519d7b311 100644 --- a/packages/ai/core/prompt/standardize-prompt.ts +++ b/packages/ai/core/prompt/standardize-prompt.ts @@ -6,7 +6,7 @@ import { convertToCoreMessages } from './convert-to-core-messages'; import { detectPromptType } from './detect-prompt-type'; import { CoreMessage, coreMessageSchema } from './message'; import { Prompt } from './prompt'; -import { UIMessage } from './ui-message'; +import { InternalUIMessage } from './ui-message'; export type StandardizedPrompt = { /** @@ -90,7 +90,7 @@ export function standardizePrompt({ const messages: CoreMessage[] = promptType === 'ui-messages' - ? convertToCoreMessages(prompt.messages as UIMessage[], { + ? convertToCoreMessages(prompt.messages as InternalUIMessage[], { tools, }) : (prompt.messages as CoreMessage[]); diff --git a/packages/ai/core/prompt/ui-message.ts b/packages/ai/core/prompt/ui-message.ts index 4bd1bd511eba..f98ef0c5f7ee 100644 --- a/packages/ai/core/prompt/ui-message.ts +++ b/packages/ai/core/prompt/ui-message.ts @@ -1,10 +1,20 @@ -import { Attachment, ToolInvocation } from '@ai-sdk/ui-utils'; +import { + Attachment, + ReasoningUIPart, + TextUIPart, + ToolInvocation, + ToolInvocationUIPart, +} from '@ai-sdk/ui-utils'; -// only for internal use - should be removed when we fully migrate to core messages -export type UIMessage = { +/** + * @internal + */ +export type InternalUIMessage = { role: 'system' | 'user' | 'assistant' | 'data'; content: string; toolInvocations?: ToolInvocation[]; experimental_attachments?: Attachment[]; + + parts?: Array; }; From bc269170ed7b9b523ac0bb9417567c503d46eb48 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 10:29:34 +0100 Subject: [PATCH 31/61] snapshot --- .../convert-to-core-messages.test.ts.snap | 342 ++++++++++++++++++ .../prompt/convert-to-core-messages.test.ts | 279 +------------- .../core/prompt/convert-to-core-messages.ts | 1 - 3 files changed, 350 insertions(+), 272 deletions(-) create mode 100644 packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap diff --git a/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap b/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap new file mode 100644 index 000000000000..dd03906ecba9 --- /dev/null +++ b/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap @@ -0,0 +1,342 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`assistant message > should handle assistant message with tool invocations 1`] = ` +[ + { + "content": [ + { + "text": "Let me calculate that for you.", + "type": "text", + }, + { + "args": { + "numbers": [ + 1, + 2, + ], + "operation": "add", + }, + "toolCallId": "call1", + "toolName": "calculator", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "3", + "toolCallId": "call1", + "toolName": "calculator", + "type": "tool-result", + }, + ], + "role": "tool", + }, +] +`; + +exports[`assistant message > should handle assistant message with tool invocations that have multi-part responses 1`] = ` +[ + { + "content": [ + { + "text": "Let me calculate that for you.", + "type": "text", + }, + { + "args": {}, + "toolCallId": "call1", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "experimental_content": [ + { + "data": "imgbase64", + "type": "image", + }, + ], + "result": [ + { + "data": "imgbase64", + "type": "image", + }, + ], + "toolCallId": "call1", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, +] +`; + +exports[`assistant message > should handle conversation with an assistant message that has empty tool invocations 1`] = ` +[ + { + "content": "text1", + "role": "user", + }, + { + "content": "text2", + "role": "assistant", + }, +] +`; + +exports[`assistant message > should handle conversation with multiple tool invocations that have step information 1`] = ` +[ + { + "content": [ + { + "text": "response", + "type": "text", + }, + { + "args": { + "value": "value-1", + }, + "toolCallId": "call-1", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-1", + "toolCallId": "call-1", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, + { + "content": [ + { + "args": { + "value": "value-2", + }, + "toolCallId": "call-2", + "toolName": "screenshot", + "type": "tool-call", + }, + { + "args": { + "value": "value-3", + }, + "toolCallId": "call-3", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-2", + "toolCallId": "call-2", + "toolName": "screenshot", + "type": "tool-result", + }, + { + "result": "result-3", + "toolCallId": "call-3", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, + { + "content": [ + { + "args": { + "value": "value-4", + }, + "toolCallId": "call-4", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-4", + "toolCallId": "call-4", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, +] +`; + +exports[`multiple messages > should convert fully typed Message[] 1`] = ` +[ + { + "content": "What is the weather in Tokyo?", + "role": "user", + }, + { + "content": "It is sunny in Tokyo.", + "role": "assistant", + }, +] +`; + +exports[`multiple messages > should handle conversation with multiple tool invocations and user message at the end 1`] = ` +[ + { + "content": [ + { + "args": { + "value": "value-1", + }, + "toolCallId": "call-1", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-1", + "toolCallId": "call-1", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, + { + "content": [ + { + "args": { + "value": "value-2", + }, + "toolCallId": "call-2", + "toolName": "screenshot", + "type": "tool-call", + }, + { + "args": { + "value": "value-3", + }, + "toolCallId": "call-3", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-2", + "toolCallId": "call-2", + "toolName": "screenshot", + "type": "tool-result", + }, + { + "result": "result-3", + "toolCallId": "call-3", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, + { + "content": [ + { + "args": { + "value": "value-4", + }, + "toolCallId": "call-4", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-4", + "toolCallId": "call-4", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, + { + "content": "response", + "role": "assistant", + }, + { + "content": "Thanks!", + "role": "user", + }, +] +`; + +exports[`user message > should handle user message with attachment URLs (file) 1`] = ` +[ + { + "content": [ + { + "text": "Check this document", + "type": "text", + }, + { + "data": "dGVzdA==", + "mimeType": "application/pdf", + "type": "file", + }, + ], + "role": "user", + }, +] +`; + +exports[`user message > should handle user message with attachment URLs 1`] = ` +[ + { + "content": [ + { + "text": "Check this image", + "type": "text", + }, + { + "image": Uint8Array [ + 116, + 101, + 115, + 116, + ], + "type": "image", + }, + ], + "role": "user", + }, +] +`; diff --git a/packages/ai/core/prompt/convert-to-core-messages.test.ts b/packages/ai/core/prompt/convert-to-core-messages.test.ts index 7881db3e5216..bfac444bf46c 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.test.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.test.ts @@ -90,15 +90,7 @@ describe('user message', () => { }, ]); - expect(result).toEqual([ - { - role: 'user', - content: [ - { type: 'text', text: 'Check this image' }, - { type: 'image', image: new Uint8Array([116, 101, 115, 116]) }, - ], - }, - ]); + expect(result).toMatchSnapshot(); }); it('should handle user message with attachment URLs (file)', () => { @@ -115,19 +107,7 @@ describe('user message', () => { }, ]); - expect(result).toEqual([ - { - role: 'user', - content: [ - { type: 'text', text: 'Check this document' }, - { - type: 'file', - data: 'dGVzdA==', - mimeType: 'application/pdf', - }, - ], - }, - ]); + expect(result).toMatchSnapshot(); }); it('should throw an error for invalid attachment URLs', () => { @@ -226,34 +206,7 @@ describe('assistant message', () => { }, ]); - expect(result).toEqual([ - { - role: 'assistant', - content: [ - { - type: 'text', - text: 'Let me calculate that for you.', - }, - { - type: 'tool-call', - toolCallId: 'call1', - toolName: 'calculator', - args: { operation: 'add', numbers: [1, 2] }, - }, - ], - }, - { - role: 'tool', - content: [ - { - type: 'tool-result', - toolCallId: 'call1', - toolName: 'calculator', - result: '3', - }, - ], - }, - ]); + expect(result).toMatchSnapshot(); }); it('should handle assistant message with tool invocations that have multi-part responses', () => { @@ -286,35 +239,7 @@ describe('assistant message', () => { { tools }, // separate tools to ensure that types are inferred correctly ); - expect(result).toEqual([ - { - role: 'assistant', - content: [ - { - type: 'text', - text: 'Let me calculate that for you.', - }, - { - type: 'tool-call', - toolCallId: 'call1', - toolName: 'screenshot', - args: {}, - }, - ], - }, - { - role: 'tool', - content: [ - { - type: 'tool-result', - toolCallId: 'call1', - toolName: 'screenshot', - result: [{ type: 'image', data: 'imgbase64' }], - experimental_content: [{ type: 'image', data: 'imgbase64' }], - }, - ], - }, - ]); + expect(result).toMatchSnapshot(); }); it('should handle conversation with an assistant message that has empty tool invocations', () => { @@ -331,16 +256,7 @@ describe('assistant message', () => { }, ]); - expect(result).toEqual([ - { - role: 'user', - content: 'text1', - }, - { - role: 'assistant', - content: 'text2', - }, - ]); + expect(result).toMatchSnapshot(); }); it('should handle conversation with multiple tool invocations that have step information', () => { @@ -396,90 +312,7 @@ describe('assistant message', () => { { tools }, // separate tools to ensure that types are inferred correctly ); - expect(result).toEqual([ - { - role: 'assistant', - content: [ - { - type: 'text', - text: 'response', - }, - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'screenshot', - args: { value: 'value-1' }, - }, - ], - }, - { - role: 'tool', - content: [ - { - type: 'tool-result', - toolCallId: 'call-1', - toolName: 'screenshot', - result: 'result-1', - }, - ], - }, - { - role: 'assistant', - content: [ - { - type: 'tool-call', - toolCallId: 'call-2', - toolName: 'screenshot', - args: { value: 'value-2' }, - }, - { - type: 'tool-call', - toolCallId: 'call-3', - toolName: 'screenshot', - args: { value: 'value-3' }, - }, - ], - }, - { - role: 'tool', - content: [ - { - type: 'tool-result', - toolCallId: 'call-2', - toolName: 'screenshot', - result: 'result-2', - }, - { - type: 'tool-result', - toolCallId: 'call-3', - toolName: 'screenshot', - result: 'result-3', - }, - ], - }, - { - role: 'assistant', - content: [ - { - type: 'tool-call', - toolCallId: 'call-4', - toolName: 'screenshot', - args: { value: 'value-4' }, - }, - ], - }, - { - role: 'tool', - content: [ - { - type: 'tool-result', - toolCallId: 'call-4', - toolName: 'screenshot', - result: 'result-4', - }, - ], - }, - ]); + expect(result).toMatchSnapshot(); }); }); @@ -514,16 +347,7 @@ describe('multiple messages', () => { const result = convertToCoreMessages(messages); - expect(result).toStrictEqual([ - { - role: 'user', - content: 'What is the weather in Tokyo?', - }, - { - role: 'assistant', - content: 'It is sunny in Tokyo.', - }, - ]); + expect(result).toMatchSnapshot(); }); it('should handle conversation with multiple tool invocations and user message at the end', () => { @@ -583,94 +407,7 @@ describe('multiple messages', () => { { tools }, // separate tools to ensure that types are inferred correctly ); - expect(result).toEqual([ - { - role: 'assistant', - content: [ - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'screenshot', - args: { value: 'value-1' }, - }, - ], - }, - { - role: 'tool', - content: [ - { - type: 'tool-result', - toolCallId: 'call-1', - toolName: 'screenshot', - result: 'result-1', - }, - ], - }, - { - role: 'assistant', - content: [ - { - type: 'tool-call', - toolCallId: 'call-2', - toolName: 'screenshot', - args: { value: 'value-2' }, - }, - { - type: 'tool-call', - toolCallId: 'call-3', - toolName: 'screenshot', - args: { value: 'value-3' }, - }, - ], - }, - { - role: 'tool', - content: [ - { - type: 'tool-result', - toolCallId: 'call-2', - toolName: 'screenshot', - result: 'result-2', - }, - { - type: 'tool-result', - toolCallId: 'call-3', - toolName: 'screenshot', - result: 'result-3', - }, - ], - }, - { - role: 'assistant', - content: [ - { - type: 'tool-call', - toolCallId: 'call-4', - toolName: 'screenshot', - args: { value: 'value-4' }, - }, - ], - }, - { - role: 'tool', - content: [ - { - type: 'tool-result', - toolCallId: 'call-4', - toolName: 'screenshot', - result: 'result-4', - }, - ], - }, - { - role: 'assistant', - content: 'response', - }, - { - role: 'user', - content: 'Thanks!', - }, - ]); + expect(result).toMatchSnapshot(); }); }); diff --git a/packages/ai/core/prompt/convert-to-core-messages.ts b/packages/ai/core/prompt/convert-to-core-messages.ts index 55935417a407..92cb6c5c49c3 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.ts @@ -1,4 +1,3 @@ -import { ToolInvocationUIPart } from '@ai-sdk/ui-utils'; import { ToolSet } from '../generate-text/tool-set'; import { CoreMessage, ToolCallPart, ToolResultPart } from '../prompt'; import { attachmentsToParts } from './attachments-to-parts'; From 60280d7fc5c21e653be8379c3313dabd74e1c3b2 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 10:30:27 +0100 Subject: [PATCH 32/61] structure --- .../convert-to-core-messages.test.ts.snap | 16 +- .../prompt/convert-to-core-messages.test.ts | 686 +++++++++--------- 2 files changed, 352 insertions(+), 350 deletions(-) diff --git a/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap b/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap index dd03906ecba9..8d64455e0717 100644 --- a/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap +++ b/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap @@ -1,6 +1,6 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html -exports[`assistant message > should handle assistant message with tool invocations 1`] = ` +exports[`convertToCoreMessages > assistant message > should handle assistant message with tool invocations 1`] = ` [ { "content": [ @@ -37,7 +37,7 @@ exports[`assistant message > should handle assistant message with tool invocatio ] `; -exports[`assistant message > should handle assistant message with tool invocations that have multi-part responses 1`] = ` +exports[`convertToCoreMessages > assistant message > should handle assistant message with tool invocations that have multi-part responses 1`] = ` [ { "content": [ @@ -79,7 +79,7 @@ exports[`assistant message > should handle assistant message with tool invocatio ] `; -exports[`assistant message > should handle conversation with an assistant message that has empty tool invocations 1`] = ` +exports[`convertToCoreMessages > assistant message > should handle conversation with an assistant message that has empty tool invocations 1`] = ` [ { "content": "text1", @@ -92,7 +92,7 @@ exports[`assistant message > should handle conversation with an assistant messag ] `; -exports[`assistant message > should handle conversation with multiple tool invocations that have step information 1`] = ` +exports[`convertToCoreMessages > assistant message > should handle conversation with multiple tool invocations that have step information 1`] = ` [ { "content": [ @@ -187,7 +187,7 @@ exports[`assistant message > should handle conversation with multiple tool invoc ] `; -exports[`multiple messages > should convert fully typed Message[] 1`] = ` +exports[`convertToCoreMessages > multiple messages > should convert fully typed Message[] 1`] = ` [ { "content": "What is the weather in Tokyo?", @@ -200,7 +200,7 @@ exports[`multiple messages > should convert fully typed Message[] 1`] = ` ] `; -exports[`multiple messages > should handle conversation with multiple tool invocations and user message at the end 1`] = ` +exports[`convertToCoreMessages > multiple messages > should handle conversation with multiple tool invocations and user message at the end 1`] = ` [ { "content": [ @@ -299,7 +299,7 @@ exports[`multiple messages > should handle conversation with multiple tool invoc ] `; -exports[`user message > should handle user message with attachment URLs (file) 1`] = ` +exports[`convertToCoreMessages > user message > should handle user message with attachment URLs (file) 1`] = ` [ { "content": [ @@ -318,7 +318,7 @@ exports[`user message > should handle user message with attachment URLs (file) 1 ] `; -exports[`user message > should handle user message with attachment URLs 1`] = ` +exports[`convertToCoreMessages > user message > should handle user message with attachment URLs 1`] = ` [ { "content": [ diff --git a/packages/ai/core/prompt/convert-to-core-messages.test.ts b/packages/ai/core/prompt/convert-to-core-messages.test.ts index bfac444bf46c..2c3dec3df0a0 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.test.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.test.ts @@ -3,225 +3,195 @@ import { convertToCoreMessages } from './convert-to-core-messages'; import { tool } from '../tool/tool'; import { z } from 'zod'; -describe('system message', () => { - it('should convert a simple system message', () => { - const result = convertToCoreMessages([ - { role: 'system', content: 'System message' }, - ]); - - expect(result).toEqual([{ role: 'system', content: 'System message' }]); - }); -}); - -describe('user message', () => { - it('should convert a simple user message', () => { - const result = convertToCoreMessages([ - { role: 'user', content: 'Hello, AI!' }, - ]); - - expect(result).toEqual([{ role: 'user', content: 'Hello, AI!' }]); - }); +describe('convertToCoreMessages', () => { + describe('system message', () => { + it('should convert a simple system message', () => { + const result = convertToCoreMessages([ + { role: 'system', content: 'System message' }, + ]); - it('should handle user message with attachments', () => { - const attachment: Attachment = { - contentType: 'image/jpeg', - url: 'https://example.com/image.jpg', - }; - - const result = convertToCoreMessages([ - { - role: 'user', - content: 'Check this image', - experimental_attachments: [attachment], - }, - ]); - - expect(result).toEqual([ - { - role: 'user', - content: [ - { type: 'text', text: 'Check this image' }, - { type: 'image', image: new URL('https://example.com/image.jpg') }, - ], - }, - ]); + expect(result).toEqual([{ role: 'system', content: 'System message' }]); + }); }); - it('should handle user message with attachments (file)', () => { - const attachment: Attachment = { - contentType: 'application/pdf', - url: 'https://example.com/document.pdf', - }; - - const result = convertToCoreMessages([ - { - role: 'user', - content: 'Check this document', - experimental_attachments: [attachment], - }, - ]); - - expect(result).toEqual([ - { - role: 'user', - content: [ - { type: 'text', text: 'Check this document' }, - { - type: 'file', - data: new URL('https://example.com/document.pdf'), - mimeType: 'application/pdf', - }, - ], - }, - ]); - }); + describe('user message', () => { + it('should convert a simple user message', () => { + const result = convertToCoreMessages([ + { role: 'user', content: 'Hello, AI!' }, + ]); - it('should handle user message with attachment URLs', () => { - const attachment: Attachment = { - contentType: 'image/jpeg', - url: 'data:image/jpg;base64,dGVzdA==', - }; - - const result = convertToCoreMessages([ - { - role: 'user', - content: 'Check this image', - experimental_attachments: [attachment], - }, - ]); - - expect(result).toMatchSnapshot(); - }); + expect(result).toEqual([{ role: 'user', content: 'Hello, AI!' }]); + }); - it('should handle user message with attachment URLs (file)', () => { - const attachment: Attachment = { - contentType: 'application/pdf', - url: 'data:application/pdf;base64,dGVzdA==', - }; - - const result = convertToCoreMessages([ - { - role: 'user', - content: 'Check this document', - experimental_attachments: [attachment], - }, - ]); - - expect(result).toMatchSnapshot(); - }); + it('should handle user message with attachments', () => { + const attachment: Attachment = { + contentType: 'image/jpeg', + url: 'https://example.com/image.jpg', + }; - it('should throw an error for invalid attachment URLs', () => { - const attachment: Attachment = { - contentType: 'image/jpeg', - url: 'invalid-url', - }; - - expect(() => { - convertToCoreMessages([ + const result = convertToCoreMessages([ { role: 'user', content: 'Check this image', experimental_attachments: [attachment], }, ]); - }).toThrow('Invalid URL: invalid-url'); - }); - it('should throw an error for file attachments without contentType', () => { - const attachment: Attachment = { - url: 'data:application/pdf;base64,dGVzdA==', - }; + expect(result).toEqual([ + { + role: 'user', + content: [ + { type: 'text', text: 'Check this image' }, + { type: 'image', image: new URL('https://example.com/image.jpg') }, + ], + }, + ]); + }); + + it('should handle user message with attachments (file)', () => { + const attachment: Attachment = { + contentType: 'application/pdf', + url: 'https://example.com/document.pdf', + }; - expect(() => { - convertToCoreMessages([ + const result = convertToCoreMessages([ { role: 'user', - content: 'Check this file', + content: 'Check this document', experimental_attachments: [attachment], }, ]); - }).toThrow( - 'If the attachment is not an image or text, it must specify a content type', - ); - }); - it('should throw an error for invalid data URL format', () => { - const attachment: Attachment = { - contentType: 'image/jpeg', - url: 'data:image/jpg;base64', - }; + expect(result).toEqual([ + { + role: 'user', + content: [ + { type: 'text', text: 'Check this document' }, + { + type: 'file', + data: new URL('https://example.com/document.pdf'), + mimeType: 'application/pdf', + }, + ], + }, + ]); + }); + + it('should handle user message with attachment URLs', () => { + const attachment: Attachment = { + contentType: 'image/jpeg', + url: 'data:image/jpg;base64,dGVzdA==', + }; - expect(() => { - convertToCoreMessages([ + const result = convertToCoreMessages([ { role: 'user', content: 'Check this image', experimental_attachments: [attachment], }, ]); - }).toThrow(`Invalid data URL format: ${attachment.url}`); - }); - it('should throw an error for unsupported attachment protocols', () => { - const attachment: Attachment = { - contentType: 'image/jpeg', - url: 'ftp://example.com/image.jpg', - }; + expect(result).toMatchSnapshot(); + }); - expect(() => { - convertToCoreMessages([ + it('should handle user message with attachment URLs (file)', () => { + const attachment: Attachment = { + contentType: 'application/pdf', + url: 'data:application/pdf;base64,dGVzdA==', + }; + + const result = convertToCoreMessages([ { role: 'user', - content: 'Check this image', + content: 'Check this document', experimental_attachments: [attachment], }, ]); - }).toThrow('Unsupported URL protocol: ftp:'); - }); -}); -describe('assistant message', () => { - it('should convert a simple assistant message', () => { - const result = convertToCoreMessages([ - { role: 'assistant', content: 'Hello, human!' }, - ]); + expect(result).toMatchSnapshot(); + }); - expect(result).toEqual([{ role: 'assistant', content: 'Hello, human!' }]); - }); + it('should throw an error for invalid attachment URLs', () => { + const attachment: Attachment = { + contentType: 'image/jpeg', + url: 'invalid-url', + }; - it('should handle assistant message with tool invocations', () => { - const result = convertToCoreMessages([ - { - role: 'assistant', - content: 'Let me calculate that for you.', - toolInvocations: [ + expect(() => { + convertToCoreMessages([ { - state: 'result', - toolCallId: 'call1', - toolName: 'calculator', - args: { operation: 'add', numbers: [1, 2] }, - result: '3', + role: 'user', + content: 'Check this image', + experimental_attachments: [attachment], }, - ], - }, - ]); + ]); + }).toThrow('Invalid URL: invalid-url'); + }); - expect(result).toMatchSnapshot(); + it('should throw an error for file attachments without contentType', () => { + const attachment: Attachment = { + url: 'data:application/pdf;base64,dGVzdA==', + }; + + expect(() => { + convertToCoreMessages([ + { + role: 'user', + content: 'Check this file', + experimental_attachments: [attachment], + }, + ]); + }).toThrow( + 'If the attachment is not an image or text, it must specify a content type', + ); + }); + + it('should throw an error for invalid data URL format', () => { + const attachment: Attachment = { + contentType: 'image/jpeg', + url: 'data:image/jpg;base64', + }; + + expect(() => { + convertToCoreMessages([ + { + role: 'user', + content: 'Check this image', + experimental_attachments: [attachment], + }, + ]); + }).toThrow(`Invalid data URL format: ${attachment.url}`); + }); + + it('should throw an error for unsupported attachment protocols', () => { + const attachment: Attachment = { + contentType: 'image/jpeg', + url: 'ftp://example.com/image.jpg', + }; + + expect(() => { + convertToCoreMessages([ + { + role: 'user', + content: 'Check this image', + experimental_attachments: [attachment], + }, + ]); + }).toThrow('Unsupported URL protocol: ftp:'); + }); }); - it('should handle assistant message with tool invocations that have multi-part responses', () => { - const tools = { - screenshot: tool({ - parameters: z.object({}), - execute: async () => 'imgbase64', - experimental_toToolResultContent: result => [ - { type: 'image', data: result }, - ], - }), - }; + describe('assistant message', () => { + it('should convert a simple assistant message', () => { + const result = convertToCoreMessages([ + { role: 'assistant', content: 'Hello, human!' }, + ]); - const result = convertToCoreMessages( - [ + expect(result).toEqual([{ role: 'assistant', content: 'Hello, human!' }]); + }); + + it('should handle assistant message with tool invocations', () => { + const result = convertToCoreMessages([ { role: 'assistant', content: 'Let me calculate that for you.', @@ -229,194 +199,226 @@ describe('assistant message', () => { { state: 'result', toolCallId: 'call1', - toolName: 'screenshot', - args: {}, - result: 'imgbase64', + toolName: 'calculator', + args: { operation: 'add', numbers: [1, 2] }, + result: '3', }, ], }, - ], - { tools }, // separate tools to ensure that types are inferred correctly - ); + ]); - expect(result).toMatchSnapshot(); - }); + expect(result).toMatchSnapshot(); + }); - it('should handle conversation with an assistant message that has empty tool invocations', () => { - const result = convertToCoreMessages([ - { - role: 'user', - content: 'text1', - toolInvocations: [], - }, - { - role: 'assistant', - content: 'text2', - toolInvocations: [], - }, - ]); - - expect(result).toMatchSnapshot(); - }); + it('should handle assistant message with tool invocations that have multi-part responses', () => { + const tools = { + screenshot: tool({ + parameters: z.object({}), + execute: async () => 'imgbase64', + experimental_toToolResultContent: result => [ + { type: 'image', data: result }, + ], + }), + }; - it('should handle conversation with multiple tool invocations that have step information', () => { - const tools = { - screenshot: tool({ - parameters: z.object({ value: z.string() }), - execute: async () => 'imgbase64', - }), - }; + const result = convertToCoreMessages( + [ + { + role: 'assistant', + content: 'Let me calculate that for you.', + toolInvocations: [ + { + state: 'result', + toolCallId: 'call1', + toolName: 'screenshot', + args: {}, + result: 'imgbase64', + }, + ], + }, + ], + { tools }, // separate tools to ensure that types are inferred correctly + ); + + expect(result).toMatchSnapshot(); + }); - const result = convertToCoreMessages( - [ + it('should handle conversation with an assistant message that has empty tool invocations', () => { + const result = convertToCoreMessages([ + { + role: 'user', + content: 'text1', + toolInvocations: [], + }, { role: 'assistant', - content: 'response', - toolInvocations: [ - { - state: 'result', - toolCallId: 'call-1', - toolName: 'screenshot', - args: { value: 'value-1' }, - result: 'result-1', - step: 0, - }, - { - state: 'result', - toolCallId: 'call-2', - toolName: 'screenshot', - args: { value: 'value-2' }, - result: 'result-2', - step: 1, - }, - - { - state: 'result', - toolCallId: 'call-3', - toolName: 'screenshot', - args: { value: 'value-3' }, - result: 'result-3', - step: 1, - }, - { - state: 'result', - toolCallId: 'call-4', - toolName: 'screenshot', - args: { value: 'value-4' }, - result: 'result-4', - step: 2, - }, - ], + content: 'text2', + toolInvocations: [], }, - ], - { tools }, // separate tools to ensure that types are inferred correctly - ); + ]); - expect(result).toMatchSnapshot(); - }); -}); + expect(result).toMatchSnapshot(); + }); -describe('multiple messages', () => { - it('should handle a conversation with multiple messages', () => { - const result = convertToCoreMessages([ - { role: 'user', content: "What's the weather like?" }, - { role: 'assistant', content: "I'll check that for you." }, - { role: 'user', content: 'Thanks!' }, - ]); - - expect(result).toEqual([ - { role: 'user', content: "What's the weather like?" }, - { role: 'assistant', content: "I'll check that for you." }, - { role: 'user', content: 'Thanks!' }, - ]); - }); + it('should handle conversation with multiple tool invocations that have step information', () => { + const tools = { + screenshot: tool({ + parameters: z.object({ value: z.string() }), + execute: async () => 'imgbase64', + }), + }; + + const result = convertToCoreMessages( + [ + { + role: 'assistant', + content: 'response', + toolInvocations: [ + { + state: 'result', + toolCallId: 'call-1', + toolName: 'screenshot', + args: { value: 'value-1' }, + result: 'result-1', + step: 0, + }, + { + state: 'result', + toolCallId: 'call-2', + toolName: 'screenshot', + args: { value: 'value-2' }, + result: 'result-2', + step: 1, + }, + + { + state: 'result', + toolCallId: 'call-3', + toolName: 'screenshot', + args: { value: 'value-3' }, + result: 'result-3', + step: 1, + }, + { + state: 'result', + toolCallId: 'call-4', + toolName: 'screenshot', + args: { value: 'value-4' }, + result: 'result-4', + step: 2, + }, + ], + }, + ], + { tools }, // separate tools to ensure that types are inferred correctly + ); - it('should convert fully typed Message[]', () => { - const messages: Message[] = [ - { - id: '1', - role: 'user', - content: 'What is the weather in Tokyo?', - }, - { - id: '2', - role: 'assistant', - content: 'It is sunny in Tokyo.', - }, - ]; - - const result = convertToCoreMessages(messages); - - expect(result).toMatchSnapshot(); + expect(result).toMatchSnapshot(); + }); }); - it('should handle conversation with multiple tool invocations and user message at the end', () => { - const tools = { - screenshot: tool({ - parameters: z.object({ value: z.string() }), - execute: async () => 'imgbase64', - }), - }; + describe('multiple messages', () => { + it('should handle a conversation with multiple messages', () => { + const result = convertToCoreMessages([ + { role: 'user', content: "What's the weather like?" }, + { role: 'assistant', content: "I'll check that for you." }, + { role: 'user', content: 'Thanks!' }, + ]); - const result = convertToCoreMessages( - [ - { - role: 'assistant', - content: 'response', - toolInvocations: [ - { - state: 'result', - toolCallId: 'call-1', - toolName: 'screenshot', - args: { value: 'value-1' }, - result: 'result-1', - step: 0, - }, - { - state: 'result', - toolCallId: 'call-2', - toolName: 'screenshot', - args: { value: 'value-2' }, - result: 'result-2', - step: 1, - }, + expect(result).toEqual([ + { role: 'user', content: "What's the weather like?" }, + { role: 'assistant', content: "I'll check that for you." }, + { role: 'user', content: 'Thanks!' }, + ]); + }); - { - state: 'result', - toolCallId: 'call-3', - toolName: 'screenshot', - args: { value: 'value-3' }, - result: 'result-3', - step: 1, - }, - { - state: 'result', - toolCallId: 'call-4', - toolName: 'screenshot', - args: { value: 'value-4' }, - result: 'result-4', - step: 2, - }, - ], - }, + it('should convert fully typed Message[]', () => { + const messages: Message[] = [ { + id: '1', role: 'user', - content: 'Thanks!', + content: 'What is the weather in Tokyo?', }, - ], - { tools }, // separate tools to ensure that types are inferred correctly - ); + { + id: '2', + role: 'assistant', + content: 'It is sunny in Tokyo.', + }, + ]; + + const result = convertToCoreMessages(messages); + + expect(result).toMatchSnapshot(); + }); + + it('should handle conversation with multiple tool invocations and user message at the end', () => { + const tools = { + screenshot: tool({ + parameters: z.object({ value: z.string() }), + execute: async () => 'imgbase64', + }), + }; + + const result = convertToCoreMessages( + [ + { + role: 'assistant', + content: 'response', + toolInvocations: [ + { + state: 'result', + toolCallId: 'call-1', + toolName: 'screenshot', + args: { value: 'value-1' }, + result: 'result-1', + step: 0, + }, + { + state: 'result', + toolCallId: 'call-2', + toolName: 'screenshot', + args: { value: 'value-2' }, + result: 'result-2', + step: 1, + }, + + { + state: 'result', + toolCallId: 'call-3', + toolName: 'screenshot', + args: { value: 'value-3' }, + result: 'result-3', + step: 1, + }, + { + state: 'result', + toolCallId: 'call-4', + toolName: 'screenshot', + args: { value: 'value-4' }, + result: 'result-4', + step: 2, + }, + ], + }, + { + role: 'user', + content: 'Thanks!', + }, + ], + { tools }, // separate tools to ensure that types are inferred correctly + ); - expect(result).toMatchSnapshot(); + expect(result).toMatchSnapshot(); + }); }); -}); -describe('error handling', () => { - it('should throw an error for unhandled roles', () => { - expect(() => { - convertToCoreMessages([ - { role: 'unknown' as any, content: 'unknown role message' }, - ]); - }).toThrow('Unsupported role: unknown'); + describe('error handling', () => { + it('should throw an error for unhandled roles', () => { + expect(() => { + convertToCoreMessages([ + { role: 'unknown' as any, content: 'unknown role message' }, + ]); + }).toThrow('Unsupported role: unknown'); + }); }); }); From 4211fc08a65e3cc55ee270b48e75b9436d072055 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 10:33:14 +0100 Subject: [PATCH 33/61] 1 --- .../ai/core/prompt/convert-to-core-messages.test.ts | 12 ++++++++++++ packages/ai/core/prompt/convert-to-core-messages.ts | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/packages/ai/core/prompt/convert-to-core-messages.test.ts b/packages/ai/core/prompt/convert-to-core-messages.test.ts index 2c3dec3df0a0..ed4c904d823b 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.test.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.test.ts @@ -190,6 +190,18 @@ describe('convertToCoreMessages', () => { expect(result).toEqual([{ role: 'assistant', content: 'Hello, human!' }]); }); + it('should convert a simple assistant message with parts', () => { + const result = convertToCoreMessages([ + { + role: 'assistant', + content: '', // empty content + parts: [{ type: 'text', text: 'Hello, human!' }], + }, + ]); + + expect(result).toEqual([{ role: 'assistant', content: 'Hello, human!' }]); + }); + it('should handle assistant message with tool invocations', () => { const result = convertToCoreMessages([ { diff --git a/packages/ai/core/prompt/convert-to-core-messages.ts b/packages/ai/core/prompt/convert-to-core-messages.ts index 92cb6c5c49c3..6030b1405f2c 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.ts @@ -43,6 +43,18 @@ export function convertToCoreMessages( } case 'assistant': { + if (message.parts != null) { + for (const part of message.parts) { + switch (part.type) { + case 'text': { + coreMessages.push({ role: 'assistant', content: part.text }); + break; + } + } + } + break; + } + const toolInvocations = message.toolInvocations; if (toolInvocations == null || toolInvocations.length === 0) { From 4cf4c2b44614bde28491298d9c2e1a50982f858d Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 10:56:46 +0100 Subject: [PATCH 34/61] 2 --- .../convert-to-core-messages.test.ts.snap | 37 ++++++ .../prompt/convert-to-core-messages.test.ts | 35 +++++- .../core/prompt/convert-to-core-messages.ts | 111 +++++++++++++++++- 3 files changed, 179 insertions(+), 4 deletions(-) diff --git a/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap b/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap index 8d64455e0717..241153815fc4 100644 --- a/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap +++ b/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap @@ -1,5 +1,42 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html +exports[`convertToCoreMessages > assistant message > should handle assistant message with tool invocations (parts) 1`] = ` +[ + { + "content": [ + { + "text": "Let me calculate that for you.", + "type": "text", + }, + { + "args": { + "numbers": [ + 1, + 2, + ], + "operation": "add", + }, + "toolCallId": "call1", + "toolName": "calculator", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "3", + "toolCallId": "call1", + "toolName": "calculator", + "type": "tool-result", + }, + ], + "role": "tool", + }, +] +`; + exports[`convertToCoreMessages > assistant message > should handle assistant message with tool invocations 1`] = ` [ { diff --git a/packages/ai/core/prompt/convert-to-core-messages.test.ts b/packages/ai/core/prompt/convert-to-core-messages.test.ts index ed4c904d823b..038e480214cb 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.test.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.test.ts @@ -190,7 +190,7 @@ describe('convertToCoreMessages', () => { expect(result).toEqual([{ role: 'assistant', content: 'Hello, human!' }]); }); - it('should convert a simple assistant message with parts', () => { + it('should convert a simple assistant message (parts)', () => { const result = convertToCoreMessages([ { role: 'assistant', @@ -199,7 +199,12 @@ describe('convertToCoreMessages', () => { }, ]); - expect(result).toEqual([{ role: 'assistant', content: 'Hello, human!' }]); + expect(result).toEqual([ + { + role: 'assistant', + content: [{ type: 'text', text: 'Hello, human!' }], + }, + ]); }); it('should handle assistant message with tool invocations', () => { @@ -222,6 +227,32 @@ describe('convertToCoreMessages', () => { expect(result).toMatchSnapshot(); }); + it('should handle assistant message with tool invocations (parts)', () => { + const result = convertToCoreMessages([ + { + role: 'assistant', + content: '', // empty content + toolInvocations: [], // empty invocations + parts: [ + { type: 'text', text: 'Let me calculate that for you.' }, + { + type: 'tool-invocation', + toolInvocation: { + state: 'result', + toolCallId: 'call1', + toolName: 'calculator', + args: { operation: 'add', numbers: [1, 2] }, + result: '3', + step: 0, + }, + }, + ], + }, + ]); + + expect(result).toMatchSnapshot(); + }); + it('should handle assistant message with tool invocations that have multi-part responses', () => { const tools = { screenshot: tool({ diff --git a/packages/ai/core/prompt/convert-to-core-messages.ts b/packages/ai/core/prompt/convert-to-core-messages.ts index 6030b1405f2c..d727212af257 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.ts @@ -1,5 +1,15 @@ +import { + ReasoningUIPart, + TextUIPart, + ToolInvocationUIPart, +} from '@ai-sdk/ui-utils'; import { ToolSet } from '../generate-text/tool-set'; -import { CoreMessage, ToolCallPart, ToolResultPart } from '../prompt'; +import { + AssistantContent, + CoreMessage, + ToolCallPart, + ToolResultPart, +} from '../prompt'; import { attachmentsToParts } from './attachments-to-parts'; import { MessageConversionError } from './message-conversion-error'; import { InternalUIMessage } from './ui-message'; @@ -44,14 +54,111 @@ export function convertToCoreMessages( case 'assistant': { if (message.parts != null) { + let currentStep = 0; + let blockHasToolInvocations = false; + let block: Array = []; + + function processBlock() { + coreMessages.push({ + role: 'assistant', + content: block.map(part => { + switch (part.type) { + case 'text': + return { + type: 'text' as const, + text: part.text, + }; + default: + return { + type: 'tool-call' as const, + toolCallId: part.toolInvocation.toolCallId, + toolName: part.toolInvocation.toolName, + args: part.toolInvocation.args, + }; + } + }), + }); + + // check if there are tool invocations with results in the block + const stepInvocations = block + .filter( + ( + part: TextUIPart | ToolInvocationUIPart, + ): part is ToolInvocationUIPart => + part.type === 'tool-invocation', + ) + .map(part => part.toolInvocation); + + // tool message with tool results + if (stepInvocations.length > 0) { + coreMessages.push({ + role: 'tool', + content: stepInvocations.map( + (toolInvocation): ToolResultPart => { + if (!('result' in toolInvocation)) { + throw new MessageConversionError({ + originalMessage: message, + message: + 'ToolInvocation must have a result: ' + + JSON.stringify(toolInvocation), + }); + } + + const { toolCallId, toolName, result } = toolInvocation; + + const tool = tools[toolName]; + return tool?.experimental_toToolResultContent != null + ? { + type: 'tool-result', + toolCallId, + toolName, + result: tool.experimental_toToolResultContent(result), + experimental_content: + tool.experimental_toToolResultContent(result), + } + : { + type: 'tool-result', + toolCallId, + toolName, + result, + }; + }, + ), + }); + } + + // updates for next block + block = []; + blockHasToolInvocations = false; + currentStep++; + } + for (const part of message.parts) { switch (part.type) { + case 'reasoning': + // reasoning is not sent back to the LLM + break; case 'text': { - coreMessages.push({ role: 'assistant', content: part.text }); + if (blockHasToolInvocations) { + processBlock(); // text must come before tool invocations + } + block.push(part); + break; + } + case 'tool-invocation': { + if (part.toolInvocation.step === currentStep) { + block.push(part); + blockHasToolInvocations = true; + } else { + processBlock(); + } break; } } } + + processBlock(); + break; } From 58918cdebb4776e78982e36a790462298a8465d5 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 10:59:08 +0100 Subject: [PATCH 35/61] 3 --- .../convert-to-core-messages.test.ts.snap | 60 +++++++++++++++++++ .../prompt/convert-to-core-messages.test.ts | 58 ++++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap b/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap index 241153815fc4..bec8894bc5fa 100644 --- a/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap +++ b/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap @@ -74,6 +74,48 @@ exports[`convertToCoreMessages > assistant message > should handle assistant mes ] `; +exports[`convertToCoreMessages > assistant message > should handle assistant message with tool invocations that have multi-part responses (parts) 1`] = ` +[ + { + "content": [ + { + "text": "Let me calculate that for you.", + "type": "text", + }, + { + "args": {}, + "toolCallId": "call1", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "experimental_content": [ + { + "data": "imgbase64", + "type": "image", + }, + ], + "result": [ + { + "data": "imgbase64", + "type": "image", + }, + ], + "toolCallId": "call1", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, +] +`; + exports[`convertToCoreMessages > assistant message > should handle assistant message with tool invocations that have multi-part responses 1`] = ` [ { @@ -116,6 +158,24 @@ exports[`convertToCoreMessages > assistant message > should handle assistant mes ] `; +exports[`convertToCoreMessages > assistant message > should handle conversation with an assistant message that has empty tool invocations (parts) 1`] = ` +[ + { + "content": "text1", + "role": "user", + }, + { + "content": [ + { + "text": "text2", + "type": "text", + }, + ], + "role": "assistant", + }, +] +`; + exports[`convertToCoreMessages > assistant message > should handle conversation with an assistant message that has empty tool invocations 1`] = ` [ { diff --git a/packages/ai/core/prompt/convert-to-core-messages.test.ts b/packages/ai/core/prompt/convert-to-core-messages.test.ts index 038e480214cb..b8e2854fd005 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.test.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.test.ts @@ -286,6 +286,45 @@ describe('convertToCoreMessages', () => { expect(result).toMatchSnapshot(); }); + it('should handle assistant message with tool invocations that have multi-part responses (parts)', () => { + const tools = { + screenshot: tool({ + parameters: z.object({}), + execute: async () => 'imgbase64', + experimental_toToolResultContent: result => [ + { type: 'image', data: result }, + ], + }), + }; + + const result = convertToCoreMessages( + [ + { + role: 'assistant', + content: '', // empty content + toolInvocations: [], // empty invocations + parts: [ + { type: 'text', text: 'Let me calculate that for you.' }, + { + type: 'tool-invocation', + toolInvocation: { + state: 'result', + toolCallId: 'call1', + toolName: 'screenshot', + args: {}, + result: 'imgbase64', + step: 0, + }, + }, + ], + }, + ], + { tools }, // separate tools to ensure that types are inferred correctly + ); + + expect(result).toMatchSnapshot(); + }); + it('should handle conversation with an assistant message that has empty tool invocations', () => { const result = convertToCoreMessages([ { @@ -303,6 +342,25 @@ describe('convertToCoreMessages', () => { expect(result).toMatchSnapshot(); }); + it('should handle conversation with an assistant message that has empty tool invocations (parts)', () => { + const result = convertToCoreMessages([ + { + role: 'user', + content: 'text1', + toolInvocations: [], + parts: [{ type: 'text', text: 'text1' }], + }, + { + role: 'assistant', + content: '', // empty content + toolInvocations: [], + parts: [{ type: 'text', text: 'text2' }], + }, + ]); + + expect(result).toMatchSnapshot(); + }); + it('should handle conversation with multiple tool invocations that have step information', () => { const tools = { screenshot: tool({ From bc54f81301b2a1f2949d0aedbd49891b6c90d98f Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 11:04:29 +0100 Subject: [PATCH 36/61] 4 --- .../convert-to-core-messages.test.ts.snap | 95 +++++++++++++++++++ .../prompt/convert-to-core-messages.test.ts | 69 ++++++++++++++ .../core/prompt/convert-to-core-messages.ts | 7 +- 3 files changed, 167 insertions(+), 4 deletions(-) diff --git a/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap b/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap index bec8894bc5fa..caf09d561870 100644 --- a/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap +++ b/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap @@ -189,6 +189,101 @@ exports[`convertToCoreMessages > assistant message > should handle conversation ] `; +exports[`convertToCoreMessages > assistant message > should handle conversation with multiple tool invocations that have step information (parts) 1`] = ` +[ + { + "content": [ + { + "text": "response", + "type": "text", + }, + { + "args": { + "value": "value-1", + }, + "toolCallId": "call-1", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-1", + "toolCallId": "call-1", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, + { + "content": [ + { + "args": { + "value": "value-2", + }, + "toolCallId": "call-2", + "toolName": "screenshot", + "type": "tool-call", + }, + { + "args": { + "value": "value-3", + }, + "toolCallId": "call-3", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-2", + "toolCallId": "call-2", + "toolName": "screenshot", + "type": "tool-result", + }, + { + "result": "result-3", + "toolCallId": "call-3", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, + { + "content": [ + { + "args": { + "value": "value-4", + }, + "toolCallId": "call-4", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-4", + "toolCallId": "call-4", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, +] +`; + exports[`convertToCoreMessages > assistant message > should handle conversation with multiple tool invocations that have step information 1`] = ` [ { diff --git a/packages/ai/core/prompt/convert-to-core-messages.test.ts b/packages/ai/core/prompt/convert-to-core-messages.test.ts index b8e2854fd005..b6411ad07e46 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.test.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.test.ts @@ -416,6 +416,75 @@ describe('convertToCoreMessages', () => { expect(result).toMatchSnapshot(); }); + + it('should handle conversation with multiple tool invocations that have step information (parts)', () => { + const tools = { + screenshot: tool({ + parameters: z.object({ value: z.string() }), + execute: async () => 'imgbase64', + }), + }; + + const result = convertToCoreMessages( + [ + { + role: 'assistant', + content: '', // empty content + toolInvocations: [], // empty invocations + parts: [ + { type: 'text', text: 'response' }, + { + type: 'tool-invocation', + toolInvocation: { + state: 'result', + toolCallId: 'call-1', + toolName: 'screenshot', + args: { value: 'value-1' }, + result: 'result-1', + step: 0, + }, + }, + { + type: 'tool-invocation', + toolInvocation: { + state: 'result', + toolCallId: 'call-2', + toolName: 'screenshot', + args: { value: 'value-2' }, + result: 'result-2', + step: 1, + }, + }, + { + type: 'tool-invocation', + toolInvocation: { + state: 'result', + toolCallId: 'call-3', + toolName: 'screenshot', + args: { value: 'value-3' }, + result: 'result-3', + step: 1, + }, + }, + { + type: 'tool-invocation', + toolInvocation: { + state: 'result', + toolCallId: 'call-4', + toolName: 'screenshot', + args: { value: 'value-4' }, + result: 'result-4', + step: 2, + }, + }, + ], + }, + ], + { tools }, // separate tools to ensure that types are inferred correctly + ); + + expect(result).toMatchSnapshot(); + }); }); describe('multiple messages', () => { diff --git a/packages/ai/core/prompt/convert-to-core-messages.ts b/packages/ai/core/prompt/convert-to-core-messages.ts index d727212af257..1c0f5c074296 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.ts @@ -146,12 +146,11 @@ export function convertToCoreMessages( break; } case 'tool-invocation': { - if (part.toolInvocation.step === currentStep) { - block.push(part); - blockHasToolInvocations = true; - } else { + if ((part.toolInvocation.step ?? 0) !== currentStep) { processBlock(); } + block.push(part); + blockHasToolInvocations = true; break; } } From 1bc6710f497d9fd7da2ee193bc62fa1a30da2e1d Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 11:14:11 +0100 Subject: [PATCH 37/61] remove --- .../core/prompt/convert-to-core-messages.ts | 16 +++------------ .../ai/core/prompt/detect-prompt-type.test.ts | 8 ++++---- .../core/prompt/message-conversion-error.ts | 6 +++--- packages/ai/core/prompt/prompt.ts | 4 ++-- packages/ai/core/prompt/standardize-prompt.ts | 4 ++-- packages/ai/core/prompt/ui-message.ts | 20 ------------------- 6 files changed, 14 insertions(+), 44 deletions(-) delete mode 100644 packages/ai/core/prompt/ui-message.ts diff --git a/packages/ai/core/prompt/convert-to-core-messages.ts b/packages/ai/core/prompt/convert-to-core-messages.ts index 1c0f5c074296..190b2146ea60 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.ts @@ -1,25 +1,15 @@ -import { - ReasoningUIPart, - TextUIPart, - ToolInvocationUIPart, -} from '@ai-sdk/ui-utils'; +import { Message, TextUIPart, ToolInvocationUIPart } from '@ai-sdk/ui-utils'; import { ToolSet } from '../generate-text/tool-set'; -import { - AssistantContent, - CoreMessage, - ToolCallPart, - ToolResultPart, -} from '../prompt'; +import { CoreMessage, ToolCallPart, ToolResultPart } from '../prompt'; import { attachmentsToParts } from './attachments-to-parts'; import { MessageConversionError } from './message-conversion-error'; -import { InternalUIMessage } from './ui-message'; /** Converts an array of messages from useChat into an array of CoreMessages that can be used with the AI core functions (e.g. `streamText`). */ export function convertToCoreMessages( - messages: Array, + messages: Array>, options?: { tools?: TOOLS }, ) { const tools = options?.tools ?? ({} as TOOLS); diff --git a/packages/ai/core/prompt/detect-prompt-type.test.ts b/packages/ai/core/prompt/detect-prompt-type.test.ts index 2473597bdce2..6f6e60752e36 100644 --- a/packages/ai/core/prompt/detect-prompt-type.test.ts +++ b/packages/ai/core/prompt/detect-prompt-type.test.ts @@ -1,5 +1,5 @@ +import { Message } from '@ai-sdk/ui-utils'; import { detectPromptType } from './detect-prompt-type'; -import type { InternalUIMessage } from './ui-message'; import type { CoreMessage } from './message'; it('should return "other" for invalid inputs', () => { @@ -13,7 +13,7 @@ it('should return "messages" for empty arrays', () => { }); it('should detect UI messages with data role', () => { - const messages: InternalUIMessage[] = [ + const messages: Omit[] = [ { role: 'data', content: 'some data', @@ -23,7 +23,7 @@ it('should detect UI messages with data role', () => { }); it('should detect UI messages with toolInvocations', () => { - const messages: InternalUIMessage[] = [ + const messages: Omit[] = [ { role: 'assistant', content: 'Hello', @@ -42,7 +42,7 @@ it('should detect UI messages with toolInvocations', () => { }); it('should detect UI messages with experimental_attachments', () => { - const messages: InternalUIMessage[] = [ + const messages: Omit[] = [ { role: 'user', content: 'Check this file', diff --git a/packages/ai/core/prompt/message-conversion-error.ts b/packages/ai/core/prompt/message-conversion-error.ts index d00d26db0118..6472b828b35b 100644 --- a/packages/ai/core/prompt/message-conversion-error.ts +++ b/packages/ai/core/prompt/message-conversion-error.ts @@ -1,5 +1,5 @@ import { AISDKError } from '@ai-sdk/provider'; -import { InternalUIMessage } from './ui-message'; +import { Message } from '@ai-sdk/ui-utils'; const name = 'AI_MessageConversionError'; const marker = `vercel.ai.error.${name}`; @@ -8,13 +8,13 @@ const symbol = Symbol.for(marker); export class MessageConversionError extends AISDKError { private readonly [symbol] = true; // used in isInstance - readonly originalMessage: InternalUIMessage; + readonly originalMessage: Omit; constructor({ originalMessage, message, }: { - originalMessage: InternalUIMessage; + originalMessage: Omit; message: string; }) { super({ name, message }); diff --git a/packages/ai/core/prompt/prompt.ts b/packages/ai/core/prompt/prompt.ts index d4a3a4effc97..5688b1609f23 100644 --- a/packages/ai/core/prompt/prompt.ts +++ b/packages/ai/core/prompt/prompt.ts @@ -1,5 +1,5 @@ +import { Message } from '@ai-sdk/ui-utils'; import { CoreMessage } from './message'; -import { InternalUIMessage } from './ui-message'; /** Prompt part of the AI function options. @@ -19,5 +19,5 @@ A simple text prompt. You can either use `prompt` or `messages` but not both. /** A list of messages. You can either use `prompt` or `messages` but not both. */ - messages?: Array | Array; + messages?: Array | Array>; }; diff --git a/packages/ai/core/prompt/standardize-prompt.ts b/packages/ai/core/prompt/standardize-prompt.ts index aaa519d7b311..bac55d20bcf6 100644 --- a/packages/ai/core/prompt/standardize-prompt.ts +++ b/packages/ai/core/prompt/standardize-prompt.ts @@ -1,12 +1,12 @@ import { InvalidPromptError } from '@ai-sdk/provider'; import { safeValidateTypes } from '@ai-sdk/provider-utils'; +import { Message } from '@ai-sdk/ui-utils'; import { z } from 'zod'; import { ToolSet } from '../generate-text/tool-set'; import { convertToCoreMessages } from './convert-to-core-messages'; import { detectPromptType } from './detect-prompt-type'; import { CoreMessage, coreMessageSchema } from './message'; import { Prompt } from './prompt'; -import { InternalUIMessage } from './ui-message'; export type StandardizedPrompt = { /** @@ -90,7 +90,7 @@ export function standardizePrompt({ const messages: CoreMessage[] = promptType === 'ui-messages' - ? convertToCoreMessages(prompt.messages as InternalUIMessage[], { + ? convertToCoreMessages(prompt.messages as Omit[], { tools, }) : (prompt.messages as CoreMessage[]); diff --git a/packages/ai/core/prompt/ui-message.ts b/packages/ai/core/prompt/ui-message.ts deleted file mode 100644 index f98ef0c5f7ee..000000000000 --- a/packages/ai/core/prompt/ui-message.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - Attachment, - ReasoningUIPart, - TextUIPart, - ToolInvocation, - ToolInvocationUIPart, -} from '@ai-sdk/ui-utils'; - -/** - * @internal - */ -export type InternalUIMessage = { - role: 'system' | 'user' | 'assistant' | 'data'; - - content: string; - toolInvocations?: ToolInvocation[]; - experimental_attachments?: Attachment[]; - - parts?: Array; -}; From 2f8e4107d8f257ee1047ee14320e46e4c66c90fc Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 11:19:00 +0100 Subject: [PATCH 38/61] 5 --- .../convert-to-core-messages.test.ts.snap | 104 ++++++++++++++++++ .../prompt/convert-to-core-messages.test.ts | 104 ++++++++++++++++++ 2 files changed, 208 insertions(+) diff --git a/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap b/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap index caf09d561870..dd6e3eed5b09 100644 --- a/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap +++ b/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap @@ -392,6 +392,110 @@ exports[`convertToCoreMessages > multiple messages > should convert fully typed ] `; +exports[`convertToCoreMessages > multiple messages > should handle conversation with multiple tool invocations and user message at the end (parts) 1`] = ` +[ + { + "content": [ + { + "args": { + "value": "value-1", + }, + "toolCallId": "call-1", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-1", + "toolCallId": "call-1", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, + { + "content": [ + { + "args": { + "value": "value-2", + }, + "toolCallId": "call-2", + "toolName": "screenshot", + "type": "tool-call", + }, + { + "args": { + "value": "value-3", + }, + "toolCallId": "call-3", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-2", + "toolCallId": "call-2", + "toolName": "screenshot", + "type": "tool-result", + }, + { + "result": "result-3", + "toolCallId": "call-3", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, + { + "content": [ + { + "args": { + "value": "value-4", + }, + "toolCallId": "call-4", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-4", + "toolCallId": "call-4", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, + { + "content": [ + { + "text": "response", + "type": "text", + }, + ], + "role": "assistant", + }, + { + "content": "Thanks!", + "role": "user", + }, +] +`; + exports[`convertToCoreMessages > multiple messages > should handle conversation with multiple tool invocations and user message at the end 1`] = ` [ { diff --git a/packages/ai/core/prompt/convert-to-core-messages.test.ts b/packages/ai/core/prompt/convert-to-core-messages.test.ts index b6411ad07e46..dd359cf86192 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.test.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.test.ts @@ -502,6 +502,35 @@ describe('convertToCoreMessages', () => { ]); }); + it('should handle a conversation with multiple messages (parts)', () => { + const result = convertToCoreMessages([ + { + role: 'user', + content: "What's the weather like?", + parts: [{ type: 'text', text: "What's the weather like?" }], + }, + { + role: 'assistant', + content: '', + parts: [{ type: 'text', text: "I'll check that for you." }], + }, + { + role: 'user', + content: 'Thanks!', + parts: [{ type: 'text', text: 'Thanks!' }], + }, + ]); + + expect(result).toEqual([ + { role: 'user', content: "What's the weather like?" }, + { + role: 'assistant', + content: [{ type: 'text', text: "I'll check that for you." }], + }, + { role: 'user', content: 'Thanks!' }, + ]); + }); + it('should convert fully typed Message[]', () => { const messages: Message[] = [ { @@ -580,6 +609,81 @@ describe('convertToCoreMessages', () => { expect(result).toMatchSnapshot(); }); + + it('should handle conversation with multiple tool invocations and user message at the end (parts)', () => { + const tools = { + screenshot: tool({ + parameters: z.object({ value: z.string() }), + execute: async () => 'imgbase64', + }), + }; + + const result = convertToCoreMessages( + [ + { + role: 'assistant', + content: '', + toolInvocations: [], + parts: [ + { + type: 'tool-invocation', + toolInvocation: { + state: 'result', + toolCallId: 'call-1', + toolName: 'screenshot', + args: { value: 'value-1' }, + result: 'result-1', + step: 0, + }, + }, + { + type: 'tool-invocation', + toolInvocation: { + state: 'result', + toolCallId: 'call-2', + toolName: 'screenshot', + args: { value: 'value-2' }, + result: 'result-2', + step: 1, + }, + }, + + { + type: 'tool-invocation', + toolInvocation: { + state: 'result', + toolCallId: 'call-3', + toolName: 'screenshot', + args: { value: 'value-3' }, + result: 'result-3', + step: 1, + }, + }, + { + type: 'tool-invocation', + toolInvocation: { + state: 'result', + toolCallId: 'call-4', + toolName: 'screenshot', + args: { value: 'value-4' }, + result: 'result-4', + step: 2, + }, + }, + { type: 'text', text: 'response' }, + ], + }, + { + role: 'user', + content: 'Thanks!', + parts: [{ type: 'text', text: 'Thanks!' }], + }, + ], + { tools }, // separate tools to ensure that types are inferred correctly + ); + + expect(result).toMatchSnapshot(); + }); }); describe('error handling', () => { From 43da544f7f47f889f94119d298dac7cadcf050c6 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 11:20:29 +0100 Subject: [PATCH 39/61] anthropic --- .../convert-to-core-messages.test.ts.snap | 108 ++++++++++++++++++ .../prompt/convert-to-core-messages.test.ts | 71 ++++++++++++ 2 files changed, 179 insertions(+) diff --git a/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap b/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap index dd6e3eed5b09..cb1d97434ed1 100644 --- a/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap +++ b/packages/ai/core/prompt/__snapshots__/convert-to-core-messages.test.ts.snap @@ -189,6 +189,114 @@ exports[`convertToCoreMessages > assistant message > should handle conversation ] `; +exports[`convertToCoreMessages > assistant message > should handle conversation with mix of tool invocations and text (parts) 1`] = ` +[ + { + "content": [ + { + "text": "i am gonna use tool1", + "type": "text", + }, + { + "args": { + "value": "value-1", + }, + "toolCallId": "call-1", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-1", + "toolCallId": "call-1", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, + { + "content": [ + { + "text": "i am gonna use tool2 and tool3", + "type": "text", + }, + { + "args": { + "value": "value-2", + }, + "toolCallId": "call-2", + "toolName": "screenshot", + "type": "tool-call", + }, + { + "args": { + "value": "value-3", + }, + "toolCallId": "call-3", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-2", + "toolCallId": "call-2", + "toolName": "screenshot", + "type": "tool-result", + }, + { + "result": "result-3", + "toolCallId": "call-3", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, + { + "content": [ + { + "args": { + "value": "value-4", + }, + "toolCallId": "call-4", + "toolName": "screenshot", + "type": "tool-call", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "result": "result-4", + "toolCallId": "call-4", + "toolName": "screenshot", + "type": "tool-result", + }, + ], + "role": "tool", + }, + { + "content": [ + { + "text": "final response", + "type": "text", + }, + ], + "role": "assistant", + }, +] +`; + exports[`convertToCoreMessages > assistant message > should handle conversation with multiple tool invocations that have step information (parts) 1`] = ` [ { diff --git a/packages/ai/core/prompt/convert-to-core-messages.test.ts b/packages/ai/core/prompt/convert-to-core-messages.test.ts index dd359cf86192..b31ea116c37b 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.test.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.test.ts @@ -485,6 +485,77 @@ describe('convertToCoreMessages', () => { expect(result).toMatchSnapshot(); }); + + it('should handle conversation with mix of tool invocations and text (parts)', () => { + const tools = { + screenshot: tool({ + parameters: z.object({ value: z.string() }), + execute: async () => 'imgbase64', + }), + }; + + const result = convertToCoreMessages( + [ + { + role: 'assistant', + content: '', // empty content + toolInvocations: [], // empty invocations + parts: [ + { type: 'text', text: 'i am gonna use tool1' }, + { + type: 'tool-invocation', + toolInvocation: { + state: 'result', + toolCallId: 'call-1', + toolName: 'screenshot', + args: { value: 'value-1' }, + result: 'result-1', + step: 0, + }, + }, + { type: 'text', text: 'i am gonna use tool2 and tool3' }, + { + type: 'tool-invocation', + toolInvocation: { + state: 'result', + toolCallId: 'call-2', + toolName: 'screenshot', + args: { value: 'value-2' }, + result: 'result-2', + step: 1, + }, + }, + { + type: 'tool-invocation', + toolInvocation: { + state: 'result', + toolCallId: 'call-3', + toolName: 'screenshot', + args: { value: 'value-3' }, + result: 'result-3', + step: 1, + }, + }, + { + type: 'tool-invocation', + toolInvocation: { + state: 'result', + toolCallId: 'call-4', + toolName: 'screenshot', + args: { value: 'value-4' }, + result: 'result-4', + step: 2, + }, + }, + { type: 'text', text: 'final response' }, + ], + }, + ], + { tools }, // separate tools to ensure that types are inferred correctly + ); + + expect(result).toMatchSnapshot(); + }); }); describe('multiple messages', () => { From 707087374e7cc12a3c99c2da5e2bcaf1904a3d13 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 4 Feb 2025 12:25:41 +0100 Subject: [PATCH 40/61] vue --- examples/next-openai/app/page.tsx | 2 +- .../next-openai/app/use-chat-tools/page.tsx | 30 ++--- .../pages/use-chat-tools/index.vue | 124 +++++++++++------- .../nuxt-openai/server/api/use-chat-tools.ts | 4 +- packages/react/src/use-chat.ts | 2 +- packages/vue/src/use-chat.ts | 21 +-- 6 files changed, 95 insertions(+), 88 deletions(-) diff --git a/examples/next-openai/app/page.tsx b/examples/next-openai/app/page.tsx index d3cddcd0b99d..453785607fa1 100644 --- a/examples/next-openai/app/page.tsx +++ b/examples/next-openai/app/page.tsx @@ -24,7 +24,7 @@ export default function Chat() { {messages.map(m => (
{m.role === 'user' ? 'User: ' : 'AI: '} - {m.content} + {m.parts.map(p => (p.type === 'text' ? p.text : undefined))}
))} diff --git a/examples/next-openai/app/use-chat-tools/page.tsx b/examples/next-openai/app/use-chat-tools/page.tsx index 6dd4e7b2f725..b7a1e7dd81e6 100644 --- a/examples/next-openai/app/use-chat-tools/page.tsx +++ b/examples/next-openai/app/use-chat-tools/page.tsx @@ -32,18 +32,15 @@ export default function Chat() { case 'text': return part.text; case 'tool-invocation': { - const invocation = part.toolInvocation; - const callId = invocation.toolCallId; + const callId = part.toolInvocation.toolCallId; - switch (invocation.toolName) { + switch (part.toolInvocation.toolName) { case 'askForConfirmation': { - switch (invocation.state) { - case 'partial-call': - return undefined; + switch (part.toolInvocation.state) { case 'call': return (
- {invocation.args.message} + {part.toolInvocation.args.message}
); } } case 'getLocation': { - switch (invocation.state) { - case 'partial-call': - return undefined; + switch (part.toolInvocation.state) { case 'call': return (
@@ -92,33 +88,33 @@ export default function Chat() { case 'result': return (
- Location: {invocation.result} + Location: {part.toolInvocation.result}
); } } case 'getWeatherInformation': { - switch (invocation.state) { + switch (part.toolInvocation.state) { // example of pre-rendering streaming tool calls: case 'partial-call': return ( 
-                            {JSON.stringify(invocation, null, 2)}
+                            {JSON.stringify(part.toolInvocation, null, 2)}
); case 'call': return (
Getting weather information for{' '} - {invocation.args.city}... + {part.toolInvocation.args.city}...
); case 'result': return (
- Weather in {invocation.args.city}:{' '} - {invocation.result} + Weather in {part.toolInvocation.args.city}:{' '} + {part.toolInvocation.result}
); } diff --git a/examples/nuxt-openai/pages/use-chat-tools/index.vue b/examples/nuxt-openai/pages/use-chat-tools/index.vue index 8e0ee045ae65..6d272a2070fc 100644 --- a/examples/nuxt-openai/pages/use-chat-tools/index.vue +++ b/examples/nuxt-openai/pages/use-chat-tools/index.vue @@ -13,65 +13,87 @@ const { input, handleSubmit, messages, addToolResult } = useChat({ } }, }); + +const messageList = computed(() => messages.value); // computer property for type inference