From 1e979881c60173b0f3a3a9bec0c1f63b4a5283d0 Mon Sep 17 00:00:00 2001 From: ferenci84 Date: Mon, 20 Jan 2025 18:09:59 +0100 Subject: [PATCH 1/2] Add replaceSystemMessage LLM option and update relevant functionality and documentation --- core/index.d.ts | 46 +++++++++++++++++++++-------------------- core/llm/countTokens.ts | 25 ++++++++++++++-------- core/llm/index.ts | 11 +++++----- core/llm/llms/index.ts | 1 + docs/docs/reference.md | 5 +++-- 5 files changed, 50 insertions(+), 38 deletions(-) diff --git a/core/index.d.ts b/core/index.d.ts index 098628c0fd..42f1eea3af 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -43,13 +43,13 @@ export interface IndexingProgressUpdate { desc: string; shouldClearIndexes?: boolean; status: - | "loading" - | "indexing" - | "done" - | "failed" - | "paused" - | "disabled" - | "cancelled"; + | "loading" + | "indexing" + | "done" + | "failed" + | "paused" + | "disabled" + | "cancelled"; debugInfo?: string; } @@ -450,6 +450,7 @@ export interface LLMOptions { title?: string; uniqueId?: string; systemMessage?: string; + replaceSystemMessage?: boolean; contextLength?: number; maxStopWords?: number; completionOptions?: CompletionOptions; @@ -650,10 +651,10 @@ export interface IDE { getCurrentFile(): Promise< | undefined | { - isUntitled: boolean; - path: string; - contents: string; - } + isUntitled: boolean; + path: string; + contents: string; + } >; getPinnedFiles(): Promise; @@ -832,11 +833,11 @@ export interface CustomCommand { export interface Prediction { type: "content"; content: - | string - | { - type: "text"; - text: string; - }[]; + | string + | { + type: "text"; + text: string; + }[]; } export interface ToolExtras { @@ -903,6 +904,7 @@ export interface ModelDescription { template?: TemplateType; completionOptions?: BaseCompletionOptions; systemMessage?: string; + replaceSystemMessage?: boolean; requestOptions?: RequestOptions; promptTemplates?: { [key: string]: string }; capabilities?: ModelCapability; @@ -1157,9 +1159,9 @@ export interface Config { embeddingsProvider?: EmbeddingsProviderDescription | ILLM; /** The model that Continue will use for tab autocompletions. */ tabAutocompleteModel?: - | CustomLLM - | ModelDescription - | (CustomLLM | ModelDescription)[]; + | CustomLLM + | ModelDescription + | (CustomLLM | ModelDescription)[]; /** Options for tab autocomplete */ tabAutocompleteOptions?: Partial; /** UI styles customization */ @@ -1249,9 +1251,9 @@ export type PackageDetailsSuccess = PackageDetails & { export type PackageDocsResult = { packageInfo: ParsedPackageInfo; } & ( - | { error: string; details?: never } - | { details: PackageDetailsSuccess; error?: never } -); + | { error: string; details?: never } + | { details: PackageDetailsSuccess; error?: never } + ); export interface TerminalOptions { reuseTerminal?: boolean; diff --git a/core/llm/countTokens.ts b/core/llm/countTokens.ts index 5f88c13ebb..adda1cbab9 100644 --- a/core/llm/countTokens.ts +++ b/core/llm/countTokens.ts @@ -1,6 +1,6 @@ import { Tiktoken, encodingForModel as _encodingForModel } from "js-tiktoken"; -import { ChatMessage, MessageContent, MessagePart } from "../index.js"; +import { ChatMessage, LLMOptions, MessageContent, MessagePart } from "../index.js"; import { renderChatMessage } from "../util/messageContent.js"; import { @@ -27,9 +27,9 @@ class LlamaEncoding implements Encoding { } class NonWorkerAsyncEncoder implements AsyncEncoder { - constructor(private readonly encoding: Encoding) {} + constructor(private readonly encoding: Encoding) { } - async close(): Promise {} + async close(): Promise { } async encode(text: string): Promise { return this.encoding.encode(text); @@ -380,15 +380,18 @@ function compileChatMessages( prompt: string | undefined = undefined, functions: any[] | undefined = undefined, systemMessage: string | undefined = undefined, + llmOptions?: LLMOptions, ): ChatMessage[] { let msgsCopy = msgs ? msgs - .map((msg) => ({ ...msg })) - .filter((msg) => !chatMessageIsEmpty(msg) && msg.role !== "system") + .map((msg) => ({ ...msg })) + .filter((msg) => !chatMessageIsEmpty(msg) && msg.role !== "system") : []; msgsCopy = addSpaceToAnyEmptyMessages(msgsCopy); + const replaceSystemMessage = llmOptions?.replaceSystemMessage ?? false; + if (prompt) { const promptMsg: ChatMessage = { role: "user", @@ -406,11 +409,15 @@ function compileChatMessages( content = renderChatMessage(msgs?.[0]); } if (systemMessage && systemMessage.trim() !== "") { - const shouldAddNewLines = content !== ""; - if (shouldAddNewLines) { - content += "\n\n"; + if (replaceSystemMessage) { + content = systemMessage; + } else { + const shouldAddNewLines = content !== ""; + if (shouldAddNewLines) { + content += "\n\n"; + } + content += systemMessage; } - content += systemMessage; } const systemChatMsg: ChatMessage = { role: "system", diff --git a/core/llm/index.ts b/core/llm/index.ts index 5ecfa33ffa..cac4370d11 100644 --- a/core/llm/index.ts +++ b/core/llm/index.ts @@ -173,11 +173,11 @@ export abstract class BaseLLM implements ILLM { options.completionOptions?.maxTokens ?? (llmInfo?.maxCompletionTokens ? Math.min( - llmInfo.maxCompletionTokens, - // Even if the model has a large maxTokens, we don't want to use that every time, - // because it takes away from the context length - this.contextLength / 4, - ) + llmInfo.maxCompletionTokens, + // Even if the model has a large maxTokens, we don't want to use that every time, + // because it takes away from the context length + this.contextLength / 4, + ) : DEFAULT_MAX_TOKENS), }; this.requestOptions = options.requestOptions; @@ -261,6 +261,7 @@ export abstract class BaseLLM implements ILLM { undefined, functions, this.systemMessage, + this._llmOptions, ); } diff --git a/core/llm/llms/index.ts b/core/llm/llms/index.ts index 20bc8b710f..cf9832ffda 100644 --- a/core/llm/llms/index.ts +++ b/core/llm/llms/index.ts @@ -137,6 +137,7 @@ export async function llmFromDescription( cls.defaultOptions?.completionOptions?.maxTokens, }, systemMessage, + replaceSystemMessage: desc.replaceSystemMessage ?? false, writeLog, uniqueId, }; diff --git a/docs/docs/reference.md b/docs/docs/reference.md index d3c491f4c1..beb8d920e1 100644 --- a/docs/docs/reference.md +++ b/docs/docs/reference.md @@ -25,7 +25,8 @@ Each model has specific configuration options tailored to its provider and funct - `template`: Chat template to format messages. Auto-detected for most models but can be overridden. See intelliJ suggestions. - `promptTemplates`: A mapping of prompt template names (e.g., `edit`) to template strings. [Customization Guide](https://docs.continue.dev/model-setup/configuration#customizing-the-edit-prompt). - `completionOptions`: Model-specific completion options, same format as top-level [`completionOptions`](#completionoptions), which they override. -- `systemMessage`: A system message that will precede responses from the LLM. +- `systemMessage`: A system message that will precede responses from the LLM. (Note: this property has higher precendence than the root `systemMessage` property) +- `replaceSystemMessage`: If `true`, replaces the system message with the one specified in the model config's `systemMessage` property or with the root `systemMessage` property. If `false`, appends the `systemMessage` to the default system message, if any exists for the model. (default: `false`) - `requestOptions`: Model-specific HTTP request options, same format as top-level [`requestOptions`](#requestoptions), which they override. - `apiType`: Specifies the type of API (`openai` or `azure`). - `apiVersion`: Azure API version (e.g., `2023-07-01-preview`). @@ -381,7 +382,7 @@ An optional token that identifies the user, primarily for authenticated services ### `systemMessage` -Defines a system message that appears before every response from the language model, providing guidance or context. +Defines a system message that appears before every response from the language model, providing guidance or context. Note: `systemMessage` in the model config takes precedence over this setting. ### `disableIndexing` From b058b609624c5dd6b29765838899daa5dd584947 Mon Sep 17 00:00:00 2001 From: ferenci84 Date: Mon, 20 Jan 2025 18:38:12 +0100 Subject: [PATCH 2/2] Add replaceSystemMessage configuration option to config_schema.json --- extensions/vscode/config_schema.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/extensions/vscode/config_schema.json b/extensions/vscode/config_schema.json index 1273a7c406..86c4bf8c6a 100644 --- a/extensions/vscode/config_schema.json +++ b/extensions/vscode/config_schema.json @@ -400,6 +400,11 @@ "description": "A system message that will always be followed by the LLM", "type": "string" }, + "replaceSystemMessage": { + "title": "Replace System Message", + "description": "If true, the system message will replace the default system message instead of being appended to it", + "type": "boolean" + }, "requestOptions": { "title": "Request Options", "description": "Options for the HTTP request to the LLM.",