diff --git a/core/config/types.ts b/core/config/types.ts index 8b7b07e3d5..df29d6071c 100644 --- a/core/config/types.ts +++ b/core/config/types.ts @@ -719,7 +719,7 @@ declare global { getPinnedFiles(): Promise; - getSearchResults(query: string): Promise; + getSearchResults(query: string, maxResults?: number): Promise; subprocess(command: string, cwd?: string): Promise<[string, string]>; diff --git a/core/context/providers/SearchContextProvider.ts b/core/context/providers/SearchContextProvider.ts index ec75994c32..b6ec401335 100644 --- a/core/context/providers/SearchContextProvider.ts +++ b/core/context/providers/SearchContextProvider.ts @@ -6,6 +6,7 @@ import { import { formatGrepSearchResults } from "../../util/grepSearch.js"; import { BaseContextProvider } from "../index.js"; +const DEFAULT_MAX_SEARCH_CONTEXT_RESULTS = 200; class SearchContextProvider extends BaseContextProvider { static description: ContextProviderDescription = { title: "search", @@ -19,8 +20,12 @@ class SearchContextProvider extends BaseContextProvider { query: string, extras: ContextProviderExtras, ): Promise { - const results = await extras.ide.getSearchResults(query); - const formatted = formatGrepSearchResults(results); + const results = await extras.ide.getSearchResults( + query, + this.options?.maxResults ?? DEFAULT_MAX_SEARCH_CONTEXT_RESULTS, + ); + // Note, search context provider will not truncate result chars, but will limit number of results + const { formatted } = formatGrepSearchResults(results); return [ { description: "Search results", diff --git a/core/index.d.ts b/core/index.d.ts index ce78a8d828..a881e7b425 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -781,7 +781,7 @@ export interface IDE { getPinnedFiles(): Promise; - getSearchResults(query: string): Promise; + getSearchResults(query: string, maxResults?: number): Promise; getFileResults(pattern: string): Promise; diff --git a/core/protocol/ide.ts b/core/protocol/ide.ts index fda1b84fe1..2363d1ca0b 100644 --- a/core/protocol/ide.ts +++ b/core/protocol/ide.ts @@ -29,7 +29,7 @@ export type ToIdeFromWebviewOrCoreProtocol = { openFile: [{ path: string }, void]; openUrl: [string, void]; runCommand: [{ command: string; options?: TerminalOptions }, void]; - getSearchResults: [{ query: string }, string]; + getSearchResults: [{ query: string; maxResults?: number }, string]; getFileResults: [{ pattern: string }, string[]]; subprocess: [{ command: string; cwd?: string }, [string, string]]; saveFile: [{ filepath: string }, void]; diff --git a/core/protocol/messenger/messageIde.ts b/core/protocol/messenger/messageIde.ts index 2bc52bca85..87e1ec6429 100644 --- a/core/protocol/messenger/messageIde.ts +++ b/core/protocol/messenger/messageIde.ts @@ -185,8 +185,8 @@ export class MessageIde implements IDE { return this.request("getPinnedFiles", undefined); } - getSearchResults(query: string): Promise { - return this.request("getSearchResults", { query }); + getSearchResults(query: string, maxResults?: number): Promise { + return this.request("getSearchResults", { query, maxResults }); } getFileResults(pattern: string): Promise { diff --git a/core/protocol/messenger/reverseMessageIde.ts b/core/protocol/messenger/reverseMessageIde.ts index f409156640..0771051aa9 100644 --- a/core/protocol/messenger/reverseMessageIde.ts +++ b/core/protocol/messenger/reverseMessageIde.ts @@ -159,7 +159,7 @@ export class ReverseMessageIde { }); this.on("getSearchResults", (data) => { - return this.ide.getSearchResults(data.query); + return this.ide.getSearchResults(data.query, data.maxResults); }); this.on("getFileResults", (data) => { diff --git a/core/tools/definitions/grepSearch.ts b/core/tools/definitions/grepSearch.ts index ca8a73a306..60a4e2710e 100644 --- a/core/tools/definitions/grepSearch.ts +++ b/core/tools/definitions/grepSearch.ts @@ -12,7 +12,8 @@ export const grepSearchTool: Tool = { group: BUILT_IN_GROUP_NAME, function: { name: BuiltInToolNames.GrepSearch, - description: "Perform a search over the repository using ripgrep.", + description: + "Perform a search over the repository using ripgrep. Output may be truncated, so use targeted queries", parameters: { type: "object", required: ["query"], diff --git a/core/tools/implementations/grepSearch.ts b/core/tools/implementations/grepSearch.ts index a506be9cd8..149b68a754 100644 --- a/core/tools/implementations/grepSearch.ts +++ b/core/tools/implementations/grepSearch.ts @@ -1,13 +1,44 @@ import { ToolImpl } from "."; +import { ContextItem } from "../.."; import { formatGrepSearchResults } from "../../util/grepSearch"; +const DEFAULT_GREP_SEARCH_RESULTS_LIMIT = 100; +const DEFAULT_GREP_SEARCH_CHAR_LIMIT = 5000; // ~1000 tokens, will keep truncation simply for now + export const grepSearchImpl: ToolImpl = async (args, extras) => { - const results = await extras.ide.getSearchResults(args.query); - return [ + const results = await extras.ide.getSearchResults( + args.query, + DEFAULT_GREP_SEARCH_RESULTS_LIMIT, + ); + const { formatted, numResults, truncated } = formatGrepSearchResults( + results, + DEFAULT_GREP_SEARCH_CHAR_LIMIT, + ); + const truncationReasons: string[] = []; + if (numResults === DEFAULT_GREP_SEARCH_RESULTS_LIMIT) { + truncationReasons.push( + `the number of results exceeded ${DEFAULT_GREP_SEARCH_RESULTS_LIMIT}`, + ); + } + if (truncated) { + truncationReasons.push( + `the number of characters exceeded ${DEFAULT_GREP_SEARCH_CHAR_LIMIT}`, + ); + } + + const contextItems: ContextItem[] = [ { name: "Search results", description: "Results from grep search", - content: formatGrepSearchResults(results), + content: formatted, }, ]; + if (truncationReasons.length > 0) { + contextItems.push({ + name: "Search truncation warning", + description: "Informs the model that search results were truncated", + content: `The above search results were truncated because ${truncationReasons.join(" and ")}. If the results are not satisfactory, try refining your search query.`, + }); + } + return contextItems; }; diff --git a/core/util/filesystem.ts b/core/util/filesystem.ts index 0bdaffbfd4..35eaa6b070 100644 --- a/core/util/filesystem.ts +++ b/core/util/filesystem.ts @@ -234,7 +234,7 @@ class FileSystemIde implements IDE { return Promise.resolve([]); } - async getSearchResults(query: string): Promise { + async getSearchResults(query: string, maxResults?: number): Promise { return ""; } diff --git a/core/util/grepSearch.ts b/core/util/grepSearch.ts index a30982e771..be5b7ee082 100644 --- a/core/util/grepSearch.ts +++ b/core/util/grepSearch.ts @@ -1,4 +1,19 @@ -export function formatGrepSearchResults(results: string): string { +/* + Formats the output of a grep search to reduce unnecessary indentation, lines, etc + Assumes a command with these params + ripgrep -i --ignore-file .continueignore --ignore-file .gitignore -C 2 --heading -m 100 -e . + + Also can truncate the output to a specified number of characters +*/ +export function formatGrepSearchResults( + results: string, + maxChars?: number, +): { + formatted: string; + numResults: number; + truncated: boolean; +} { + let numResults = 0; const keepLines: string[] = []; function countLeadingSpaces(line: string) { @@ -45,6 +60,7 @@ export function formatGrepSearchResults(results: string): string { if (line.startsWith("./") || line === "--") { processResult(resultLines); // process previous result resultLines = [line]; + numResults++; continue; } @@ -57,5 +73,18 @@ export function formatGrepSearchResults(results: string): string { } processResult(resultLines); - return keepLines.join("\n"); + const formatted = keepLines.join("\n"); + if (maxChars && formatted.length > maxChars) { + return { + formatted: formatted.substring(0, maxChars), + numResults, + truncated: true, + }; + } else { + return { + formatted, + numResults, + truncated: false, + }; + } } diff --git a/core/util/grepSearch.vitest.ts b/core/util/grepSearch.vitest.ts new file mode 100644 index 0000000000..df1a71efd8 --- /dev/null +++ b/core/util/grepSearch.vitest.ts @@ -0,0 +1,217 @@ +import { expect, test } from "vitest"; +import { formatGrepSearchResults } from "./grepSearch"; + +// Sample grep output mimicking what would come from ripgrep with the params: +// ripgrep -i --ignore-file .continueignore --ignore-file .gitignore -C 2 --heading -m 100 -e . +const sampleGrepOutput = `./program.cs + Console.WriteLine("Hello World!"); + Calculator calc = new Calculator(); + calc.Add(5).Subtract(3); + Console.WriteLine("Result: " + calc.GetResult()); + } +-- + } + + public Calculator Subtract(double number) + { + result -= number; + +./test.kt + } + + fun subtract(number: Double): Test { + result -= number + return this + +./test.php + } + + public function subtract($number) { + $this->result -= $number; + return $this; +-- + +$calc = new Calculator(); +$calc->add(10)->subtract(5); +echo "Result: " . $calc->getResult() . "\n"; + + +./Calculator.java + } + + public Calculator subtract(double number) { + result -= number; + return this; +-- + public static void main(String[] args) { + Calculator calc = new Calculator(); + calc.add(10).subtract(5); + System.out.println("Result: " + calc.getResult()); + } + +./calculator_test/Calculator.java + } + + public int subtract(int a, int b) { + return a - b; + } + +./test.rb + end + + def subtract(number) + @result -= number + self +-- + +calc = Calculator.new +calc.add(5).subtract(3) +puts "Result: #{calc.get_result}" + +./test.js + return this; + } + subtract(number) { + return this; + } + +./test.py + return self + + def subtract(self, number): + self.result -= number + return self + +./test.ts + } + + subtract(number: number): Calculator { + this.result -= number; + return this;`; + +test("formats grep search results correctly", () => { + const result = formatGrepSearchResults(sampleGrepOutput); + + expect(result.numResults).toBe(13); // 4 file paths in the sample + expect(result.truncated).toBe(false); + + // Check that all file paths are preserved + expect(result.formatted).toContain("./program.cs"); + expect(result.formatted).toContain("./test.kt"); + expect(result.formatted).toContain("./test.php"); + expect(result.formatted).toContain("./Calculator.java"); + + // Check that content is preserved and properly indented + expect(result.formatted).toContain('Console.WriteLine("Hello World!");'); + expect(result.formatted).toContain("fun subtract(number: Double): Test {"); + expect(result.formatted).toContain("public function subtract($number) {"); + expect(result.formatted).toContain( + "public Calculator subtract(double number) {", + ); +}); + +test("handles empty input", () => { + const result = formatGrepSearchResults(""); + + expect(result.numResults).toBe(0); + expect(result.truncated).toBe(false); + expect(result.formatted).toBe(""); +}); + +test("handles input with only file paths (no content) - lines should be skipped", () => { + const input = + "./empty-file.ts\n./another-empty.js\n./has-content.ts\n searchMatch"; + const result = formatGrepSearchResults(input); + + expect(result.numResults).toBe(3); + expect(result.formatted).toBe("./has-content.ts\n searchMatch"); +}); + +test("truncates output when exceeding maxChars", () => { + const maxChars = 50; + const result = formatGrepSearchResults(sampleGrepOutput, maxChars); + + expect(result.truncated).toBe(true); + expect(result.formatted.length).toBe(maxChars); +}); + +test("normalizes indentation to 2 spaces", () => { + const input = `./file.ts + function test() { + const x = 1; + console.log(x); + }`; + + const result = formatGrepSearchResults(input); + + // The function should normalize the indentation to 2 spaces + expect(result.formatted).toContain("./file.ts"); + expect(result.formatted).toContain(" function test() {"); + expect(result.formatted).toContain(" const x = 1;"); + expect(result.formatted).toContain(" console.log(x);"); + expect(result.formatted).toContain(" }"); +}); + +test("skips leading single-char or empty lines after file paths", () => { + const input = `./file.ts + +function test() { + console.log("test"); +}`; + + const result = formatGrepSearchResults(input); + + expect(result.formatted).toContain("./file.ts"); + expect(result.formatted).not.toContain("\n \n"); // The single space line should be skipped + expect(result.formatted).toContain("function test() {"); +}); + +test("processes separator lines correctly", () => { + const input = `./file1.ts +function one() { + return 1; +} +-- +function two() { + return 2; +} + +./file2.ts +class Test { + method() {} +}`; + + const result = formatGrepSearchResults(input); + + expect(result.numResults).toBe(3); + expect(result.formatted).toContain("./file1.ts"); + expect(result.formatted).toContain("--"); + expect(result.formatted).toContain("function two() {"); + expect(result.formatted).toContain("./file2.ts"); +}); + +test("increases indentation when original is less than 2 spaces", () => { + const input = `./minimal-indent.ts +function test() { + singleSpaceIndent(); +}`; + + const result = formatGrepSearchResults(input); + + expect(result.formatted).toContain(" function test() {"); + expect(result.formatted).toContain(" singleSpaceIndent();"); + expect(result.formatted).toContain(" }"); +}); + +test("decreases indentation when original is more than 2 spaces", () => { + const input = `./excessive-indent.ts + function test() { + tooMuchIndent(); + }`; + + const result = formatGrepSearchResults(input); + + expect(result.formatted).toContain(" function test() {"); + expect(result.formatted).toContain(" tooMuchIndent();"); + expect(result.formatted).toContain(" }"); +}); diff --git a/docs/docs/agent/how-it-works.md b/docs/docs/agent/how-it-works.md index 1191fa9f81..1bfeb075ca 100644 --- a/docs/docs/agent/how-it-works.md +++ b/docs/docs/agent/how-it-works.md @@ -31,7 +31,7 @@ Continue includes several built-in tools which provide the model access to IDE f - **Read file** (`builtin_read_file`): read the contents of a file within the project - **Read currently open file** (`builtin_read_currently_open_file`): read the contents of the currently open file - **Create new file** (`builtin_create_new_file`): Create a new file within the project, with path and contents specified by the model -- **Exact search** (`builtin_exact_search`): perform a `ripgrep` search within the project +- **Grep search** (`builtin_grep_search`): perform a `ripgrep` search within the project. Results are limited to 100 results and 5000 characters (model is warned if this is exceeded) - **Run terminal command** (`builtin_run_terminal_command`): run a terminal command from the workspace root - **Search web** (`builtin_search_web`): Perform a web search to get top results - **View diff** (`builtin_view_diff`): View the current working git diff diff --git a/docs/docs/customize/context-providers.mdx b/docs/docs/customize/context-providers.mdx index 204f680f05..c55bbfbbd9 100644 --- a/docs/docs/customize/context-providers.mdx +++ b/docs/docs/customize/context-providers.mdx @@ -288,6 +288,8 @@ Reference the results of codebase search, just like the results you would get fr ```yaml title="config.yaml" context: - provider: search + params: + maxResults: 100 # optional, defaults to 200 ``` @@ -295,7 +297,10 @@ Reference the results of codebase search, just like the results you would get fr { "contextProviders": [ { - "name": "search" + "name": "search", + "params": { + "maxResults": 100 // optional, defaults to 200 + } } ] } diff --git a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IdeProtocolClient.kt b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IdeProtocolClient.kt index cd2b3b86ac..4c2151921e 100644 --- a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IdeProtocolClient.kt +++ b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IdeProtocolClient.kt @@ -356,7 +356,7 @@ class IdeProtocolClient( dataElement.toString(), GetSearchResultsParams::class.java ) - val results = ide.getSearchResults(params.query) + val results = ide.getSearchResults(params.query, params.maxResults) respond(results) } diff --git a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IntelliJIde.kt b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IntelliJIde.kt index 908ba2edd1..6fb0dc950a 100644 --- a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IntelliJIde.kt +++ b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IntelliJIde.kt @@ -344,11 +344,11 @@ class IntelliJIDE( throw NotImplementedError("Ripgrep not supported, this workspace is remote") } } - override suspend fun getSearchResults(query: String): String { + override suspend fun getSearchResults(query: String, maxResults: Int?): String { val ideInfo = this.getIdeInfo() if (ideInfo.remoteName == "local") { try { - val command = GeneralCommandLine( + val commandArgs = mutableListOf( ripgrep, "-i", "--ignore-file", @@ -357,11 +357,21 @@ class IntelliJIDE( ".gitignore", "-C", "2", - "--heading", - "-e", - query, - "." + "--heading" ) + + // Conditionally add maxResults flag + if (maxResults != null) { + commandArgs.add("-m") + commandArgs.add(maxResults.toString()) + } + + // Add the search query and path + commandArgs.add("-e") + commandArgs.add(query) + commandArgs.add(".") + + val command = GeneralCommandLine(commandArgs) command.setWorkDirectory(project.basePath) return ExecUtil.execAndGetOutput(command).stdout diff --git a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/protocol/ide.kt b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/protocol/ide.kt index 4c66fcc0cb..807b9d0db6 100644 --- a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/protocol/ide.kt +++ b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/protocol/ide.kt @@ -21,7 +21,7 @@ typealias OpenUrlParam = String typealias getTagsParams = String -data class GetSearchResultsParams(val query: String) +data class GetSearchResultsParams(val query: String, val maxResults: Int?) data class GetFileResultsParams(val pattern: String) diff --git a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/types.kt b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/types.kt index 71e7092e07..4311733152 100644 --- a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/types.kt +++ b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/types.kt @@ -157,7 +157,7 @@ interface IDE { suspend fun getPinnedFiles(): List - suspend fun getSearchResults(query: String): String + suspend fun getSearchResults(query: String, maxResults: Int?): String suspend fun getFileResults(pattern: String): List diff --git a/extensions/vscode/config_schema.json b/extensions/vscode/config_schema.json index 76686f24ea..29d8824896 100644 --- a/extensions/vscode/config_schema.json +++ b/extensions/vscode/config_schema.json @@ -2144,6 +2144,29 @@ } } }, + { + "if": { + "properties": { + "name": { + "enum": ["search"] + } + } + }, + "then": { + "properties": { + "params": { + "properties": { + "maxResults": { + "title": "Max Results", + "description": "The maximum number of results to return", + "default": 200, + "type": "integer" + } + } + } + } + } + }, { "if": { "properties": { diff --git a/extensions/vscode/src/VsCodeIde.ts b/extensions/vscode/src/VsCodeIde.ts index dcc17d1795..bb701db6a7 100644 --- a/extensions/vscode/src/VsCodeIde.ts +++ b/extensions/vscode/src/VsCodeIde.ts @@ -524,7 +524,7 @@ class VsCodeIde implements IDE { } } - async getSearchResults(query: string): Promise { + async getSearchResults(query: string, maxResults?: number): Promise { if (vscode.env.remoteName) { throw new Error("Ripgrep not supported, this workspace is remote"); } @@ -539,6 +539,7 @@ class VsCodeIde implements IDE { "-C", "2", // Show 2 lines of context "--heading", // Only show filepath once per result + ...(maxResults ? ["-m", maxResults.toString()] : []), "-e", query, // Pattern to search for ".", // Directory to search in @@ -547,7 +548,20 @@ class VsCodeIde implements IDE { results.push(dirResults); } - return results.join("\n"); + const allResults = results.join("\n"); + if (maxResults) { + // In case of multiple workspaces, do max results per workspace and then truncate to maxResults + // Will prioritize first workspace results, fine for now + // Results are separated by either ./ or -- + const matches = Array.from(allResults.matchAll(/(\n--|\n\.\/)/g)); + if (matches.length > maxResults) { + return allResults.substring(0, matches[maxResults].index); + } else { + return allResults; + } + } else { + return allResults; + } } async getProblems(fileUri?: string | undefined): Promise { diff --git a/extensions/vscode/src/extension/VsCodeMessenger.ts b/extensions/vscode/src/extension/VsCodeMessenger.ts index b0c0a4d7b8..3e11b11e64 100644 --- a/extensions/vscode/src/extension/VsCodeMessenger.ts +++ b/extensions/vscode/src/extension/VsCodeMessenger.ts @@ -321,7 +321,7 @@ export class VsCodeMessenger { await ide.runCommand(msg.data.command); }); this.onWebviewOrCore("getSearchResults", async (msg) => { - return ide.getSearchResults(msg.data.query); + return ide.getSearchResults(msg.data.query, msg.data.maxResults); }); this.onWebviewOrCore("getFileResults", async (msg) => { return ide.getFileResults(msg.data.pattern);