From 074f6d9603fd357861a9d829f37cdd6d02dc1731 Mon Sep 17 00:00:00 2001 From: Grace Brigham <30578281+GraceBrigham@users.noreply.github.com> Date: Mon, 25 Nov 2024 09:51:35 -0800 Subject: [PATCH] Agents - Vector stores files & file batches (#31906) Co-authored-by: Grace Brigham --- sdk/ai/ai-projects/review/ai-projects.api.md | 27 +++ .../samples-dev/agents/run_steps.ts | 55 +++++ .../agents/vector_store_file_batches.ts | 60 ++++++ .../samples-dev/agents/vector_store_files.ts | 54 +++++ sdk/ai/ai-projects/src/agents/index.ts | 76 ++++++- sdk/ai/ai-projects/src/agents/inputOutputs.ts | 1 + sdk/ai/ai-projects/src/agents/vectorStores.ts | 10 +- .../src/agents/vectorStoresFileBatches.ts | 67 ++++++ .../src/agents/vectorStoresFiles.ts | 69 ++++++ .../src/agents/vectorStoresModels.ts | 36 ++++ .../agents/vectorStoresFileBatches.spec.ts | 196 ++++++++++++++++++ .../public/agents/vectorStoresFiles.spec.ts | 150 ++++++++++++++ 12 files changed, 795 insertions(+), 6 deletions(-) create mode 100644 sdk/ai/ai-projects/samples-dev/agents/run_steps.ts create mode 100644 sdk/ai/ai-projects/samples-dev/agents/vector_store_file_batches.ts create mode 100644 sdk/ai/ai-projects/samples-dev/agents/vector_store_files.ts create mode 100644 sdk/ai/ai-projects/src/agents/vectorStoresFileBatches.ts create mode 100644 sdk/ai/ai-projects/src/agents/vectorStoresFiles.ts create mode 100644 sdk/ai/ai-projects/src/agents/vectorStoresModels.ts create mode 100644 sdk/ai/ai-projects/test/public/agents/vectorStoresFileBatches.spec.ts create mode 100644 sdk/ai/ai-projects/test/public/agents/vectorStoresFiles.spec.ts diff --git a/sdk/ai/ai-projects/review/ai-projects.api.md b/sdk/ai/ai-projects/review/ai-projects.api.md index 8f4b4ece8981..5f1bde86d00c 100644 --- a/sdk/ai/ai-projects/review/ai-projects.api.md +++ b/sdk/ai/ai-projects/review/ai-projects.api.md @@ -98,6 +98,7 @@ export type AgentsNamedToolChoiceTypeOutput = string; // @public (undocumented) export interface AgentsOperations { cancelRun: (threadId: string, runId: string, requestParams?: OptionalRequestParameters) => Promise; + cancelVectorStoreFileBatch: (vectorStoreId: string, batchId: string, requestParams?: OptionalRequestParameters) => Promise; createAgent: (model: string, options?: Omit, requestParams?: OptionalRequestParameters) => Promise; createMessage: (threadId: string, options: ThreadMessageOptions, requestParams?: OptionalRequestParameters) => Promise; createRun: (threadId: string, assistantId: string, options?: Omit, requestParams?: OptionalRequestParameters) => Promise; @@ -106,10 +107,13 @@ export interface AgentsOperations { createThreadAndRun: (assistantId: string, options?: Omit, requestParams?: OptionalRequestParameters) => Promise; createThreadAndRunStreaming: (assistantId: string, options?: Omit, requestParams?: OptionalRequestParameters) => Promise; createVectorStore: (options?: VectorStoreOptions, requestParams?: OptionalRequestParameters) => Promise; + createVectorStoreFile: (vectorStoreId: string, options?: CreateVectorStoreFileOptions, requestParams?: OptionalRequestParameters) => Promise; + createVectorStoreFileBatch: (vectorStoreId: string, options?: CreateVectorStoreFileBatchOptions, requestParams?: OptionalRequestParameters) => Promise; deleteAgent: (assistantId: string, requestParams?: OptionalRequestParameters) => Promise; deleteFile: (fileId: string, requestParams?: OptionalRequestParameters) => Promise; deleteThread: (threadId: string, requestParams?: OptionalRequestParameters) => Promise; deleteVectorStore: (vectorStoreId: string, requestParams?: OptionalRequestParameters) => Promise; + deleteVectorStoreFile: (vectorStoreId: string, fileId: string, requestParams?: OptionalRequestParameters) => Promise; getAgent: (assistantId: string, requestParams?: OptionalRequestParameters) => Promise; getFile: (fileId: string, requestParams?: OptionalRequestParameters) => Promise; getFileContent: (fileId: string, requestParams?: OptionalRequestParameters) => Promise; @@ -117,11 +121,15 @@ export interface AgentsOperations { getRunStep: (threadId: string, runId: string, stepId: string, requestParams?: OptionalRequestParameters) => Promise; getThread: (threadId: string, requestParams?: OptionalRequestParameters) => Promise; getVectorStore: (vectorStoreId: string, requestParams?: OptionalRequestParameters) => Promise; + getVectorStoreFile: (vectorStoreId: string, fileId: string, requestParams?: OptionalRequestParameters) => Promise; + getVectorStoreFileBatch: (vectorStoreId: string, batchId: string) => Promise; listAgents: (options?: ListQueryParameters, requestParams?: OptionalRequestParameters) => Promise; listFiles: (purpose?: FilePurpose, requestParams?: OptionalRequestParameters) => Promise; listMessages: (threadId: string, runId?: string, options?: ListQueryParameters, requestParams?: OptionalRequestParameters) => Promise; listRuns: (threadId: string, options?: ListQueryParameters, requestParams?: OptionalRequestParameters) => Promise; listRunSteps: (threadId: string, runId: string, options?: ListQueryParameters, requestParams?: OptionalRequestParameters) => Promise; + listVectorStoreFileBatchFiles: (vectorStoreId: string, batchId: string, options?: ListQueryParameters & FileStatusFilter, requestParams?: OptionalRequestParameters) => Promise; + listVectorStoreFiles: (vectorStoreId: string, options?: ListQueryParameters & FileStatusFilter, requestParams?: OptionalRequestParameters) => Promise; listVectorStores: (options?: ListQueryParameters, requestParams?: OptionalRequestParameters) => Promise; modifyVectorStore: (vectorStoreId: string, options?: VectorStoreUpdateOptions, requestParams?: OptionalRequestParameters) => Promise; submitToolOutputsToRun: (threadId: string, runId: string, tool_outputs: Array, stream?: boolean | null, options?: OptionalRequestParameters) => Promise; @@ -323,6 +331,20 @@ export interface CreateRunOptions { truncation_strategy?: TruncationObject | null; } +// @public +export interface CreateVectorStoreFileBatchOptions { + chunkingStrategy?: VectorStoreChunkingStrategyRequest; + dataSources?: VectorStoreDataSource[]; + fileIds?: string[]; +} + +// @public +export interface CreateVectorStoreFileOptions { + chunkingStrategy?: VectorStoreChunkingStrategyRequest; + dataSources?: Array; + fileId?: string; +} + // @public export interface CredentialsApiKeyAuthOutput { key: string; @@ -500,6 +522,11 @@ export interface FileSearchToolResourceOutput { // @public export type FileStateOutput = string; +// @public +export interface FileStatusFilter { + filter?: VectorStoreFileStatusFilter; +} + // @public export type Frequency = string; diff --git a/sdk/ai/ai-projects/samples-dev/agents/run_steps.ts b/sdk/ai/ai-projects/samples-dev/agents/run_steps.ts new file mode 100644 index 000000000000..e2028b3108f1 --- /dev/null +++ b/sdk/ai/ai-projects/samples-dev/agents/run_steps.ts @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { AIProjectsClient } from "@azure/ai-projects"; +import { DefaultAzureCredential } from "@azure/identity"; +import * as dotenv from "dotenv"; +dotenv.config(); + +const connectionString = process.env["AZURE_AI_PROJECTS_CONNECTION_STRING"] || ";;;"; + +async function main(): Promise { + const client = AIProjectsClient.fromConnectionString(connectionString, new DefaultAzureCredential()); + + // Create agent + const agent = await client.agents.createAgent("gpt-4o", { name: "my-agent", instructions: "You are a helpful agent" }); + console.log(`Created agent, agent ID: ${agent.id}`); + + // Create thread + const thread = await client.agents.createThread(); + console.log(`Created thread, thread ID: ${thread.id}`); + + // Create message + const message = await client.agents.createMessage(thread.id, { role: "user", content: "hello, world!" }); + console.log(`Created message, message ID: ${message.id}`); + + // Create run + let run = await client.agents.createRun(thread.id, agent.id); + console.log(`Created run, run ID: ${run.id}`); + + // Wait for run to complete + while (["queued", "in_progress", "requires_action"].includes(run.status)) { + await new Promise(resolve => setTimeout(resolve, 1000)); + run = await client.agents.getRun(thread.id, run.id); + console.log(`Run status: ${run.status}`); + } + + // List run steps + const runSteps = await client.agents.listRunSteps(thread.id, run.id); + console.log(`Listed run steps, run ID: ${run.id}`); + + // Get specific run step + const stepId = runSteps.data[0].id; + const step = await client.agents.getRunStep(thread.id, run.id, stepId); + console.log(`Retrieved run step, step ID: ${step.id}`); + + // Clean up + await client.agents.deleteThread(thread.id); + console.log(`Deleted thread, thread ID: ${thread.id}`); + await client.agents.deleteAgent(agent.id); + console.log(`Deleted agent, agent ID: ${agent.id}`); +} + +main().catch((err) => { + console.error("The sample encountered an error:", err); +}); diff --git a/sdk/ai/ai-projects/samples-dev/agents/vector_store_file_batches.ts b/sdk/ai/ai-projects/samples-dev/agents/vector_store_file_batches.ts new file mode 100644 index 000000000000..122ce833d2c8 --- /dev/null +++ b/sdk/ai/ai-projects/samples-dev/agents/vector_store_file_batches.ts @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { AIProjectsClient } from "@azure/ai-projects"; +import { DefaultAzureCredential } from "@azure/identity"; +import * as dotenv from "dotenv"; +import { Readable } from "stream"; +dotenv.config(); + +const connectionString = process.env["AZURE_AI_PROJECTS_CONNECTION_STRING"] || ">;;;"; + +export async function main(): Promise { + const client = AIProjectsClient.fromConnectionString(connectionString || "", new DefaultAzureCredential()); + + // Create vector store + const vectorStore = await client.agents.createVectorStore(); + console.log(`Created vector store, vector store ID: ${vectorStore.id}`); + + // Create and upload first file + const file1Content = "Hello, Vector Store!"; + const readable1 = new Readable(); + readable1.push(file1Content); + readable1.push(null); // end the stream + const file1 = await client.agents.uploadFile(readable1, "assistants", "vector-file1.txt"); + console.log(`Uploaded file1, file ID: ${file1.id}`); + + // Create and upload second file + const file2Content = "This is another file for the Vector Store!"; + const readable2 = new Readable(); + readable2.push(file2Content); + readable2.push(null); // end the stream + const file2 = await client.agents.uploadFile(readable2, "assistants", "vector-file2.txt"); + console.log(`Uploaded file2, file ID: ${file2.id}`); + + // Create vector store file batch + const vectorStoreFileBatch = await client.agents.createVectorStoreFileBatch(vectorStore.id, { fileIds: [file1.id, file2.id] }); + console.log(`Created vector store file batch, vector store file batch ID: ${vectorStoreFileBatch.id}`); + + // Retrieve vector store file batch + const _vectorStoreFileBatch = await client.agents.getVectorStoreFileBatch(vectorStore.id, vectorStoreFileBatch.id); + console.log(`Retrieved vector store file batch, vector store file batch ID: ${_vectorStoreFileBatch.id}`); + + // List vector store files in the batch + const vectorStoreFiles = await client.agents.listVectorStoreFileBatchFiles(vectorStore.id, vectorStoreFileBatch.id); + console.log(`List of vector store files in the batch: ${vectorStoreFiles.data.map(f => f.id).join(", ")}`); + + // Delete files + await client.agents.deleteFile(file1.id); + console.log(`Deleted file1, file ID: ${file1.id}`); + await client.agents.deleteFile(file2.id); + console.log(`Deleted file2, file ID: ${file2.id}`); + + // Delete vector store + await client.agents.deleteVectorStore(vectorStore.id); + console.log(`Deleted vector store, vector store ID: ${vectorStore.id}`); +} + +main().catch((err) => { + console.error("The sample encountered an error:", err); +}); diff --git a/sdk/ai/ai-projects/samples-dev/agents/vector_store_files.ts b/sdk/ai/ai-projects/samples-dev/agents/vector_store_files.ts new file mode 100644 index 000000000000..6b46a0d44848 --- /dev/null +++ b/sdk/ai/ai-projects/samples-dev/agents/vector_store_files.ts @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { AIProjectsClient } from "@azure/ai-projects"; +import { DefaultAzureCredential } from "@azure/identity"; +import * as dotenv from "dotenv"; +import { Readable } from "stream"; +dotenv.config(); + +const connectionString = process.env["AZURE_AI_PROJECTS_CONNECTION_STRING"] || ">;;;"; + +export async function main(): Promise { + const client = AIProjectsClient.fromConnectionString(connectionString || "", new DefaultAzureCredential()); + + // Create vector store + const vectorStore = await client.agents.createVectorStore(); + console.log(`Created vector store, vector store ID: ${vectorStore.id}`); + + // Create and upload file + const fileContent = "Hello, Vector Store!"; + const readable = new Readable(); + readable.push(fileContent); + readable.push(null); // end the stream + const file = await client.agents.uploadFile(readable, "assistants", "vector-file.txt"); + console.log(`Uploaded file, file ID: ${file.id}`); + + // Create vector store file + const vectorStoreFile = await client.agents.createVectorStoreFile(vectorStore.id, { fileId: file.id }); + console.log(`Created vector store file, vector store file ID: ${vectorStoreFile.id}`); + + // Retrieve vector store file + const _vectorStoreFile = await client.agents.getVectorStoreFile(vectorStore.id, vectorStoreFile.id); + console.log(`Retrieved vector store file, vector store file ID: ${_vectorStoreFile.id}`); + + // List vector store files + const vectorStoreFiles = await client.agents.listVectorStoreFiles(vectorStore.id); + console.log(`List of vector store files: ${vectorStoreFiles.data.map(f => f.id).join(", ")}`); + + // Delete vector store file + await client.agents.deleteVectorStoreFile(vectorStore.id, vectorStoreFile.id); + console.log(`Deleted vector store file, vector store file ID: ${vectorStoreFile.id}`); + + // Delete file + await client.agents.deleteFile(file.id); + console.log(`Deleted file, file ID: ${file.id}`); + + // Delete vector store + await client.agents.deleteVectorStore(vectorStore.id); + console.log(`Deleted vector store, vector store ID: ${vectorStore.id}`); +} + +main().catch((err) => { + console.error("The sample encountered an error:", err); +}); diff --git a/sdk/ai/ai-projects/src/agents/index.ts b/sdk/ai/ai-projects/src/agents/index.ts index 9c72234aa392..5d5951214ee8 100644 --- a/sdk/ai/ai-projects/src/agents/index.ts +++ b/sdk/ai/ai-projects/src/agents/index.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import { Client } from "@azure-rest/core-client"; -import { AgentDeletionStatusOutput, AgentOutput, AgentThreadOutput, FileDeletionStatusOutput, FileListResponseOutput, OpenAIFileOutput, OpenAIPageableListOfAgentOutput, OpenAIPageableListOfRunStepOutput, OpenAIPageableListOfThreadMessageOutput, OpenAIPageableListOfThreadRunOutput, OpenAIPageableListOfVectorStoreOutput, RunStepOutput, ThreadDeletionStatusOutput, ThreadMessageOutput, ThreadRunOutput, VectorStoreDeletionStatusOutput, VectorStoreOutput } from "../generated/src/outputModels.js"; +import { AgentDeletionStatusOutput, AgentOutput, AgentThreadOutput, FileDeletionStatusOutput, FileListResponseOutput, OpenAIFileOutput, OpenAIPageableListOfAgentOutput, OpenAIPageableListOfRunStepOutput, OpenAIPageableListOfThreadMessageOutput, OpenAIPageableListOfThreadRunOutput, OpenAIPageableListOfVectorStoreFileOutput, OpenAIPageableListOfVectorStoreOutput, RunStepOutput, ThreadDeletionStatusOutput, ThreadMessageOutput, ThreadRunOutput, VectorStoreDeletionStatusOutput, VectorStoreFileBatchOutput, VectorStoreFileDeletionStatusOutput, VectorStoreFileOutput, VectorStoreOutput } from "../generated/src/outputModels.js"; import { createAgent, deleteAgent, getAgent, listAgents, updateAgent } from "./assistants.js"; import { deleteFile, getFile, getFileContent, listFiles, uploadFile } from "./files.js"; import { createThread, deleteThread, getThread, updateThread } from "./threads.js"; @@ -14,6 +14,9 @@ import { UpdateMessageOptions } from "./messagesModels.js"; import { AgentEventMessageStream, ListQueryParameters, OptionalRequestParameters, UpdateRunOptions } from "./inputOutputs.js"; import { createVectorStore, deleteVectorStore, getVectorStore, listVectorStores, modifyVectorStore } from "./vectorStores.js"; import { getRunStep, listRunSteps } from "./runSteps.js"; +import { CreateVectorStoreFileBatchOptions, CreateVectorStoreFileOptions, FileStatusFilter } from "./vectorStoresModels.js"; +import { createVectorStoreFile, deleteVectorStoreFile, getVectorStoreFile, listVectorStoreFiles } from "./vectorStoresFiles.js"; +import { cancelVectorStoreFileBatch, createVectorStoreFileBatch, getVectorStoreFileBatch, listVectorStoreFileBatchFiles } from "./vectorStoresFileBatches.js"; export interface AgentsOperations { /** Creates a new agent. */ @@ -199,6 +202,59 @@ export interface AgentsOperations { requestParams?: OptionalRequestParameters, ) => Promise; + /** Create a vector store file by attching a file to a vector store. */ + createVectorStoreFile: ( + vectorStoreId: string, + options?: CreateVectorStoreFileOptions, + requestParams?: OptionalRequestParameters, + ) => Promise; + /** Retrieves a vector store file. */ + getVectorStoreFile: ( + vectorStoreId: string, + fileId: string, + requestParams?: OptionalRequestParameters, + ) => Promise; + /** Returns a list of vector store files. */ + listVectorStoreFiles: ( + vectorStoreId: string, + options?: ListQueryParameters & FileStatusFilter, + requestParams?: OptionalRequestParameters, + ) => Promise; + /** + * Delete a vector store file. This will remove the file from the vector store but the file itself will not be deleted. + * To delete the file, use the delete file endpoint. + */ + deleteVectorStoreFile: ( + vectorStoreId: string, + fileId: string, + requestParams?: OptionalRequestParameters, + ) => Promise; + + /** Create a vector store file batch. */ + createVectorStoreFileBatch: ( + vectorStoreId: string, + options?: CreateVectorStoreFileBatchOptions, + requestParams?: OptionalRequestParameters, + ) => Promise; + /** Retrieve a vector store file batch. */ + getVectorStoreFileBatch: ( + vectorStoreId: string, + batchId: string, + ) => Promise; + /** Cancel a vector store file batch. This attempts to cancel the processing of files in this batch as soon as possible. */ + cancelVectorStoreFileBatch: ( + vectorStoreId: string, + batchId: string, + requestParams?: OptionalRequestParameters, + ) => Promise; + /** Returns a list of vector store files in a batch. */ + listVectorStoreFileBatchFiles: ( + vectorStoreId: string, + batchId: string, + options?: ListQueryParameters & FileStatusFilter, + requestParams?: OptionalRequestParameters, + ) => Promise; + /** Gets a single run step from a thread run. */ getRunStep: ( threadId: string, @@ -300,6 +356,24 @@ function getAgents(context: Client): AgentsOperations { deleteVectorStore: (vectorStoreId: string, requestParams?: OptionalRequestParameters) => deleteVectorStore(context, vectorStoreId, requestParams), + createVectorStoreFile: (vectorStoreId: string, options?: CreateVectorStoreFileOptions, requestParams?: OptionalRequestParameters) => + createVectorStoreFile(context, vectorStoreId, { ...requestParams, body: {file_id: options?.fileId, data_sources: options?.dataSources, chunking_strategy: options?.chunkingStrategy} }), + getVectorStoreFile: (vectorStoreId: string, fileId: string, requestParams?: OptionalRequestParameters) => + getVectorStoreFile(context, vectorStoreId, fileId, requestParams), + listVectorStoreFiles: (vectorStoreId: string, options?: ListQueryParameters & FileStatusFilter, requestParams?: OptionalRequestParameters) => + listVectorStoreFiles(context, vectorStoreId, { ...requestParams, queryParameters: options as Record }), + deleteVectorStoreFile: (vectorStoreId: string, fileId: string, requestParams?: OptionalRequestParameters) => + deleteVectorStoreFile(context, vectorStoreId, fileId, requestParams), + + createVectorStoreFileBatch: (vectorStoreId: string, options?: CreateVectorStoreFileBatchOptions, requestParams?: OptionalRequestParameters) => + createVectorStoreFileBatch(context, vectorStoreId, { ...requestParams, body: { file_ids: options?.fileIds, data_sources: options?.dataSources, chunking_strategy: options?.chunkingStrategy } }), + getVectorStoreFileBatch: (vectorStoreId: string, batchId: string, requestParams?: OptionalRequestParameters) => + getVectorStoreFileBatch(context, vectorStoreId, batchId, requestParams), + cancelVectorStoreFileBatch: (vectorStoreId: string, batchId: string, requestParams?: OptionalRequestParameters) => + cancelVectorStoreFileBatch(context, vectorStoreId, batchId, requestParams), + listVectorStoreFileBatchFiles: (vectorStoreId: string, batchId: string, options?: ListQueryParameters & FileStatusFilter, requestParams?: OptionalRequestParameters) => + listVectorStoreFileBatchFiles(context, vectorStoreId, batchId, { ...requestParams, queryParameters: options as Record }), + getRunStep: (threadId: string, runId: string, stepId: string, requestParams?: OptionalRequestParameters) => getRunStep(context, threadId, runId, stepId, { ...requestParams }), listRunSteps: (threadId: string, runId: string, options?: ListQueryParameters, requestParams?: OptionalRequestParameters) => diff --git a/sdk/ai/ai-projects/src/agents/inputOutputs.ts b/sdk/ai/ai-projects/src/agents/inputOutputs.ts index a432d3e2dae6..02345833253c 100644 --- a/sdk/ai/ai-projects/src/agents/inputOutputs.ts +++ b/sdk/ai/ai-projects/src/agents/inputOutputs.ts @@ -8,4 +8,5 @@ export * from "../generated/src/models.js" export * from "../generated/src/outputModels.js" export * from "./customModels.js" export * from "./runModels.js" +export * from "./vectorStoresModels.js"; export * from "./utils.js"; diff --git a/sdk/ai/ai-projects/src/agents/vectorStores.ts b/sdk/ai/ai-projects/src/agents/vectorStores.ts index 47ab95399b2c..fa63ebd2aa18 100644 --- a/sdk/ai/ai-projects/src/agents/vectorStores.ts +++ b/sdk/ai/ai-projects/src/agents/vectorStores.ts @@ -8,7 +8,7 @@ import { OpenAIPageableListOfVectorStoreOutput, VectorStoreDeletionStatusOutput, const expectedStatuses = ["200"]; /** Returns a list of vector stores. */ -export async function listVectorStores ( +export async function listVectorStores( context: Client, options?: ListVectorStoresParameters, ): Promise { @@ -21,7 +21,7 @@ export async function listVectorStores ( } /** Creates a vector store. */ -export async function createVectorStore ( +export async function createVectorStore( context: Client, options?: CreateVectorStoreParameters, ): Promise { @@ -34,7 +34,7 @@ export async function createVectorStore ( } /** Returns the vector store object matching the specified ID. */ -export async function getVectorStore ( +export async function getVectorStore( context: Client, vectorStoreId: string, options?: GetVectorStoreParameters, @@ -50,7 +50,7 @@ export async function getVectorStore ( } /** The ID of the vector store to modify. */ -export async function modifyVectorStore ( +export async function modifyVectorStore( context: Client, vectorStoreId: string, options?: ModifyVectorStoreParameters, @@ -66,7 +66,7 @@ export async function modifyVectorStore ( } /** Deletes the vector store object matching the specified ID. */ -export async function deleteVectorStore ( +export async function deleteVectorStore( context: Client, vectorStoreId: string, options?: DeleteVectorStoreParameters, diff --git a/sdk/ai/ai-projects/src/agents/vectorStoresFileBatches.ts b/sdk/ai/ai-projects/src/agents/vectorStoresFileBatches.ts new file mode 100644 index 000000000000..9b020341057a --- /dev/null +++ b/sdk/ai/ai-projects/src/agents/vectorStoresFileBatches.ts @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Client, createRestError } from "@azure-rest/core-client"; +import { CreateVectorStoreFileBatchParameters, CancelVectorStoreFileBatchParameters, GetVectorStoreFileBatchParameters, ListVectorStoreFileBatchFilesParameters } from "../generated/src/parameters.js"; +import { OpenAIPageableListOfVectorStoreFileOutput, VectorStoreFileBatchOutput } from "../generated/src/outputModels.js"; + +const expectedStatuses = ["200"]; + +/** Create a vector store file batch. */ +export async function createVectorStoreFileBatch( + context: Client, + vectorStoreId: string, + options?: CreateVectorStoreFileBatchParameters, +): Promise { + const result = await context.path("/vector_stores/{vectorStoreId}/file_batches", vectorStoreId).post(options); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; +} + +/** Retrieve a vector store file batch. */ +export async function getVectorStoreFileBatch( + context: Client, + vectorStoreId: string, + batchId: string, + options?: GetVectorStoreFileBatchParameters, +): Promise { + const result = await context + .path("/vector_stores/{vectorStoreId}/file_batches/{batchId}", vectorStoreId, batchId) + .get(options); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; +} + +/** Cancel a vector store file batch. This attempts to cancel the processing of files in this batch as soon as possible. */ +export async function cancelVectorStoreFileBatch( + context: Client, + vectorStoreId: string, + batchId: string, + options?: CancelVectorStoreFileBatchParameters, +): Promise { + const result = await context + .path("/vector_stores/{vectorStoreId}/file_batches/{batchId}/cancel", vectorStoreId, batchId) + .post(options); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; +} + +/** Returns a list of vector store files in a batch. */ +export async function listVectorStoreFileBatchFiles( + context: Client, + vectorStoreId: string, + batchId: string, + options?: ListVectorStoreFileBatchFilesParameters, +): Promise { + const result = await context.path("/vector_stores/{vectorStoreId}/file_batches/{batchId}/files", vectorStoreId, batchId).get(options); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; +} diff --git a/sdk/ai/ai-projects/src/agents/vectorStoresFiles.ts b/sdk/ai/ai-projects/src/agents/vectorStoresFiles.ts new file mode 100644 index 000000000000..31fea9e9341f --- /dev/null +++ b/sdk/ai/ai-projects/src/agents/vectorStoresFiles.ts @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Client, createRestError } from "@azure-rest/core-client"; +import { ListVectorStoreFilesParameters, CreateVectorStoreFileParameters, GetVectorStoreFileParameters, DeleteVectorStoreFileParameters } from "../generated/src/parameters.js"; +import { OpenAIPageableListOfVectorStoreFileOutput, VectorStoreFileDeletionStatusOutput, VectorStoreFileOutput } from "../generated/src/outputModels.js"; + +const expectedStatuses = ["200"]; + +/** Returns a list of vector store files. */ +export async function listVectorStoreFiles( + context: Client, + vectorStoreId: string, + options?: ListVectorStoreFilesParameters, +): Promise { + const result = await context.path("/vector_stores/{vectorStoreId}/files", vectorStoreId).get(options); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; +} + +/** Create a vector store file by attaching a file to a vector store. */ +export async function createVectorStoreFile( + context: Client, + vectorStoreId: string, + options?: CreateVectorStoreFileParameters, +): Promise { + const result = await context.path("/vector_stores/{vectorStoreId}/files", vectorStoreId).post(options); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; +} + +/** Retrieves a vector store file. */ +export async function getVectorStoreFile( + context: Client, + vectorStoreId: string, + fileId: string, + options?: GetVectorStoreFileParameters, +): Promise { + const result = await context + .path("/vector_stores/{vectorStoreId}/files/{fileId}", vectorStoreId, fileId) + .get(options); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; +} + +/** + * Delete a vector store file. This will remove the file from the vector store but the file itself will not be deleted. + * To delete the file, use the delete file endpoint. + */ +export async function deleteVectorStoreFile( + context: Client, + vectorStoreId: string, + fileId: string, + options?: DeleteVectorStoreFileParameters, +): Promise { + const result = await context + .path("/vector_stores/{vectorStoreId}/files/{fileId}", vectorStoreId, fileId) + .delete(options); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; +} diff --git a/sdk/ai/ai-projects/src/agents/vectorStoresModels.ts b/sdk/ai/ai-projects/src/agents/vectorStoresModels.ts new file mode 100644 index 000000000000..2ad9027395c9 --- /dev/null +++ b/sdk/ai/ai-projects/src/agents/vectorStoresModels.ts @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { VectorStoreChunkingStrategyRequest, VectorStoreDataSource, VectorStoreFileStatusFilter } from "../generated/src/models.js"; + +/** Request object for creating a vector store file. */ +export interface CreateVectorStoreFileOptions { + /** A File ID that the vector store should use. Useful for tools like `file_search` that can access files. */ + fileId?: string; + + /** The data sources to be used. This option is mutually exclusive with fileId. */ + dataSources?: Array; + + /** The chunking strategy used to chunk the file(s). If not set, will use the auto strategy. */ + chunkingStrategy?: VectorStoreChunkingStrategyRequest; +} + +/** Request object for creating a vector store file batch. */ +export interface CreateVectorStoreFileBatchOptions { + /** A list of File IDs that the vector store should use. Useful for tools like `file_search` that can access files. */ + fileIds?: string[]; + + /** The data sources to be used. This option is mutually exclusive with fileId. */ + dataSources?: VectorStoreDataSource[]; + + /** The chunking strategy used to chunk the file(s). If not set, will use the auto strategy. */ + chunkingStrategy?: VectorStoreChunkingStrategyRequest; +} + +/** Filter by file status. */ +export interface FileStatusFilter { + /** + * Possible values: "in_progress", "completed", "failed", "cancelled" + */ + filter?: VectorStoreFileStatusFilter; +} diff --git a/sdk/ai/ai-projects/test/public/agents/vectorStoresFileBatches.spec.ts b/sdk/ai/ai-projects/test/public/agents/vectorStoresFileBatches.spec.ts new file mode 100644 index 000000000000..37f2d78c967b --- /dev/null +++ b/sdk/ai/ai-projects/test/public/agents/vectorStoresFileBatches.spec.ts @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Recorder, VitestTestContext } from "@azure-tools/test-recorder"; +import { AgentsOperations, AIProjectsClient } from "../../../src/index.js"; +import { createRecorder, createProjectsClient } from "../utils/createClient.js"; +import { assert, beforeEach, afterEach, it, describe } from "vitest"; + +describe("Agents - vector stores file batches", () => { + let recorder: Recorder; + let projectsClient : AIProjectsClient; + let agents: AgentsOperations + + beforeEach(async function (context: VitestTestContext) { + recorder = await createRecorder(context); + projectsClient = createProjectsClient(recorder); + agents = projectsClient.agents + }); + + afterEach(async function () { + await recorder.stop(); + }); + + it("client and agents operations are accessible", async function () { + assert.isNotNull(projectsClient); + assert.isNotNull(agents); + }); + + it("should create a vector store file batch", async function () { + // Create vector store + const vectorStore = await agents.createVectorStore(); + console.log(`Created vector store, vector store ID: ${vectorStore.id}`); + + // Upload files + const file1Content = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("fileContent")); + controller.close(); + } + }); + const file1 = await agents.uploadFile(file1Content, "assistants", "file1.txt"); + console.log(`Uploaded file1, file1 ID: ${file1.id}`); + + const file2Content = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("fileContent")); + controller.close(); + } + }); + const file2 = await agents.uploadFile(file2Content, "assistants", "file2.txt"); + console.log(`Uploaded file2, file2 ID: ${file2.id}`); + + // Create vector store file batch + const vectorStoreFileBatch = await agents.createVectorStoreFileBatch(vectorStore.id, {fileIds: [file1.id, file2.id]}); + assert.isNotNull(vectorStoreFileBatch); + assert.isNotEmpty(vectorStoreFileBatch.id); + assert.equal(vectorStoreFileBatch.vector_store_id, vectorStore.id); + console.log(`Created vector store file batch, vector store file batch ID: ${vectorStoreFileBatch.id}`); + + // Clean up + await agents.deleteFile(file1.id) + console.log(`Deleted file1, file1 ID: ${file1.id}`); + await agents.deleteFile(file2.id) + console.log(`Deleted file2, file2 ID: ${file2.id}`); + await agents.deleteVectorStore(vectorStore.id); + console.log(`Deleted vector store, vector store ID: ${vectorStore.id}`); + }); + + it("should retrieve a vector store file batch", async function () { + // Create vector store + const vectorStore = await agents.createVectorStore(); + console.log(`Created vector store, vector store ID: ${vectorStore.id}`); + + // Upload files + const file1Content = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("fileContent")); + controller.close(); + } + }); + const file1 = await agents.uploadFile(file1Content, "assistants", "file1.txt"); + console.log(`Uploaded file1, file1 ID: ${file1.id}`); + + const file2Content = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("fileContent")); + controller.close(); + } + }); + const file2 = await agents.uploadFile(file2Content, "assistants", "file2.txt"); + console.log(`Uploaded file2, file2 ID: ${file2.id}`); + + // Create vector store file batch + const vectorStoreFileBatch = await agents.createVectorStoreFileBatch(vectorStore.id, {fileIds: [file1.id, file2.id]}); + console.log(`Created vector store file batch, vector store file batch ID: ${vectorStoreFileBatch.id}`); + + // Retrieve vector store file batch + const _vectorStoreFileBatch = await agents.getVectorStoreFileBatch(vectorStore.id, vectorStoreFileBatch.id); + assert.isNotNull(_vectorStoreFileBatch); + assert.equal(_vectorStoreFileBatch.id, vectorStoreFileBatch.id); + console.log(`Retrieved vector store file batch, vector store file batch ID: ${_vectorStoreFileBatch.id}`); + + // Clean up + await agents.deleteFile(file1.id) + console.log(`Deleted file1, file1 ID: ${file1.id}`); + await agents.deleteFile(file2.id) + console.log(`Deleted file2, file2 ID: ${file2.id}`); + await agents.deleteVectorStore(vectorStore.id); + console.log(`Deleted vector store, vector store ID: ${vectorStore.id}`); + }); + + it("should list vector store file batches", async function () { + // Create vector store + const vectorStore = await agents.createVectorStore(); + console.log(`Created vector store, vector store ID: ${vectorStore.id}`); + + // Upload files + const file1Content = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("fileContent")); + controller.close(); + } + }); + const file1 = await agents.uploadFile(file1Content, "assistants", "file1.txt"); + console.log(`Uploaded file1, file1 ID: ${file1.id}`); + + const file2Content = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("fileContent")); + controller.close(); + } + }); + const file2 = await agents.uploadFile(file2Content, "assistants", "file2.txt"); + console.log(`Uploaded file2, file2 ID: ${file2.id}`); + + // Create vector store file batch + const vectorStoreFileBatch = await agents.createVectorStoreFileBatch(vectorStore.id, {fileIds: [file1.id, file2.id]}); + console.log(`Created vector store file batch, vector store file batch ID: ${vectorStoreFileBatch.id}`); + + // List vector store files in the batch + const vectorStoreFiles = await agents.listVectorStoreFileBatchFiles(vectorStore.id, vectorStoreFileBatch.id); + assert.isNotNull(vectorStoreFiles); + assert.equal(vectorStoreFiles.data.length, 2); + console.log(`Listed ${vectorStoreFiles.data.length} vector store files in the batch, vector store file batch ID: ${vectorStoreFileBatch.id}`); + + // Clean up + await agents.deleteFile(file1.id) + console.log(`Deleted file1, file1 ID: ${file1.id}`); + await agents.deleteFile(file2.id) + console.log(`Deleted file2, file2 ID: ${file2.id}`); + await agents.deleteVectorStore(vectorStore.id); + console.log(`Deleted vector store, vector store ID: ${vectorStore.id}`); + }); + + it("should cancel a vector store file batch", async function () { + // Create vector store + const vectorStore = await agents.createVectorStore(); + console.log(`Created vector store, vector store ID: ${vectorStore.id}`); + + // Upload files + const file1Content = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("fileContent")); + controller.close(); + } + }); + const file1 = await agents.uploadFile(file1Content, "assistants", "file1.txt"); + console.log(`Uploaded file1, file1 ID: ${file1.id}`); + + const file2Content = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("fileContent")); + controller.close(); + } + }); + const file2 = await agents.uploadFile(file2Content, "assistants", "file2.txt"); + console.log(`Uploaded file2, file2 ID: ${file2.id}`); + + // Create vector store file batch + const vectorStoreFileBatch = await agents.createVectorStoreFileBatch(vectorStore.id, {fileIds: [file1.id, file2.id]}); + console.log(`Created vector store file batch, vector store file batch ID: ${vectorStoreFileBatch.id}`); + + // Cancel vector store file batch + const cancelled = await agents.cancelVectorStoreFileBatch(vectorStore.id, vectorStoreFileBatch.id); + assert.isNotNull(cancelled.status); + + // Clean up + await agents.deleteFile(file1.id) + console.log(`Deleted file1, file1 ID: ${file1.id}`); + await agents.deleteFile(file2.id) + console.log(`Deleted file2, file2 ID: ${file2.id}`); + await agents.deleteVectorStore(vectorStore.id); + console.log(`Deleted vector store, vector store ID: ${vectorStore.id}`); + }); + +}); diff --git a/sdk/ai/ai-projects/test/public/agents/vectorStoresFiles.spec.ts b/sdk/ai/ai-projects/test/public/agents/vectorStoresFiles.spec.ts new file mode 100644 index 000000000000..3b38384cd9e5 --- /dev/null +++ b/sdk/ai/ai-projects/test/public/agents/vectorStoresFiles.spec.ts @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Recorder, VitestTestContext } from "@azure-tools/test-recorder"; +import { AgentsOperations, AIProjectsClient } from "../../../src/index.js"; +import { createRecorder, createProjectsClient } from "../utils/createClient.js"; +import { assert, beforeEach, afterEach, it, describe } from "vitest"; + +describe("Agents - vector stores files", () => { + let recorder: Recorder; + let projectsClient : AIProjectsClient; + let agents: AgentsOperations + + beforeEach(async function (context: VitestTestContext) { + recorder = await createRecorder(context); + projectsClient = createProjectsClient(recorder); + agents = projectsClient.agents + }); + + afterEach(async function () { + await recorder.stop(); + }); + + it("client and agents operations are accessible", async function () { + assert.isNotNull(projectsClient); + assert.isNotNull(agents); + }); + + it("should create a vector store file", async function () { + // Create vector store + const vectorStore = await agents.createVectorStore(); + console.log(`Created vector store, vector store ID: ${vectorStore.id}`); + + // Upload file + const fileContent = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("fileContent")); + controller.close(); + } + }); + const file = await agents.uploadFile(fileContent, "assistants", "filename.txt"); + console.log(`Uploaded file, file ID: ${file.id}`); + + // Create vector store file + const vectorStoreFile = await agents.createVectorStoreFile(vectorStore.id, {fileId: file.id}); + assert.isNotNull(vectorStoreFile); + assert.isNotEmpty(vectorStoreFile.id); + console.log(`Created vector store file, vector store file ID: ${vectorStoreFile.id}`); + + // Clean up + await agents.deleteVectorStoreFile(vectorStore.id, vectorStoreFile.id); + console.log(`Deleted vector store file, vector store file ID: ${vectorStoreFile.id}`); + await agents.deleteFile(file.id) + console.log(`Deleted file, file ID: ${file.id}`); + await agents.deleteVectorStore(vectorStore.id); + console.log(`Deleted vector store, vector store ID: ${vectorStore.id}`); + }); + + it("should retrieve a vector store file", async function () { + // Create vector store + const vectorStore = await agents.createVectorStore(); + console.log(`Created vector store, vector store ID: ${vectorStore.id}`); + + // Upload file + const fileContent = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("fileContent")); + controller.close(); + } + }); + const file = await agents.uploadFile(fileContent, "assistants", "filename.txt"); + console.log(`Uploaded file, file ID: ${file.id}`); + + // Create vector store file + const vectorStoreFile = await agents.createVectorStoreFile(vectorStore.id, {fileId: file.id}); + console.log(`Created vector store file, vector store file ID: ${vectorStoreFile.id}`); + + // Retrieve vector store file + const _vectorStoreFile = await agents.getVectorStoreFile(vectorStore.id, vectorStoreFile.id); + assert.isNotNull(_vectorStoreFile); + assert.equal(_vectorStoreFile.id, vectorStoreFile.id); + console.log(`Retrieved vector store file, vector store file ID: ${_vectorStoreFile.id}`); + + // Clean up + await agents.deleteVectorStoreFile(vectorStore.id, vectorStoreFile.id); + console.log(`Deleted vector store file, vector store file ID: ${vectorStoreFile.id}`); + await agents.deleteFile(file.id) + console.log(`Deleted file, file ID: ${file.id}`); + await agents.deleteVectorStore(vectorStore.id); + console.log(`Deleted vector store, vector store ID: ${vectorStore.id}`); + }); + + it("should list vector store files", async function () { + // Create vector store + const vectorStore = await agents.createVectorStore(); + console.log(`Created vector store, vector store ID: ${vectorStore.id}`); + + // Upload file + const fileContent = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("fileContent")); + controller.close(); + } + }); + const file = await agents.uploadFile(fileContent, "assistants", "filename.txt"); + console.log(`Uploaded file, file ID: ${file.id}`); + + // Create vector store file + const vectorStoreFile = await agents.createVectorStoreFile(vectorStore.id, {fileId: file.id}); + console.log(`Created vector store file, vector store file ID: ${vectorStoreFile.id}`); + + // Clean up + await agents.deleteVectorStoreFile(vectorStore.id, vectorStoreFile.id); + console.log(`Deleted vector store file, vector store file ID: ${vectorStoreFile.id}`); + await agents.deleteFile(file.id) + console.log(`Deleted file, file ID: ${file.id}`); + await agents.deleteVectorStore(vectorStore.id); + console.log(`Deleted vector store, vector store ID: ${vectorStore.id}`); + }); + + it("should delete a vector store file", async function () { + // Create vector store + const vectorStore = await agents.createVectorStore(); + console.log(`Created vector store, vector store ID: ${vectorStore.id}`); + + // Upload file + const fileContent = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("fileContent")); + controller.close(); + } + }); + const file = await agents.uploadFile(fileContent, "assistants", "fileName.txt"); + console.log(`Uploaded file, file ID: ${file.id}`); + + // Create vector store file + const vectorStoreFile = await agents.createVectorStoreFile(vectorStore.id, {fileId: file.id}); + console.log(`Created vector store file, vector store file ID: ${vectorStoreFile.id}`); + + // Clean up + const deletionStatus = await agents.deleteVectorStoreFile(vectorStore.id, vectorStoreFile.id); + assert(deletionStatus.deleted) + console.log(`Deleted vector store file, vector store file ID: ${vectorStoreFile.id}`); + await agents.deleteFile(file.id) + console.log(`Deleted file, file ID: ${file.id}`); + await agents.deleteVectorStore(vectorStore.id); + console.log(`Deleted vector store, vector store ID: ${vectorStore.id}`); + }); + +});