diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index b0032bcdd601..01ce90b74c21 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -1618,6 +1618,26 @@ packages: - supports-color dev: false + /@azure/monitor-opentelemetry-exporter@1.0.0-beta.27: + resolution: {integrity: sha512-21iXu9ubtPB7iO3ghnzMMdB0KwHpz7Zl1a9xSBR3Gl8IDUlXOBjMn6OT9+ycj9VZrTyEKiV59T9VTf0IlokPYQ==} + engines: {node: '>=18.0.0'} + dependencies: + '@azure/core-auth': 1.8.0 + '@azure/core-client': 1.9.2 + '@azure/core-rest-pipeline': 1.17.0 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.53.0 + '@opentelemetry/core': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.53.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.27.0 + tslib: 2.7.0 + transitivePeerDependencies: + - supports-color + dev: false + /@azure/msal-browser@3.26.1: resolution: {integrity: sha512-y78sr9g61aCAH9fcLO1um+oHFXc1/5Ap88RIsUSuzkm0BHzFnN+PXGaQeuM1h5Qf5dTnWNOd6JqkskkMPAhh7Q==} engines: {node: '>=0.8.0'} @@ -10804,6 +10824,12 @@ packages: hasBin: true dev: false + /typescript@5.7.2: + resolution: {integrity: sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg==} + engines: {node: '>=14.17'} + hasBin: true + dev: false + /ua-parser-js@0.7.39: resolution: {integrity: sha512-IZ6acm6RhQHNibSt7+c09hhvsKy9WUr4DVbeq9U8o71qxyYtJpQeDxQnMrVqnIFMLcQjHO0I9wgfO2vIahht4w==} hasBin: true @@ -12118,11 +12144,16 @@ packages: dev: false file:projects/ai-projects.tgz: - resolution: {integrity: sha512-kc59ifhzGCQM5eo2AEvxx5VBGjF09tNlPfSKfMwijzN/c3smypANT1hsnO3BIrc3BH+jeqJ0dRKhpZBTJfUY2w==, tarball: file:projects/ai-projects.tgz} + resolution: {integrity: sha512-BvwFnx09ZqF5mSkxQ78M8uQ0nWtIIkaoOrVigMx/iLwPrhvKuw0ZZfQsLF4lDEJgfbVlvsfJ5KSTHvZQccK1xQ==, tarball: file:projects/ai-projects.tgz} name: '@rush-temp/ai-projects' version: 0.0.0 dependencies: + '@azure/core-lro': 2.7.2 + '@azure/monitor-opentelemetry-exporter': 1.0.0-beta.27 '@microsoft/api-extractor': 7.47.9(@types/node@18.19.55) + '@opentelemetry/api': 1.9.0 + '@opentelemetry/instrumentation': 0.53.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-node': 1.26.0(@opentelemetry/api@1.9.0) '@types/node': 18.19.55 '@vitest/browser': 2.1.3(playwright@1.48.0)(typescript@5.5.4)(vitest@2.1.3) '@vitest/coverage-istanbul': 2.1.3(vitest@2.1.3) @@ -21158,7 +21189,7 @@ packages: name: '@rush-temp/dev-tool' version: 0.0.0 dependencies: - '@_ts/max': /typescript@5.6.3 + '@_ts/max': /typescript@5.7.2 '@_ts/min': /typescript@4.2.4 '@azure/identity': 4.4.1 '@eslint/js': 9.12.0 diff --git a/sdk/ai/ai-projects/assets.json b/sdk/ai/ai-projects/assets.json index 559ec443b07f..1e6703b1d480 100644 --- a/sdk/ai/ai-projects/assets.json +++ b/sdk/ai/ai-projects/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "js", "TagPrefix": "js/ai/ai-projects", - "Tag": "js/ai/ai-projects_34268fa552" + "Tag": "js/ai/ai-projects_57b254f3af" } diff --git a/sdk/ai/ai-projects/package.json b/sdk/ai/ai-projects/package.json index ca1c5f21a589..9119393164bf 100644 --- a/sdk/ai/ai-projects/package.json +++ b/sdk/ai/ai-projects/package.json @@ -49,8 +49,12 @@ "//metadata": { "constantPaths": [ { - "path": "src/projectsClient.ts", + "path": "src/generated/src/projectsClient.ts", "prefix": "userAgentInfo" + }, + { + "path": "src/constants.ts", + "prefix": "SDK_VERSION" } ] }, @@ -64,15 +68,22 @@ "@azure/core-lro": "^2.0.0", "tslib": "^2.6.2", "@azure/core-paging": "^1.5.0", - "@azure/core-sse": "^2.1.3" + "@azure/core-sse": "^2.1.3", + "@azure/core-tracing": "^1.2.0" }, "devDependencies": { "@azure/dev-tool": "^1.0.0", "@azure/eslint-plugin-azure-sdk": "^3.0.0", "@azure/identity": "^4.3.0", + "@azure/opentelemetry-instrumentation-azure-sdk": "^1.0.0-beta.7", + "@azure/monitor-opentelemetry-exporter": "^1.0.0-beta.27", "@azure-tools/test-credential": "^2.0.0", "@azure-tools/test-recorder": "^4.1.0", + "@azure-tools/test-utils-vitest": "^1.0.0", "@microsoft/api-extractor": "^7.40.3", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/instrumentation": "0.53.0", + "@opentelemetry/sdk-trace-node": "^1.9.0", "@vitest/browser": "^2.0.5", "@vitest/coverage-istanbul": "^2.0.5", "@types/node": "^18.0.0", @@ -114,23 +125,34 @@ "./package.json": "./package.json", ".": { "browser": { + "source": "./src/index.ts", "types": "./dist/browser/index.d.ts", "default": "./dist/browser/index.js" }, "react-native": { + "source": "./src/index.ts", "types": "./dist/react-native/index.d.ts", "default": "./dist/react-native/index.js" }, "import": { + "source": "./src/index.ts", "types": "./dist/esm/index.d.ts", "default": "./dist/esm/index.js" }, "require": { + "source": "./src/index.ts", "types": "./dist/commonjs/index.d.ts", "default": "./dist/commonjs/index.js" } } }, + "//sampleConfiguration": { + "productName": "Azure AI Projects", + "productSlugs": [ + "azure" + ], + "apiRefLink": "https://learn.microsoft.com/javascript/api/@azure/ai-projects" + }, "main": "./dist/commonjs/index.js", "types": "./dist/commonjs/index.d.ts", "module": "./dist/esm/index.js" diff --git a/sdk/ai/ai-projects/review/ai-projects.api.md b/sdk/ai/ai-projects/review/ai-projects.api.md index ecd7a4d95cc4..eeb36b69e308 100644 --- a/sdk/ai/ai-projects/review/ai-projects.api.md +++ b/sdk/ai/ai-projects/review/ai-projects.api.md @@ -172,6 +172,7 @@ export class AIProjectsClient { readonly agents: AgentsOperations; readonly connections: ConnectionsOperations; static fromConnectionString(connectionString: string, credential: TokenCredential, options?: AIProjectsClientOptions): AIProjectsClient; + readonly telemetry: TelemetryOperations; } // @public (undocumented) @@ -1459,6 +1460,14 @@ export interface SystemDataOutput { readonly lastModifiedAt?: string; } +// @public +export interface TelemetryOperations { + getConnectionString(): Promise; + getSettings(): TelemetryOptions; + // Warning: (ae-forgotten-export) The symbol "TelemetryOptions" needs to be exported by the entry point index.d.ts + updateSettings(options: TelemetryOptions): void; +} + // @public export interface ThreadDeletionStatusOutput { deleted: boolean; diff --git a/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingConsole.ts b/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingConsole.ts new file mode 100644 index 000000000000..e12df5fa5fad --- /dev/null +++ b/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingConsole.ts @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Demonstrates How to instrument and get tracing using open telemetry. + * + * @summary Create Agent and instrument using open telemetry. + */ + +import { trace, context } from "@opentelemetry/api"; +import { registerInstrumentations } from "@opentelemetry/instrumentation"; +import { createAzureSdkInstrumentation } from "@azure/opentelemetry-instrumentation-azure-sdk"; +import { AzureMonitorTraceExporter } from "@azure/monitor-opentelemetry-exporter" +import { + ConsoleSpanExporter, + NodeTracerProvider, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-node"; + +import * as dotenv from "dotenv"; +dotenv.config(); + +const provider = new NodeTracerProvider(); +provider.addSpanProcessor(new SimpleSpanProcessor(new ConsoleSpanExporter())); +provider.register(); + +registerInstrumentations({ + instrumentations: [createAzureSdkInstrumentation()], +}); + +import { AIProjectsClient } from "@azure/ai-projects" +import { delay } from "@azure/core-util"; +import { DefaultAzureCredential } from "@azure/identity"; + +const connectionString = process.env["AZURE_AI_PROJECTS_CONNECTION_STRING"] || ">;;;"; +let appInsightsConnectionString = process.env["APPLICATIONINSIGHTS_CONNECTION_STRING"] + +export async function main(): Promise { + + const tracer = trace.getTracer("Agents Sample", "1.0.0"); + + const client = AIProjectsClient.fromConnectionString(connectionString || "", new DefaultAzureCredential()); + + if (!appInsightsConnectionString) { + appInsightsConnectionString = await client.telemetry.getConnectionString(); + } + + if (appInsightsConnectionString) { + const exporter = new AzureMonitorTraceExporter({ connectionString: appInsightsConnectionString }); + provider.addSpanProcessor(new SimpleSpanProcessor(exporter)); + } + + await tracer.startActiveSpan("main", async (span) => { + + client.telemetry.updateSettings({enableContentRecording: true}) + + const agent = await client.agents.createAgent("gpt-4o", { name: "my-agent", instructions: "You are helpful agent" }, { tracingOptions: { tracingContext: context.active() } }); + + + console.log(`Created agent, agent ID : ${agent.id}`); + + const thread = await client.agents.createThread(); + console.log(`Created Thread, thread ID: ${thread.id}`); + + // Create message + const message = await client.agents.createMessage(thread.id, { role: "user", content: "Hello, tell me a joke" }) + console.log(`Created message, message ID ${message.id}`); + + // Create run + let run = await client.agents.createRun(thread.id, agent.id); + console.log(`Created Run, Run ID: ${run.id}`); + + while (["queued", "in_progress", "requires_action"].includes(run.status)) { + await delay(1000); + run = await client.agents.getRun(thread.id, run.id); + console.log(`Current Run status - ${run.status}, run ID: ${run.id}`); + } + + await client.agents.deleteAgent(agent.id); + + console.log(`Deleted agent`); + + await client.agents.listMessages(thread.id) + + span.end(); + }); +} + +main().catch((err) => { + console.error("The sample encountered an error:", err); +}); diff --git a/sdk/ai/ai-projects/src/agents/assistants.ts b/sdk/ai/ai-projects/src/agents/assistants.ts index e27054cd026e..3c833e4ddda5 100644 --- a/sdk/ai/ai-projects/src/agents/assistants.ts +++ b/sdk/ai/ai-projects/src/agents/assistants.ts @@ -5,7 +5,9 @@ import { Client, createRestError } from "@azure-rest/core-client"; import { AgentDeletionStatusOutput, AgentOutput, OpenAIPageableListOfAgentOutput } from "../generated/src/outputModels.js"; import { CreateAgentParameters, DeleteAgentParameters, GetAgentParameters, ListAgentsParameters, UpdateAgentParameters } from "../generated/src/parameters.js"; import { validateLimit, validateMetadata, validateOrder, validateVectorStoreDataType } from "./inputValidations.js"; - +import { TracingUtility } from "../tracing.js"; +import { traceEndCreateOrUpdateAgent, traceStartCreateOrUpdateAgent } from "./assistantsTrace.js"; +import { traceEndAgentGeneric, traceStartAgentGeneric } from "./traceUtility.js"; const expectedStatuses = ["200"]; @@ -21,80 +23,93 @@ enum Tools { /** Creates a new agent. */ export async function createAgent( - context: Client, - options: CreateAgentParameters, - ): Promise { - validateCreateAgentParameters(options); - const result = await context.path("/assistants").post(options); - if (!expectedStatuses.includes(result.status)) { + context: Client, + options: CreateAgentParameters, +): Promise { + validateCreateAgentParameters(options); + return TracingUtility.withSpan("CreateAgent", options, + async (updatedOptions) => { + const result = await context.path("/assistants").post(updatedOptions); + if (!expectedStatuses.includes(result.status)) { throw createRestError(result); - } - return result.body; - } + } + return result.body; + }, + traceStartCreateOrUpdateAgent, traceEndCreateOrUpdateAgent, + ); +} - /** Gets a list of agents that were previously created. */ - export async function listAgents( - context: Client, - options?: ListAgentsParameters, - ): Promise { - validateListAgentsParameters(options); - const result = await context - .path("/assistants") - .get(options); +/** Gets a list of agents that were previously created. */ +export async function listAgents( + context: Client, + options: ListAgentsParameters = {}, +): Promise { + validateListAgentsParameters(options); + return TracingUtility.withSpan("ListAgents", options || {}, async (updateOptions) => { + const result = await context + .path("/assistants") + .get(updateOptions); if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); + throw createRestError(result); } - return result.body; - } + return result.body; + }); +} - /** Retrieves an existing agent. */ +/** Retrieves an existing agent. */ export async function getAgent( - context: Client, - assistantId: string, - options?: GetAgentParameters, - ): Promise { - validateAssistantId(assistantId); + context: Client, + assistantId: string, + options: GetAgentParameters = {}, +): Promise { + validateAssistantId(assistantId); + return TracingUtility.withSpan("GetAgent", options || {}, async (updateOptions) => { const result = await context - .path("/assistants/{assistantId}", assistantId) - .get(options); + .path("/assistants/{assistantId}", assistantId) + .get(updateOptions); if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); + throw createRestError(result); } - return result.body; - } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { agentId: assistantId } })); +} - /** Modifies an existing agent. */ +/** Modifies an existing agent. */ export async function updateAgent( - context: Client, - assistantId: string, - options?: UpdateAgentParameters, - ): Promise { - validateUpdateAgentParameters(assistantId, options); + context: Client, + assistantId: string, + options: UpdateAgentParameters = { body: {} }, +): Promise { + validateUpdateAgentParameters(assistantId, options); + return TracingUtility.withSpan("UpdateAgent", options, async (updateOptions) => { const result = await context - .path("/assistants/{assistantId}", assistantId) - .post(options); + .path("/assistants/{assistantId}", assistantId) + .post(updateOptions); if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); + throw createRestError(result); } return result.body; - } + }, (span, updatedOptions) => traceStartCreateOrUpdateAgent(span, updatedOptions, assistantId), traceEndCreateOrUpdateAgent,); +} - /** Deletes an agent. */ +/** Deletes an agent. */ export async function deleteAgent( - context: Client, - assistantId: string, - options?: DeleteAgentParameters, - ): Promise { - validateAssistantId(assistantId); + context: Client, + assistantId: string, + options: DeleteAgentParameters = {}, +): Promise { + validateAssistantId(assistantId); + return TracingUtility.withSpan("DeleteAgent", options, async (updateOptions) => { const result = await context - .path("/assistants/{assistantId}", assistantId) - .delete(options); + .path("/assistants/{assistantId}", assistantId) + .delete(updateOptions); if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); + throw createRestError(result); } return result.body; - } + }, traceStartAgentGeneric, traceEndAgentGeneric); +} function validateCreateAgentParameters(options: CreateAgentParameters | UpdateAgentParameters): void { if (options.body.tools) { @@ -119,10 +134,10 @@ function validateCreateAgentParameters(options: CreateAgentParameters | UpdateAg throw new Error("Only one vector store ID is allowed"); } if (options.body.tool_resources.file_search.vector_stores) { - if (options.body.tool_resources.file_search.vector_stores.length > 1) { - throw new Error("Only one vector store is allowed"); - } - validateVectorStoreDataType(options.body.tool_resources.file_search.vector_stores[0]?.configuration.data_sources); + if (options.body.tool_resources.file_search.vector_stores.length > 1) { + throw new Error("Only one vector store is allowed"); + } + validateVectorStoreDataType(options.body.tool_resources.file_search.vector_stores[0]?.configuration.data_sources); } } if (options.body.tool_resources.azure_ai_search) { @@ -132,7 +147,7 @@ function validateCreateAgentParameters(options: CreateAgentParameters | UpdateAg } } if (options.body.temperature && (options.body.temperature < 0 || options.body.temperature > 2)) { - throw new Error("Temperature must be between 0 and 2"); + throw new Error("Temperature must be between 0 and 2"); } if (options.body.metadata) { validateMetadata(options.body.metadata); @@ -150,7 +165,7 @@ function validateListAgentsParameters(options?: ListAgentsParameters): void { function validateAssistantId(assistantId: string): void { if (!assistantId) { - throw new Error("Assistant ID is required"); + throw new Error("Assistant ID is required"); } } diff --git a/sdk/ai/ai-projects/src/agents/assistantsTrace.ts b/sdk/ai/ai-projects/src/agents/assistantsTrace.ts new file mode 100644 index 000000000000..111e03ac5655 --- /dev/null +++ b/sdk/ai/ai-projects/src/agents/assistantsTrace.ts @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { AgentOutput } from "../generated/src/outputModels.js"; +import { TracingAttributeOptions, TracingUtility, TracingOperationName, Span } from "../tracing.js"; +import { CreateAgentParameters, UpdateAgentParameters } from "../generated/src/parameters.js"; +import { addInstructionsEvent, formatAgentApiResponse, UpdateWithAgentAttributes } from "./traceUtility.js"; + +/** + * Traces the start of creating or updating an agent. + * @param span - The span to trace. + * @param options - The options for creating an agent. + */ +export function traceStartCreateOrUpdateAgent(span: Span, options: CreateAgentParameters | UpdateAgentParameters, agentId?:string): void { + const attributes: TracingAttributeOptions = { + operationName: TracingOperationName.CREATE_AGENT, + name: options.body.name ?? undefined, + model: options.body.model, + description: options.body.description ?? undefined, + instructions: options.body.instructions ?? undefined, + topP: options.body.top_p ?? undefined, + temperature: options.body.temperature ?? undefined, + responseFormat: formatAgentApiResponse(options.body.response_format), + }; + if(agentId){ + attributes.operationName = TracingOperationName.CREATE_UPDATE_AGENT + attributes.agentId = agentId + } + TracingUtility.setSpanAttributes(span, TracingOperationName.CREATE_AGENT, UpdateWithAgentAttributes(attributes)) + addInstructionsEvent(span, options.body); +} + + +/** + * Traces the end of creating an agent. + * @param span - The span to trace. + * @param _options - The options for creating an agent. + * @param result - The result of creating an agent. + */ +export async function traceEndCreateOrUpdateAgent(span: Span, _options: CreateAgentParameters | UpdateAgentParameters, result: Promise): Promise { + const resolvedResult = await result; + const attributes: TracingAttributeOptions = { + agentId: resolvedResult.id, + }; + TracingUtility.updateSpanAttributes(span, attributes); +} diff --git a/sdk/ai/ai-projects/src/agents/messages.ts b/sdk/ai/ai-projects/src/agents/messages.ts index c42cdd491bb0..e32e1d0b82c8 100644 --- a/sdk/ai/ai-projects/src/agents/messages.ts +++ b/sdk/ai/ai-projects/src/agents/messages.ts @@ -5,6 +5,9 @@ import { Client, createRestError } from "@azure-rest/core-client"; import { OpenAIPageableListOfThreadMessageOutput, ThreadMessageOutput } from "../generated/src/outputModels.js"; import { CreateMessageParameters, ListMessagesParameters, UpdateMessageParameters } from "../generated/src/parameters.js"; import { validateMetadata, validateVectorStoreDataType } from "./inputValidations.js"; +import { TracingUtility } from "../tracing.js"; +import { traceEndCreateMessage, traceEndListMessages, traceStartCreateMessage, traceStartListMessages } from "./messagesTrace.js"; +import { traceStartAgentGeneric } from "./traceUtility.js"; const expectedStatuses = ["200"]; @@ -16,30 +19,34 @@ export async function createMessage( ): Promise { validateThreadId(threadId); validateCreateMessageParameters(options); - const result = await context - .path("/threads/{threadId}/messages", threadId) - .post(options); - if (!expectedStatuses.includes(result.status)) { + return TracingUtility.withSpan("CreateMessage", options, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/messages", threadId) + .post(updateOptions); + if (!expectedStatuses.includes(result.status)) { throw createRestError(result); - } - return result.body; + } + return result.body; + }, (span, updatedOptions) => traceStartCreateMessage(span, threadId, updatedOptions), traceEndCreateMessage); } /** Gets a list of messages that exist on a thread. */ export async function listMessages( context: Client, threadId: string, - options?: ListMessagesParameters, + options: ListMessagesParameters = {}, ): Promise { validateThreadId(threadId); validateListMessagesParameters(options); - const result = await context - .path("/threads/{threadId}/messages", threadId) - .get(options); - if (!expectedStatuses.includes(result.status)) { + return TracingUtility.withSpan("ListMessages", options, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/messages", threadId) + .get(updateOptions); + if (!expectedStatuses.includes(result.status)) { throw createRestError(result); - } - return result.body; + } + return result.body; + }, (span, updatedOptions) => traceStartListMessages(span, threadId, updatedOptions), traceEndListMessages); } /** Modifies an existing message on an existing thread. */ @@ -47,17 +54,19 @@ export async function updateMessage( context: Client, threadId: string, messageId: string, - options?: UpdateMessageParameters, + options: UpdateMessageParameters = { body: {} }, ): Promise { validateThreadId(threadId); validateMessageId(messageId); - const result = await context - .path("/threads/{threadId}/messages/{messageId}", threadId, messageId) - .post(options); - if (!expectedStatuses.includes(result.status)) { + return TracingUtility.withSpan("UpdateMessage", options, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/messages/{messageId}", threadId, messageId) + .post(updateOptions); + if (!expectedStatuses.includes(result.status)) { throw createRestError(result); - } - return result.body; + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { threadId: threadId, messageId: messageId } })); } function validateThreadId(threadId: string): void { diff --git a/sdk/ai/ai-projects/src/agents/messagesTrace.ts b/sdk/ai/ai-projects/src/agents/messagesTrace.ts new file mode 100644 index 000000000000..451b89727150 --- /dev/null +++ b/sdk/ai/ai-projects/src/agents/messagesTrace.ts @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { CreateMessageParameters, ListMessagesParameters } from "../generated/src/parameters.js"; +import { TracingAttributes, TracingUtility, TracingOperationName, Span } from "../tracing.js"; +import { OpenAIPageableListOfThreadMessageOutput, ThreadMessageOutput } from "../generated/src/outputModels.js"; +import { addMessageEvent } from "./traceUtility.js"; + +export function traceStartCreateMessage(span: Span, threadId: string, options: CreateMessageParameters): void { + TracingUtility.setSpanAttributes(span, TracingOperationName.CREATE_MESSAGE, { threadId: threadId, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }); + addMessageEvent(span, { ...options.body, thread_id: threadId }); +} + +export async function traceEndCreateMessage(span: Span, _options: CreateMessageParameters, result: Promise): Promise { + const resolvedResult = await result; + TracingUtility.updateSpanAttributes(span, { messageId: resolvedResult.id }); +} + +export function traceStartListMessages(span: Span, threadId: string, _options: ListMessagesParameters) : void { + TracingUtility.setSpanAttributes(span, TracingOperationName.LIST_MESSAGES, { threadId: threadId, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }); +} + +export async function traceEndListMessages(span: Span, _options: ListMessagesParameters, result: Promise) : Promise { + const resolvedResult = await result; + resolvedResult.data?.forEach(message => { + addMessageEvent(span, message) + }); +} diff --git a/sdk/ai/ai-projects/src/agents/runTrace.ts b/sdk/ai/ai-projects/src/agents/runTrace.ts new file mode 100644 index 000000000000..953257c108d9 --- /dev/null +++ b/sdk/ai/ai-projects/src/agents/runTrace.ts @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { CreateRunParameters, CreateThreadAndRunParameters, SubmitToolOutputsToRunParameters, UpdateRunParameters } from "../generated/src/parameters.js"; +import { ThreadRunOutput } from "../generated/src/outputModels.js"; +import { TracingAttributeOptions, TracingUtility, TracingOperationName, Span } from "../tracing.js"; +import { addInstructionsEvent, addMessageEvent, addToolMessagesEvent, formatAgentApiResponse, UpdateWithAgentAttributes } from "./traceUtility.js"; + +export function traceStartCreateRun(span: Span, options: CreateRunParameters | CreateThreadAndRunParameters, threadId?: string, operationName: string = TracingOperationName.CREATE_RUN): void { + + const attributes: TracingAttributeOptions = { + threadId: threadId, + agentId: options.body.assistant_id, + model: options.body.model ?? undefined, + instructions: options.body.instructions ?? undefined, + temperature: options.body.temperature ?? undefined, + topP: options.body.top_p ?? undefined, + maxCompletionTokens: options.body.max_completion_tokens ?? undefined, + maxPromptTokens: options.body.max_prompt_tokens ?? undefined, + responseFormat: formatAgentApiResponse(options.body.response_format), + } + if ((options as CreateRunParameters).body.additional_instructions) { + attributes.additional_instructions = (options as CreateRunParameters).body.additional_instructions ?? undefined; + } + TracingUtility.setSpanAttributes(span, operationName, UpdateWithAgentAttributes(attributes)); + setSpanEvents(span, options); +} + +export function traceStartCreateThreadAndRun(span: Span, options: CreateThreadAndRunParameters): void { + traceStartCreateRun(span, options, undefined, TracingOperationName.CREATE_THREAD_RUN); +} + +export async function traceEndCreateOrUpdateRun(span: Span, _options: CreateRunParameters | UpdateRunParameters, result: Promise): Promise { + const resolvedResult = await result; + TracingUtility.updateSpanAttributes(span, { runId: resolvedResult.id, runStatus: resolvedResult.status, responseModel: resolvedResult.model, usageCompletionTokens: resolvedResult.usage?.completion_tokens, usagePromptTokens: resolvedResult.usage?.prompt_tokens }); +} + +export function traceStartSubmitToolOutputsToRun(span: Span, options: SubmitToolOutputsToRunParameters, threadId: string, + runId: string,): void { + const attributes: TracingAttributeOptions = { threadId: threadId, runId: runId } + TracingUtility.setSpanAttributes(span, TracingOperationName.SUBMIT_TOOL_OUTPUTS, UpdateWithAgentAttributes(attributes)); + addToolMessagesEvent(span, options.body.tool_outputs); +} + +export async function traceEndSubmitToolOutputsToRun(span: Span, _options: SubmitToolOutputsToRunParameters, result: Promise): Promise { + const resolvedResult = await result; + TracingUtility.updateSpanAttributes(span, { runId: resolvedResult.id, runStatus: resolvedResult.status, responseModel: resolvedResult.model, usageCompletionTokens: resolvedResult.usage?.completion_tokens, usagePromptTokens: resolvedResult.usage?.prompt_tokens }); +} + +function setSpanEvents(span: Span, options: CreateRunParameters): void { + addInstructionsEvent(span, { ...options.body, agentId: options.body.assistant_id }); + options.body.additional_messages?.forEach((message) => { + addMessageEvent(span, message); + }); +} + diff --git a/sdk/ai/ai-projects/src/agents/runs.ts b/sdk/ai/ai-projects/src/agents/runs.ts index 1efb61552479..45997a69f411 100644 --- a/sdk/ai/ai-projects/src/agents/runs.ts +++ b/sdk/ai/ai-projects/src/agents/runs.ts @@ -5,6 +5,9 @@ import { Client, createRestError } from "@azure-rest/core-client"; import { CancelRunParameters, CreateRunParameters, CreateThreadAndRunParameters, GetRunParameters, ListRunsParameters, SubmitToolOutputsToRunParameters, UpdateRunParameters } from "../generated/src/parameters.js"; import { OpenAIPageableListOfThreadRunOutput, ThreadRunOutput } from "../generated/src/outputModels.js"; import { validateLimit, validateMessages, validateMetadata, validateOrder, validateRunId, validateThreadId, validateTools, validateTruncationStrategy } from "./inputValidations.js"; +import { TracingUtility } from "../tracing.js"; +import { traceEndCreateOrUpdateRun, traceEndSubmitToolOutputsToRun, traceStartCreateRun, traceStartCreateThreadAndRun, traceStartSubmitToolOutputsToRun } from "./runTrace.js"; +import { traceStartAgentGeneric } from "./traceUtility.js"; const expectedStatuses = ["200"]; @@ -17,13 +20,15 @@ export async function createRun( validateThreadId(threadId); validateCreateRunParameters(options); options.body.stream = false; - const result = await context - .path("/threads/{threadId}/runs", threadId) - .post(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("CreateRun", options, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/runs", threadId) + .post(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartCreateRun(span, updatedOptions, threadId), traceEndCreateOrUpdateRun); } /** Gets a list of runs for a specified thread. */ @@ -33,13 +38,15 @@ export async function listRuns( options?: ListRunsParameters, ): Promise { validateListRunsParameters(threadId, options); - const result = await context - .path("/threads/{threadId}/runs", threadId) - .get(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("ListRuns", options || {}, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/runs", threadId) + .get(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { threadId: threadId } })); } /** Gets an existing run from an existing thread. */ @@ -51,13 +58,15 @@ export async function getRun( ): Promise { validateThreadId(threadId); validateRunId(runId); - const result = await context - .path("/threads/{threadId}/runs/{runId}", threadId, runId) - .get(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("GetRun", options || {}, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/runs/{runId}", threadId, runId) + .get(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { threadId: threadId, runId: runId } })); } /** Modifies an existing thread run. */ @@ -68,13 +77,15 @@ export async function updateRun( options?: UpdateRunParameters, ): Promise { validateUpdateRunParameters(threadId, runId, options); - const result = await context - .path("/threads/{threadId}/runs/{runId}", threadId, runId) - .post(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("UpdateRun", options || { body: {} }, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/runs/{runId}", threadId, runId) + .post(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { threadId: threadId, runId: runId } }), traceEndCreateOrUpdateRun); } /** Submits outputs from tools as requested by tool calls in a run. Runs that need submitted tool outputs will have a status of 'requires_action' with a required_action.type of 'submit_tool_outputs'. */ @@ -87,13 +98,15 @@ export async function submitToolOutputsToRun( validateThreadId(threadId); validateRunId(runId); options.body.stream = false; - const result = await context - .path("/threads/{threadId}/runs/{runId}/submit_tool_outputs", threadId, runId) - .post(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("SubmitToolOutputsToRun", options, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/runs/{runId}/submit_tool_outputs", threadId, runId) + .post(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartSubmitToolOutputsToRun(span, updatedOptions, threadId, runId), traceEndSubmitToolOutputsToRun); } /** Cancels a run of an in progress thread. */ @@ -105,13 +118,15 @@ export async function cancelRun( ): Promise { validateThreadId(threadId); validateRunId(runId); - const result = await context - .path("/threads/{threadId}/runs/{runId}/cancel", threadId, runId) - .post(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("CancelRun", options || {}, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/runs/{runId}/cancel", threadId, runId) + .post(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }); } /** Creates a new thread and immediately starts a run of that thread. */ @@ -121,17 +136,19 @@ export async function createThreadAndRun( ): Promise { validateCreateThreadAndRunParameters(options); options.body.stream = false; - const result = await context.path("/threads/runs").post(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("CreateThreadAndRun", options, async (updateOptions) => { + const result = await context.path("/threads/runs").post(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, traceStartCreateThreadAndRun, traceEndCreateOrUpdateRun); } function validateListRunsParameters(thread_id: string, options?: ListRunsParameters): void { validateThreadId(thread_id); if (options?.queryParameters?.limit && (options.queryParameters.limit < 1 || options.queryParameters.limit > 100)) { - throw new Error("Limit must be between 1 and 100"); + throw new Error("Limit must be between 1 and 100"); } if (options?.queryParameters?.limit) { validateLimit(options.queryParameters.limit); @@ -144,12 +161,12 @@ function validateListRunsParameters(thread_id: string, options?: ListRunsParamet function validateUpdateRunParameters(thread_id: string, run_id: string, options?: UpdateRunParameters): void { validateThreadId(thread_id); validateRunId(run_id); - if(options?.body.metadata){ + if (options?.body.metadata) { validateMetadata(options.body.metadata); } } -function validateCreateRunParameters(options: CreateRunParameters| CreateThreadAndRunParameters): void { +function validateCreateRunParameters(options: CreateRunParameters | CreateThreadAndRunParameters): void { if ('additional_messages' in options.body && options.body.additional_messages) { options.body.additional_messages.forEach(message => validateMessages(message.role)); } diff --git a/sdk/ai/ai-projects/src/agents/threads.ts b/sdk/ai/ai-projects/src/agents/threads.ts index 9e86dc519d73..de9a8edef529 100644 --- a/sdk/ai/ai-projects/src/agents/threads.ts +++ b/sdk/ai/ai-projects/src/agents/threads.ts @@ -4,69 +4,80 @@ import { Client, createRestError } from "@azure-rest/core-client"; import { CreateThreadParameters, DeleteThreadParameters, GetThreadParameters, UpdateThreadParameters } from "../generated/src/parameters.js"; import { AgentThreadOutput, ThreadDeletionStatusOutput } from "../generated/src/outputModels.js"; +import { TracingUtility } from "../tracing.js"; +import { traceEndCreateThread, traceStartCreateThread } from "./threadsTrace.js"; import { validateMessages, validateMetadata, validateThreadId, validateToolResources } from "./inputValidations.js"; +import { traceStartAgentGeneric } from "./traceUtility.js"; const expectedStatuses = ["200"]; /** Creates a new thread. Threads contain messages and can be run by agents. */ export async function createThread( context: Client, - options?: CreateThreadParameters, + options: CreateThreadParameters = { body: {} }, ): Promise { validateCreateThreadParameters(options); - const result = await context.path("/threads").post(options); - if (!expectedStatuses.includes(result.status)) { + return TracingUtility.withSpan("CreateThread", options, async (updatedOptions) => { + const result = await context.path("/threads").post(updatedOptions); + if (!expectedStatuses.includes(result.status)) { throw createRestError(result); - } - return result.body; + } + return result.body; + }, traceStartCreateThread, traceEndCreateThread); } /** Gets information about an existing thread. */ export async function getThread( context: Client, threadId: string, - options?: GetThreadParameters, + options: GetThreadParameters = {}, ): Promise { validateThreadId(threadId); - const result = await context - .path("/threads/{threadId}", threadId) - .get(options); - if (!expectedStatuses.includes(result.status)) { + return TracingUtility.withSpan("GetThread", options, async (updatedOptions) => { + const result = await context + .path("/threads/{threadId}", threadId) + .get(updatedOptions); + if (!expectedStatuses.includes(result.status)) { throw createRestError(result); - } - return result.body; + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { threadId: threadId } })); } /** Modifies an existing thread. */ export async function updateThread( context: Client, threadId: string, - options?: UpdateThreadParameters, + options: UpdateThreadParameters = { body: {} }, ): Promise { validateUpdateThreadParameters(threadId, options); - const result = await context - .path("/threads/{threadId}", threadId) - .post(options); - if (!expectedStatuses.includes(result.status)) { + return TracingUtility.withSpan("UpdateThread", options, async (updatedOptions) => { + const result = await context + .path("/threads/{threadId}", threadId) + .post(updatedOptions); + if (!expectedStatuses.includes(result.status)) { throw createRestError(result); - } - return result.body; + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { threadId: threadId } })); } /** Deletes an existing thread. */ export async function deleteThread( context: Client, threadId: string, - options?: DeleteThreadParameters, + options: DeleteThreadParameters = {}, ): Promise { validateThreadId(threadId); - const result = await context - .path("/threads/{threadId}", threadId) - .delete(options); - if (!expectedStatuses.includes(result.status)) { + return TracingUtility.withSpan("DeleteThread", options, async (updatedOptions) => { + const result = await context + .path("/threads/{threadId}", threadId) + .delete(updatedOptions); + if (!expectedStatuses.includes(result.status)) { throw createRestError(result); - } - return result.body; + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { threadId: threadId } })); } @@ -75,9 +86,9 @@ function validateCreateThreadParameters(options?: CreateThreadParameters): void options.body.messages.forEach(message => validateMessages(message.role)); } if (options?.body.tool_resources) { - validateToolResources(options.body.tool_resources); + validateToolResources(options.body.tool_resources); } - if (options?.body.metadata){ + if (options?.body.metadata) { validateMetadata(options.body.metadata); } } @@ -85,9 +96,9 @@ function validateCreateThreadParameters(options?: CreateThreadParameters): void function validateUpdateThreadParameters(threadId: string, options?: UpdateThreadParameters): void { validateThreadId(threadId); if (options?.body.tool_resources) { - validateToolResources(options.body.tool_resources); + validateToolResources(options.body.tool_resources); } - if (options?.body.metadata){ + if (options?.body.metadata) { validateMetadata(options.body.metadata); } } diff --git a/sdk/ai/ai-projects/src/agents/threadsTrace.ts b/sdk/ai/ai-projects/src/agents/threadsTrace.ts new file mode 100644 index 000000000000..859bb8bcc3b2 --- /dev/null +++ b/sdk/ai/ai-projects/src/agents/threadsTrace.ts @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { AgentThreadOutput } from "../generated/src/outputModels.js"; +import { TracingUtility, TracingOperationName, Span } from "../tracing.js"; +import { CreateThreadParameters } from "../generated/src/parameters.js"; +import { addMessageEvent, UpdateWithAgentAttributes } from "./traceUtility.js"; + +export function traceStartCreateThread(span: Span, options: CreateThreadParameters): void { + TracingUtility.setSpanAttributes(span, TracingOperationName.CREATE_THREAD, UpdateWithAgentAttributes({})); + setSpanEvents(span, options); +} + +export async function traceEndCreateThread(span: Span, _options: CreateThreadParameters, result: Promise): Promise { + const resolvedResult = await result; + TracingUtility.updateSpanAttributes(span, { threadId: resolvedResult.id }); +} + +function setSpanEvents(span: Span, options: CreateThreadParameters): void { + options.body.messages?.forEach((message) => { + addMessageEvent(span, message); + }); +} diff --git a/sdk/ai/ai-projects/src/agents/traceUtility.ts b/sdk/ai/ai-projects/src/agents/traceUtility.ts new file mode 100644 index 000000000000..75655514c55a --- /dev/null +++ b/sdk/ai/ai-projects/src/agents/traceUtility.ts @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { AgentsApiResponseFormat, AgentsApiResponseFormatOption, MessageContent, RunStepCompletionUsageOutput, ThreadMessage, ThreadMessageOptions, ToolOutput } from "./inputOutputs.js"; +import { OptionsWithTracing, Span, TracingAttributeOptions, TracingAttributes, TracingUtility } from "../tracing.js"; +import { getTelemetryOptions } from "../telemetry/telemetry.js"; + +export function traceStartAgentGeneric (span: Span, options: Options): void { + const attributeOptions = options.tracingAttributeOptions || {} ; + TracingUtility.setSpanAttributes(span, options.tracingAttributeOptions?.operationName || "Agent_Operation" , UpdateWithAgentAttributes(attributeOptions)) +} +export function traceEndAgentGeneric (span: Span, _options: Options): void { + const attributeOptions = {}; + TracingUtility.updateSpanAttributes(span, UpdateWithAgentAttributes(attributeOptions)) +} + +export function UpdateWithAgentAttributes(attributeOptions: Omit): Omit { + attributeOptions.genAiSystem = TracingAttributes.AZ_AI_AGENT_SYSTEM + return attributeOptions; +} + +/** + * Adds a message event to the span. + * @param span - The span to add the event to. + * @param messageAttributes - The attributes of the message event. + */ +export function addMessageEvent(span: Span, messageAttributes: ThreadMessageOptions | ThreadMessage, usage?: RunStepCompletionUsageOutput): void { + + const eventBody: Record = {}; + const telemetryOptions = getTelemetryOptions() + if (telemetryOptions.enableContentRecording) { + eventBody.content = getMessageContent(messageAttributes.content); + } + eventBody.role = messageAttributes.role; + if (messageAttributes.attachments) { + eventBody.attachments = messageAttributes.attachments.map((attachment) => { + return { + id: attachment.file_id, + tools: attachment.tools.map((tool) => tool.type) + }; + }); + } + const threadId = (messageAttributes as ThreadMessage).thread_id; + const agentId = (messageAttributes as ThreadMessage).assistant_id ?? undefined; + const threadRunId = (messageAttributes as ThreadMessage).run_id; + const messageStatus = (messageAttributes as ThreadMessage).status; + const messageId = (messageAttributes as ThreadMessage).id; + const incompleteDetails = (messageAttributes as ThreadMessage).incomplete_details + if (incompleteDetails) { + eventBody.incomplete_details = incompleteDetails; + } + const usagePromptTokens = usage?.prompt_tokens; + const usageCompletionTokens = usage?.completion_tokens; + const attributes = { eventContent: JSON.stringify(eventBody), threadId, agentId, threadRunId, messageStatus, messageId, usagePromptTokens, usageCompletionTokens, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }; + TracingUtility.addSpanEvent(span, `gen_ai.${messageAttributes.role}.message`, attributes); +} + +/** + * Adds an instruction event to the span. + * @param span - The span to add the event to. + * @param instructionAttributes - The attributes of the instruction event. + */ +export function addInstructionsEvent(span: Span, instructionAttributes: { instructions?: string | null, additional_instructions?: string | null, threadId?: string, agentId?: string }): void { + const eventBody: Record = {}; + if (instructionAttributes.instructions || instructionAttributes.additional_instructions) { + eventBody.content = instructionAttributes.instructions && instructionAttributes.additional_instructions ? `${instructionAttributes.instructions} ${instructionAttributes.additional_instructions}` : instructionAttributes.instructions || instructionAttributes.additional_instructions; + } + const attributes = { eventContent: JSON.stringify(eventBody), threadId: instructionAttributes.threadId, agentId: instructionAttributes.agentId, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }; + TracingUtility.addSpanEvent(span, "gen_ai.system.message", attributes); + +} + +/** + * Formats the agent API response. + * @param responseFormat - The response format option. + * @returns The formatted response as a string, or null/undefined. + */ +export function formatAgentApiResponse(responseFormat: AgentsApiResponseFormatOption | null | undefined): string | undefined { + if (typeof responseFormat === "string" || responseFormat === undefined || responseFormat === null) { + return responseFormat ?? undefined; + } + if ((responseFormat as AgentsApiResponseFormat).type) { + return (responseFormat as AgentsApiResponseFormat).type ?? undefined; + } + return undefined; +} + +/** + * Adds a tool messages event to the span + * @param span - The span to add the event to. + * @param tool_outputs - List of tool oupts + */ +export function addToolMessagesEvent(span: Span, tool_outputs: Array): void { + tool_outputs.forEach(tool_output => { + const eventBody = { "content": tool_output.output, "id": tool_output.tool_call_id } + TracingUtility.addSpanEvent(span, "gen_ai.tool.message", { eventContent: JSON.stringify(eventBody), genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }); + }); + +} + +function getMessageContent(messageContent: string | MessageContent[]): string | {} { + type MessageContentExtended = MessageContent & { [key: string]: any; }; + if (!Array.isArray(messageContent)) { + return messageContent; + } + const contentBody: { [key: string]: any } = {}; + messageContent.forEach(content => { + const typedContent = content.type; + const { value, annotations } = (content as MessageContentExtended)[typedContent]; + contentBody[typedContent] = { value, annotations }; + }); + return contentBody; +} diff --git a/sdk/ai/ai-projects/src/aiProjectsClient.ts b/sdk/ai/ai-projects/src/aiProjectsClient.ts index 1c0492ff4347..b1edc5576199 100644 --- a/sdk/ai/ai-projects/src/aiProjectsClient.ts +++ b/sdk/ai/ai-projects/src/aiProjectsClient.ts @@ -2,16 +2,18 @@ // Licensed under the MIT License. import { TokenCredential } from "@azure/core-auth"; import createClient, { ProjectsClientOptions } from "./generated/src/projectsClient.js"; -import { Client } from "@azure-rest/core-client"; import { AgentsOperations, getAgentsOperations } from "./agents/index.js"; import { ConnectionsOperations, getConnectionsOperations } from "./connections/index.js"; +import { getTelemetryOperations, TelemetryOperations } from "./telemetry/index.js"; +import { Client } from "@azure-rest/core-client"; -export interface AIProjectsClientOptions extends ProjectsClientOptions{ +export interface AIProjectsClientOptions extends ProjectsClientOptions { } export class AIProjectsClient { private _client: Client; private _connectionClient: Client; + private _telemetryClient: Client; /* * @param endpointParam - The Azure AI Studio project endpoint, in the form `https://.api.azureml.ms` or `https://..api.azureml.ms`, where is the Azure region where the project is deployed (e.g. westus) and is the GUID of the Enterprise private link. @@ -37,13 +39,23 @@ export class AIProjectsClient { credential, options, ); + this._connectionClient = createClient(endpointParam, subscriptionId, resourceGroupName, projectName, credential, - {...options, endpoint:connectionEndPoint}) + { ...options, endpoint: connectionEndPoint }) + + this._telemetryClient = createClient(endpointParam, subscriptionId, + resourceGroupName, + projectName, + credential, + { ...options, apiVersion: "2020-02-02", endpoint: 'https://management.azure.com' }) + this.agents = getAgentsOperations(this._client); this.connections = getConnectionsOperations(this._connectionClient); + this.telemetry = getTelemetryOperations(this._telemetryClient, this.connections); + } /** @@ -90,4 +102,7 @@ export class AIProjectsClient { /** The operation groups for connections */ public readonly connections: ConnectionsOperations; + + /** The operation groups for telemetry */ + public readonly telemetry: TelemetryOperations; } diff --git a/sdk/ai/ai-projects/src/constants.ts b/sdk/ai/ai-projects/src/constants.ts new file mode 100644 index 000000000000..fe3152c1805e --- /dev/null +++ b/sdk/ai/ai-projects/src/constants.ts @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Current version of the `@azure/ai-projects` package. + */ +export const SDK_VERSION = `1.0.0-beta.1`; + +/** + * The package name of the `@azure/ai-projects` package. + */ +export const PACKAGE_NAME = "@azure/ai-projects"; diff --git a/sdk/ai/ai-projects/src/index.ts b/sdk/ai/ai-projects/src/index.ts index 72bd951fe365..0d5b48cbd826 100644 --- a/sdk/ai/ai-projects/src/index.ts +++ b/sdk/ai/ai-projects/src/index.ts @@ -5,6 +5,7 @@ import { AIProjectsClient, AIProjectsClientOptions } from "./aiProjectsClient.js import { ProjectsClientOptions } from "./generated/src/projectsClient.js" export { AgentsOperations } from "./agents/index.js"; export { ConnectionsOperations } from "./connections/index.js"; +export { TelemetryOperations } from "./telemetry/index.js"; export * from "./agents/inputOutputs.js"; export * from "./connections/inputOutput.js"; diff --git a/sdk/ai/ai-projects/src/logger.ts b/sdk/ai/ai-projects/src/logger.ts new file mode 100644 index 000000000000..9f092c1689f3 --- /dev/null +++ b/sdk/ai/ai-projects/src/logger.ts @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createClientLogger } from "@azure/logger"; +import { PACKAGE_NAME } from "./constants.js"; +export const logger = createClientLogger(PACKAGE_NAME); diff --git a/sdk/ai/ai-projects/src/telemetry/index.ts b/sdk/ai/ai-projects/src/telemetry/index.ts new file mode 100644 index 000000000000..9857b7acfda3 --- /dev/null +++ b/sdk/ai/ai-projects/src/telemetry/index.ts @@ -0,0 +1,46 @@ + +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + + +import { Client } from "@azure-rest/core-client"; +import { ConnectionsOperations } from "../connections/index.js"; +import { getConnectionString, getTelemetryOptions, resetTelemetryOptions, TelemetryOptions, updateTelemetryOptions } from "./telemetry.js"; + +/** + * Telemetry operations + **/ +export interface TelemetryOperations { + /** + * Get the appinsights connection string confired in the workspace + * @returns The telemetry connection string + **/ + getConnectionString(): Promise + + /** + * Update the telemetry settings + * @param options - The telemetry options + * @returns void + * */ + updateSettings(options: TelemetryOptions): void + + /** + * get the telemetry settings + * @returns The telemetry options + * */ + getSettings(): TelemetryOptions +} + +/** + * Get the telemetry operations + * @param connection - The connections operations + * @returns The telemetry operations + **/ +export function getTelemetryOperations(context: Client, connection: ConnectionsOperations): TelemetryOperations { + resetTelemetryOptions(); + return { + getConnectionString: () => getConnectionString(context, connection), + updateSettings: (options: TelemetryOptions) => updateTelemetryOptions(options), + getSettings: () => getTelemetryOptions() + } +} diff --git a/sdk/ai/ai-projects/src/telemetry/telemetry.ts b/sdk/ai/ai-projects/src/telemetry/telemetry.ts new file mode 100644 index 000000000000..f5f1e1f25261 --- /dev/null +++ b/sdk/ai/ai-projects/src/telemetry/telemetry.ts @@ -0,0 +1,67 @@ + +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { ConnectionsOperations } from "../connections/index.js"; +import { GetAppInsightsResponseOutput } from "../agents/inputOutputs.js"; +import { Client, createRestError } from "@azure-rest/core-client"; + +const expectedStatuses = ["200"]; + +/** + * Telemetry options + */ +export interface TelemetryOptions { + enableContentRecording: boolean; +} + +const telemetryOptions: TelemetryOptions & { connectionString: string | undefined } = { + enableContentRecording: false, + connectionString: undefined +} + +/** + * Update the telemetry settings + * @param options - The telemetry options + */ +export function updateTelemetryOptions(options: TelemetryOptions): void { + telemetryOptions.enableContentRecording = options.enableContentRecording; +}; + +/** + * Get the telemetry options + * @returns The telemetry options + */ +export function getTelemetryOptions(): TelemetryOptions { + return structuredClone(telemetryOptions); +}; + +/** + * Reset the telemetry options + */ +export function resetTelemetryOptions(): void { + telemetryOptions.connectionString = undefined; + telemetryOptions.enableContentRecording = false; +} + +/** + * Get the appinsights connection string confired in the workspace + * @param connection - get the connection string + * @returns The telemetry connection string + */ +export async function getConnectionString(context: Client, connection: ConnectionsOperations): Promise { + if (!telemetryOptions.connectionString) { + const workspace = await connection.getWorkspace(); + if (workspace.properties.applicationInsights) { + const result = await context.path("/{appInsightsResourceUrl}", workspace.properties.applicationInsights).get({skipUrlEncoding:true}); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + telemetryOptions.connectionString = (result.body as GetAppInsightsResponseOutput).properties.ConnectionString; + } + else { + throw new Error("Application Insights connection string not found.") + } + } + return telemetryOptions.connectionString as string; +} diff --git a/sdk/ai/ai-projects/src/tracing.ts b/sdk/ai/ai-projects/src/tracing.ts new file mode 100644 index 000000000000..7bf17dcbb566 --- /dev/null +++ b/sdk/ai/ai-projects/src/tracing.ts @@ -0,0 +1,291 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createTracingClient, OperationTracingOptions, Resolved, SpanStatusError, TracingSpan } from "@azure/core-tracing"; +import { PACKAGE_NAME, SDK_VERSION } from "./constants.js"; +import { getErrorMessage } from "@azure/core-util"; +import { logger } from "./logger.js"; + +export enum TracingAttributes { + GEN_AI_MESSAGE_ID = "gen_ai.message.id", + GEN_AI_MESSAGE_STATUS = "gen_ai.message.status", + GEN_AI_THREAD_ID = "gen_ai.thread.id", + GEN_AI_THREAD_RUN_ID = "gen_ai.thread.run.id", + GEN_AI_AGENT_ID = "gen_ai.agent.id", + GEN_AI_AGENT_NAME = "gen_ai.agent.name", + GEN_AI_AGENT_DESCRIPTION = "gen_ai.agent.description", + GEN_AI_OPERATION_NAME = "gen_ai.operation.name", + GEN_AI_THREAD_RUN_STATUS = "gen_ai.thread.run.status", + GEN_AI_REQUEST_MODEL = "gen_ai.request.model", + GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature", + GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p", + GEN_AI_REQUEST_MAX_INPUT_TOKENS = "gen_ai.request.max_input_tokens", + GEN_AI_REQUEST_MAX_OUTPUT_TOKENS = "gen_ai.request.max_output_tokens", + GEN_AI_RESPONSE_MODEL = "gen_ai.response.model", + GEN_AI_SYSTEM = "gen_ai.system", + SERVER_ADDRESS = "server.address", + AZ_AI_AGENT_SYSTEM = "az.ai.agents", + GEN_AI_TOOL_NAME = "gen_ai.tool.name", + GEN_AI_TOOL_CALL_ID = "gen_ai.tool.call.id", + GEN_AI_REQUEST_RESPONSE_FORMAT = "gen_ai.request.response_format", + GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens", + GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens", + GEN_AI_SYSTEM_MESSAGE = "gen_ai.system.message", + GEN_AI_EVENT_CONTENT = "gen_ai.event.content", + ERROR_TYPE = "error.type" +} +export enum TracingOperationName { + CREATE_AGENT = "create_agent", + CREATE_UPDATE_AGENT = "update_agent", + CREATE_THREAD = "create_thread", + CREATE_MESSAGE = "create_message", + CREATE_RUN = "create_run", + START_THREAD_RUN = "start_thread_run", + EXECUTE_TOOL = "execute_tool", + LIST_MESSAGES = "list_messages", + SUBMIT_TOOL_OUTPUTS = "submit_tool_outputs", + CREATE_THREAD_RUN = "create_thread_run", +} + +export interface TracingAttributeOptions { + operationName?: string; + name?: string; + description?: string; + serverAddress?: string; + threadId?: string; + agentId?: string; + instructions?: string; + additional_instructions?: string; + runId?: string; + runStatus?: string; + responseModel?: string; + model?: string; + temperature?: number; + topP?: number; + maxPromptTokens?: number; + maxCompletionTokens?: number; + responseFormat?: string; + genAiSystem?: string; + messageId?: string; + messageStatus?: string; + eventContent?: string; + usagePromptTokens?: number; + usageCompletionTokens?: number; +} + +export type Span = Omit; +export type OptionsWithTracing = { tracingOptions?: OperationTracingOptions, tracingAttributeOptions?: TracingAttributeOptions}; + +export class TracingUtility { + private static tracingClient = createTracingClient({ + namespace: "Microsoft.CognitiveServices", + packageName: PACKAGE_NAME, + packageVersion: SDK_VERSION, + + }); + + static async withSpan ReturnType>(name: string, options: Options, request: Request, + startTrace?: (span: Span, updatedOptions: Options,) => void, + endTrace?: (span: Span, updatedOptions: Options, result: ReturnType) => void, + ): Promise>> { + return TracingUtility.tracingClient.withSpan(name, options, async (updatedOptions: Options, span: Span) => { + if (startTrace) { + try { + updatedOptions.tracingAttributeOptions = {...updatedOptions.tracingAttributeOptions, operationName: name}; + startTrace(span, updatedOptions); + } + catch (e) { + logger.warning(`Skipping updating span before request execution due to an error: ${getErrorMessage(e)}`); + } + + } + let result: ReturnType | undefined; + try { + result = await request(updatedOptions); + } catch (error) { + const errorStatus: SpanStatusError = { status: "error" } + if (error instanceof Error) { + errorStatus.error = error; + } + throw error; + } + + if (endTrace && result !== undefined) { + try { + endTrace(span, updatedOptions, result); + } + catch (e) { + logger.warning(`Skipping updating span after request execution due to an error: ${getErrorMessage(e)}`); + } + } + return result; + }, { spanKind: "client" }); + } + + static updateSpanAttributes(span: Span, attributeOptions: Omit): void { + TracingUtility.setAttributes(span, attributeOptions); + } + + static setSpanAttributes( + span: Span, + operationName: string, + attributeOptions: TracingAttributeOptions + ): void { + attributeOptions.operationName = operationName; + TracingUtility.setAttributes(span, attributeOptions); + } + + static setAttributes( + span: Span, + attributeOptions: TracingAttributeOptions + ): void { + if (span.isRecording()) { + const { + name, + operationName, + description, + serverAddress, + threadId, + agentId, + messageId, + runId, + model, + temperature, + topP, + maxPromptTokens, + maxCompletionTokens, + responseFormat, + runStatus, + responseModel, + usageCompletionTokens, + usagePromptTokens, + genAiSystem = TracingAttributes.AZ_AI_AGENT_SYSTEM + } = attributeOptions; + + if (genAiSystem) { + span.setAttribute(TracingAttributes.GEN_AI_SYSTEM, genAiSystem); + } + if (name) { + span.setAttribute(TracingAttributes.GEN_AI_AGENT_NAME, name); + } + if (description) { + span.setAttribute(TracingAttributes.GEN_AI_AGENT_DESCRIPTION, description); + } + + if (serverAddress) { + span.setAttribute(TracingAttributes.SERVER_ADDRESS, serverAddress); + } + + if (threadId) { + span.setAttribute(TracingAttributes.GEN_AI_THREAD_ID, threadId); + } + + if (agentId) { + span.setAttribute(TracingAttributes.GEN_AI_AGENT_ID, agentId); + } + + if (runId) { + span.setAttribute(TracingAttributes.GEN_AI_THREAD_RUN_ID, runId); + } + + if (messageId) { + span.setAttribute(TracingAttributes.GEN_AI_MESSAGE_ID, messageId); + } + if (model) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MODEL, model); + } + + if (temperature !== null) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_TEMPERATURE, temperature); + } + + if (topP !== null) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_TOP_P, topP); + } + + if (maxPromptTokens !== null) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MAX_INPUT_TOKENS, maxPromptTokens); + } + + if (maxCompletionTokens !== null) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MAX_OUTPUT_TOKENS, maxCompletionTokens); + } + + if (responseFormat) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_RESPONSE_FORMAT, responseFormat); + } + + if (runStatus) { + span.setAttribute(TracingAttributes.GEN_AI_THREAD_RUN_STATUS, runStatus); + } + + if (responseModel) { + span.setAttribute(TracingAttributes.GEN_AI_RESPONSE_MODEL, responseModel); + } + + if (usagePromptTokens) { + span.setAttribute(TracingAttributes.GEN_AI_USAGE_INPUT_TOKENS, usagePromptTokens); + } + + if (usageCompletionTokens) { + span.setAttribute(TracingAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, usageCompletionTokens); + } + if (operationName) { + span.setAttribute(TracingAttributes.GEN_AI_OPERATION_NAME, operationName); + } + } + return; + } + + static addSpanEvent( + span: Span, + eventName: string, + attributeOptions: Omit + ): void { + if (span.isRecording()) { + const { threadId, agentId, runId, messageId, eventContent, usageCompletionTokens, usagePromptTokens, messageStatus } = attributeOptions; + const attributes: Record = {}; + + if (eventContent) { + attributes[TracingAttributes.GEN_AI_EVENT_CONTENT] = eventContent; + } + + if (threadId) { + attributes[TracingAttributes.GEN_AI_THREAD_ID] = threadId; + } + + if (agentId) { + attributes[TracingAttributes.GEN_AI_AGENT_ID] = agentId; + } + + if (runId) { + attributes[TracingAttributes.GEN_AI_THREAD_RUN_ID] = runId; + } + + if (messageId) { + attributes[TracingAttributes.GEN_AI_MESSAGE_ID] = messageId; + } + if (messageStatus) { + attributes[TracingAttributes.GEN_AI_MESSAGE_STATUS] = messageStatus; + } + + if (usagePromptTokens) { + attributes[TracingAttributes.GEN_AI_USAGE_INPUT_TOKENS] = usagePromptTokens; + } + + if (usageCompletionTokens) { + attributes[TracingAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] = usageCompletionTokens; + } + + if (span.addEvent) { + span.addEvent(eventName, { attributes }); + } + } + return; + } + +} + + + diff --git a/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts b/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts new file mode 100644 index 000000000000..36ba7bbc96e1 --- /dev/null +++ b/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { AgentOutput, AgentsOperations, AgentThreadOutput, AIProjectsClient, ThreadMessageOutput, ThreadRunOutput } from "../../../src/index.js"; +import { createMockProjectsClient } from "../utils/createClient.js"; +import { assert, beforeEach, afterEach, it, describe, vi } from "vitest"; +import { MockInstrumenter, MockTracingSpan } from "@azure-tools/test-utils-vitest"; +import { AddEventOptions, Instrumenter, InstrumenterSpanOptions, TracingContext, TracingSpan, useInstrumenter } from "@azure/core-tracing"; + +interface ExtendedMockTrackingSpan extends MockTracingSpan { + events?: { name: string, attributes: Record }[] + addEvent?(eventName: string, options?: AddEventOptions): void; +} +class ExtendedMockInstrumenter extends MockInstrumenter { + extendSpan(span: any): void { + if (!span.events) { + span.events = []; + } + span.addEvent = (eventName: string, options?: AddEventOptions) => { + span.events.push({ name: eventName, ...options }); + } + } + startSpan(name: string, + spanOptions?: InstrumenterSpanOptions): { span: TracingSpan; tracingContext: TracingContext } { + const { span, tracingContext } = super.startSpan(name, spanOptions); + this.extendSpan(span); + return { span, tracingContext } + } + +} + +describe("Agent Tracing", () => { + let instrumenter: Instrumenter; + let projectsClient: AIProjectsClient; + let agents: AgentsOperations; + let response: any = {}; + let status = 200; + beforeEach(async function () { + instrumenter = new ExtendedMockInstrumenter() + useInstrumenter(instrumenter); + + projectsClient = createMockProjectsClient(() => { return { bodyAsText: JSON.stringify(response), status: status } }); + agents = projectsClient.agents + }); + + afterEach(async function () { + (instrumenter as MockInstrumenter).reset(); + vi.clearAllMocks(); + response = {}; + status = 200; + }); + + it("create agent", async function () { + const agentResponse: Partial = { id: "agentId", object: "assistant" }; + response = agentResponse; + status = 200; + const request = { name: "agentName", instructions: "You are helpful agent", response_format: "json" }; + const model = "gpt-4o"; + await agents.createAgent(model, request); + const mockedInstrumenter = instrumenter as MockInstrumenter; + assert.isAbove(mockedInstrumenter.startedSpans.length, 0); + const span = mockedInstrumenter.startedSpans[0] as ExtendedMockTrackingSpan; + assert.equal(span.attributes["gen_ai.agent.id"], agentResponse.id); + assert.equal(span.attributes["gen_ai.operation.name"], "create_agent"); + assert.equal(span.attributes["gen_ai.agent.name"], request.name); + assert.equal(span.attributes["gen_ai.request.model"], model); + const event = span.events?.find(e => e.name === "gen_ai.system.message"); + assert.isDefined(event); + assert.equal(event?.attributes["gen_ai.event.content"], JSON.stringify({ content: request.instructions })); + + }) + + it("create run", async function () { + const runResponse: Partial = { id: "runId", object: "thread.run", status: "queued", thread_id: "threadId", assistant_id: "agentId" }; + response = runResponse; + status = 200; + await agents.createRun("threadId", "agentId"); + const mockedInstrumenter = instrumenter as MockInstrumenter; + assert.isAbove(mockedInstrumenter.startedSpans.length, 0); + const span = mockedInstrumenter.startedSpans[0] as ExtendedMockTrackingSpan; + assert.equal(span.attributes["gen_ai.thread.id"], runResponse.thread_id); + assert.equal(span.attributes["gen_ai.operation.name"], "create_run"); + assert.equal(span.attributes["gen_ai.agent.id"], runResponse.assistant_id); + assert.equal(span.attributes["gen_ai.thread.run.status"], runResponse.status); + assert.equal(span.events!.length, 1); + }) + + it("create Thread", async function () { + const threadResponse: Partial = { id: "threadId", object: "thread" }; + response = threadResponse; + status = 200; + await agents.createThread(); + const mockedInstrumenter = instrumenter as MockInstrumenter; + assert.isAbove(mockedInstrumenter.startedSpans.length, 0); + const span = mockedInstrumenter.startedSpans[0]; + assert.equal(span.attributes["gen_ai.thread.id"], threadResponse.id); + assert.equal(span.attributes["gen_ai.operation.name"], "create_thread"); + }) + + it("create Message", async function () { + const messageResponse: Partial = { id: "messageId", object: "thread.message", thread_id: "threadId" }; + projectsClient.telemetry.updateSettings({ enableContentRecording: true }); + response = messageResponse; + status = 200; + const request = { content: "hello, world!", role: "user" }; + await agents.createMessage("threadId", request); + const mockedInstrumenter = instrumenter as MockInstrumenter; + assert.isAbove(mockedInstrumenter.startedSpans.length, 0); + const span = mockedInstrumenter.startedSpans[0] as ExtendedMockTrackingSpan; + assert.equal(span.attributes["gen_ai.thread.id"], messageResponse.thread_id); + assert.equal(span.attributes["gen_ai.operation.name"], "create_message"); + const event = span.events?.find(e => e.name === "gen_ai.user.message"); + assert.isDefined(event); + assert.equal(event?.attributes["gen_ai.event.content"], JSON.stringify(request)); + assert.equal(event?.attributes["gen_ai.thread.id"], messageResponse.thread_id); + assert.equal(event?.name, "gen_ai.user.message"); + }) + + it("list messages", async function () { + const listMessages = { object: "list", data: [{ id: "messageId", object: "thread.message", thread_id: "threadId", role: "assistant", content: [{ type: "text", text: { value: "You are helpful agent" } }] }, { id: "messageId2", object: "thread.message", thread_id: "threadId", role: "user", content: [{ type: "text", text: { value: "Hello, tell me a joke" } }] }] }; + response = listMessages; + projectsClient.telemetry.updateSettings({ enableContentRecording: true }); + status = 200; + await agents.listMessages("threadId"); + const mockedInstrumenter = instrumenter as MockInstrumenter; + assert.isAbove(mockedInstrumenter.startedSpans.length, 0); + const span = mockedInstrumenter.startedSpans[0] as ExtendedMockTrackingSpan; + assert.isDefined(span.events); + assert.equal(span.events!.length, 2); + assert.equal(span.events![0].attributes["gen_ai.event.content"], JSON.stringify({ content: { text: listMessages.data[0].content[0].text }, role: listMessages.data[0].role })); + assert.equal(span.events![0].name, "gen_ai.assistant.message"); + assert.equal(span.events![1].attributes["gen_ai.event.content"], JSON.stringify({ content: { text: listMessages.data[1].content[0].text }, role: listMessages.data[1].role })); + assert.equal(span.events![1].name, "gen_ai.user.message"); + }) + + it("Submit tool outputs to run", async function () { + const submitToolOutputs = { object: "thread.run", id: "runId", status: "queued", thread_id: "threadId", assistant_id: "agentId" }; + response = submitToolOutputs; + status = 200; + const toolOutputs = [{ tool_call_id: "toolcallId1", output: "output1" }, { tool_call_id: "toolcallId2", output: "output2" }]; + await agents.submitToolOutputsToRun("threadId", "runId", toolOutputs); + const mockedInstrumenter = instrumenter as MockInstrumenter; + assert.isAbove(mockedInstrumenter.startedSpans.length, 0); + const span = mockedInstrumenter.startedSpans[0] as ExtendedMockTrackingSpan; + assert.equal(span.attributes["gen_ai.thread.id"], submitToolOutputs.thread_id); + assert.equal(span.attributes["gen_ai.operation.name"], "submit_tool_outputs"); + assert.equal(span.attributes["gen_ai.thread.run.status"], submitToolOutputs.status); + assert.isDefined(span.events); + assert.equal(span.events!.length, 2); + assert.equal(span.events![0].attributes["gen_ai.event.content"], JSON.stringify({ content: toolOutputs[0].output, id: toolOutputs[0].tool_call_id })); + assert.equal(span.events![0].name, "gen_ai.tool.message"); + assert.equal(span.events![1].attributes["gen_ai.event.content"], JSON.stringify({ content: toolOutputs[1].output, id: toolOutputs[1].tool_call_id })); + assert.equal(span.events![1].name, "gen_ai.tool.message"); + }); +}); diff --git a/sdk/ai/ai-projects/test/public/telemetry/telemetry.spec.ts b/sdk/ai/ai-projects/test/public/telemetry/telemetry.spec.ts new file mode 100644 index 000000000000..b600fea073c8 --- /dev/null +++ b/sdk/ai/ai-projects/test/public/telemetry/telemetry.spec.ts @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Recorder, VitestTestContext } from "@azure-tools/test-recorder"; +import { AIProjectsClient, TelemetryOperations } from "../../../src/index.js"; +import { createRecorder, createProjectsClient } from "../utils/createClient.js"; +import { assert, beforeEach, afterEach, it, describe } from "vitest"; + +describe("AI Projects - Telemetry", () => { + let recorder: Recorder; + let projectsClient: AIProjectsClient; + let telemetry: TelemetryOperations + + beforeEach(async function (context: VitestTestContext) { + recorder = await createRecorder(context); + projectsClient = createProjectsClient(recorder); + telemetry = projectsClient.telemetry; + }); + + afterEach(async function () { + await recorder.stop(); + }); + it("client and connection operations are accessible", async function () { + assert.isNotNull(projectsClient); + assert.isNotNull(telemetry); + }); + + + + it("update settings", async function () { + assert.equal(telemetry.getSettings().enableContentRecording, false); + telemetry.updateSettings({ enableContentRecording: true }); + assert.equal(telemetry.getSettings().enableContentRecording, true); + }); + + it("get settings", async function () { + const options = telemetry.getSettings(); + assert.equal(options.enableContentRecording, false); + options.enableContentRecording = true; + assert.equal(telemetry.getSettings().enableContentRecording, false); + }); + + it("get app insights connection string", async function () { + const connectionString = await telemetry.getConnectionString(); + assert.isNotEmpty(connectionString); + }); + +}); diff --git a/sdk/ai/ai-projects/test/public/utils/createClient.ts b/sdk/ai/ai-projects/test/public/utils/createClient.ts index 8b0d095cc6df..328c31f3db96 100644 --- a/sdk/ai/ai-projects/test/public/utils/createClient.ts +++ b/sdk/ai/ai-projects/test/public/utils/createClient.ts @@ -9,8 +9,11 @@ import { import { createTestCredential } from "@azure-tools/test-credential"; import { AIProjectsClient } from "../../../src/index.js"; import { ClientOptions } from "@azure-rest/core-client"; +import { createHttpHeaders, PipelineRequest, PipelineResponse } from "@azure/core-rest-pipeline"; const replaceableVariables: Record = { + GENERIC_STRING: "Sanitized", + ENDPOINT: "Sanitized.azure.com", SUBSCRIPTION_ID: "00000000-0000-0000-0000-000000000000", RESOURCE_GROUP_NAME: "00000", WORKSPACE_NAME: "00000", @@ -28,6 +31,10 @@ const recorderEnvSetup: RecorderStartOptions = { { regex: true, target: "(%2F|/)?subscriptions(%2F|/)([-\\w\\._\\(\\)]+)", value: replaceableVariables.SUBSCRIPTION_ID, groupForReplace: "3" }, { regex: true, target: "(%2F|/)?resource[gG]roups(%2F|/)([-\\w\\._\\(\\)]+)", value: replaceableVariables.RESOURCE_GROUP_NAME, groupForReplace: "3" }, { regex: true, target: "/workspaces/([-\\w\\._\\(\\)]+)", value: replaceableVariables.WORKSPACE_NAME, groupForReplace: "1" }, + { regex: true, target: "/userAssignedIdentities/([-\\w\\._\\(\\)]+)", value: replaceableVariables.GENERIC_STRING, groupForReplace: "1" }, + { regex: true, target: "/components/([-\\w\\._\\(\\)]+)", value: replaceableVariables.GENERIC_STRING, groupForReplace: "1" }, + { regex: true, target: "/vaults/([-\\w\\._\\(\\)]+)", value: replaceableVariables.GENERIC_STRING, groupForReplace: "1" }, + { regex: true, target: "(azureml|http|https):\\/\\/([^\\/]+)", value: replaceableVariables.ENDPOINT, groupForReplace: "2" }, ], bodyKeySanitizers: [ { jsonPath: "properties.ConnectionString", value: "InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://region.applicationinsights.azure.com/;LiveEndpoint=https://region.livediagnostics.monitor.azure.com/;ApplicationId=00000000-0000-0000-0000-000000000000"}, @@ -56,5 +63,26 @@ export function createProjectsClient( ): AIProjectsClient { const credential = createTestCredential(); const connectionString = process.env["AZURE_AI_PROJECTS_CONNECTION_STRING"] || ""; - return AIProjectsClient.fromConnectionString(connectionString, credential, recorder?.configureClientOptions(options ?? {})); + return AIProjectsClient.fromConnectionString( + connectionString, + credential, + recorder ? recorder.configureClientOptions(options ?? {}) : options + ); +} + +export function createMockProjectsClient(responseFn: (request:PipelineRequest) => Partial): AIProjectsClient { + const options: ClientOptions = { additionalPolicies: [] }; + options.additionalPolicies?.push({ + policy: { + name: "RequestMockPolicy", + sendRequest:(async (req) => { + const response = responseFn(req); + return { headers: createHttpHeaders(), status: 200, request: req, ...response } as PipelineResponse; + }) + }, + position: "perCall" + }); + const credential = createTestCredential(); + const connectionString = process.env["AZURE_AI_PROJECTS_CONNECTION_STRING"] || ""; + return AIProjectsClient.fromConnectionString(connectionString, credential, options); }