From 1dc85b4a49702d75df06dd20a4fef306b8447019 Mon Sep 17 00:00:00 2001 From: Ganesh Bheemarasetty Date: Fri, 22 Nov 2024 11:31:44 -0800 Subject: [PATCH 1/9] First version of agent tracking --- common/config/rush/pnpm-lock.yaml | 5 +- sdk/ai/ai-projects/package.json | 17 +- .../agents/agents_basics_tracing_console.ts | 50 ++++++ sdk/ai/ai-projects/src/agents/assistants.ts | 149 ++++++++++++------ sdk/ai/ai-projects/src/tracing.ts | 146 +++++++++++++++++ 5 files changed, 313 insertions(+), 54 deletions(-) create mode 100644 sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts create mode 100644 sdk/ai/ai-projects/src/tracing.ts diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index b0032bcdd601..abc331da3c7a 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -12118,11 +12118,14 @@ packages: dev: false file:projects/ai-projects.tgz: - resolution: {integrity: sha512-kc59ifhzGCQM5eo2AEvxx5VBGjF09tNlPfSKfMwijzN/c3smypANT1hsnO3BIrc3BH+jeqJ0dRKhpZBTJfUY2w==, tarball: file:projects/ai-projects.tgz} + resolution: {integrity: sha512-AteUpToB0cbZyNZ3+DwqN/EI2BeqnfqbOhLL6W5LeLPHOrKsG84ujxOFjTPedPrqrJSvb08UtwRwvam4BXnBTA==, tarball: file:projects/ai-projects.tgz} name: '@rush-temp/ai-projects' version: 0.0.0 dependencies: '@microsoft/api-extractor': 7.47.9(@types/node@18.19.55) + '@opentelemetry/api': 1.9.0 + '@opentelemetry/instrumentation': 0.53.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-node': 1.26.0(@opentelemetry/api@1.9.0) '@types/node': 18.19.55 '@vitest/browser': 2.1.3(playwright@1.48.0)(typescript@5.5.4)(vitest@2.1.3) '@vitest/coverage-istanbul': 2.1.3(vitest@2.1.3) diff --git a/sdk/ai/ai-projects/package.json b/sdk/ai/ai-projects/package.json index 8be3d413a696..cf59a24c7fd9 100644 --- a/sdk/ai/ai-projects/package.json +++ b/sdk/ai/ai-projects/package.json @@ -58,19 +58,24 @@ "@azure-rest/core-client": "^2.1.0", "@azure/core-auth": "^1.6.0", "@azure/core-rest-pipeline": "^1.5.0", - "@azure/core-util": "^1.9.0", - "@azure/logger": "^1.1.4", - "tslib": "^2.6.2", "@azure/core-paging": "^1.5.0", - "@azure/core-sse": "^2.1.3" + "@azure/logger": "^1.1.4", + "@azure/core-sse": "^2.1.3", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.9.0", + "tslib": "^2.6.2" }, "devDependencies": { "@azure/dev-tool": "^1.0.0", "@azure/eslint-plugin-azure-sdk": "^3.0.0", "@azure/identity": "^4.3.0", + "@azure/opentelemetry-instrumentation-azure-sdk": "^1.0.0-beta.7", "@azure-tools/test-credential": "^2.0.0", "@azure-tools/test-recorder": "^4.1.0", "@microsoft/api-extractor": "^7.40.3", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/instrumentation": "0.53.0", + "@opentelemetry/sdk-trace-node": "^1.9.0", "@vitest/browser": "^2.0.5", "@vitest/coverage-istanbul": "^2.0.5", "@types/node": "^18.0.0", @@ -112,18 +117,22 @@ "./package.json": "./package.json", ".": { "browser": { + "source": "./src/index.ts", "types": "./dist/browser/index.d.ts", "default": "./dist/browser/index.js" }, "react-native": { + "source": "./src/index.ts", "types": "./dist/react-native/index.d.ts", "default": "./dist/react-native/index.js" }, "import": { + "source": "./src/index.ts", "types": "./dist/esm/index.d.ts", "default": "./dist/esm/index.js" }, "require": { + "source": "./src/index.ts", "types": "./dist/commonjs/index.d.ts", "default": "./dist/commonjs/index.js" } diff --git a/sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts b/sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts new file mode 100644 index 000000000000..75f39031f79a --- /dev/null +++ b/sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + + +import { context } from "@opentelemetry/api"; +import { registerInstrumentations } from "@opentelemetry/instrumentation"; +import { createAzureSdkInstrumentation } from "@azure/opentelemetry-instrumentation-azure-sdk"; +import { + ConsoleSpanExporter, + NodeTracerProvider, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-node"; + + +import * as dotenv from "dotenv"; +dotenv.config(); + +const provider = new NodeTracerProvider(); +provider.addSpanProcessor(new SimpleSpanProcessor(new ConsoleSpanExporter())); +provider.register(); + +registerInstrumentations({ + instrumentations: [createAzureSdkInstrumentation()], +}); + + + +import { AIProjectsClient } from "@azure/ai-projects" +import { DefaultAzureCredential } from "@azure/identity"; + +const connectionString = process.env["AZURE_AI_PROJECTS_CONNECTION_STRING"] || ">;;;"; + +export async function main(): Promise { + + const client = AIProjectsClient.fromConnectionString(connectionString || "", new DefaultAzureCredential()); + + + const agent = await client.agents.createAgent("gpt-4o", { name: "my-agent", instructions: "You are helpful agent" }, { tracingOptions: { tracingContext: context.active() } }); + + + console.log(`Created agent, agent ID : ${agent.id}`); + + client.agents.deleteAgent(agent.id); + + console.log(`Deleted agent`); +} + +main().catch((err) => { + console.error("The sample encountered an error:", err); +}); diff --git a/sdk/ai/ai-projects/src/agents/assistants.ts b/sdk/ai/ai-projects/src/agents/assistants.ts index 07da09d90963..ef45c13afdef 100644 --- a/sdk/ai/ai-projects/src/agents/assistants.ts +++ b/sdk/ai/ai-projects/src/agents/assistants.ts @@ -2,81 +2,132 @@ // Licensed under the MIT License. import { Client, createRestError } from "@azure-rest/core-client"; +import { AgentsApiResponseFormatOption, CreateAgentOptions, AgentsApiResponseFormat } from "../generated/src/models.js"; import { AgentDeletionStatusOutput, AgentOutput, OpenAIPageableListOfAgentOutput } from "../generated/src/outputModels.js"; -import { CreateAgentParameters, DeleteAgentParameters, GetAgentParameters, UpdateAgentParameters, ListAgentsParameters } from "../generated/src/parameters.js"; +import { CreateAgentParameters, DeleteAgentParameters, GetAgentParameters, ListAgentsParameters, UpdateAgentParameters } from "../generated/src/parameters.js"; +import { setSpanAttributes, TracingAttributeOptions, tracingClient, TrackingOperationName } from "../tracing.js"; const expectedStatuses = ["200"]; /** Creates a new agent. */ export async function createAgent( - context: Client, - options: CreateAgentParameters, - ): Promise { - const result = await context.path("/assistants").post(options); - + context: Client, + options: CreateAgentParameters +): Promise { + return tracingClient.withSpan("createAgent", options, async (updatedOptions, span) => { + setSpanAttributes(span, getAgentAttributes(options.body)) + const result = await context.path("/assistants").post(updatedOptions); + if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); + throw createRestError(result); } - return result.body; - } + const resultBody = result.body as AgentOutput; + setSpanAttributes(span, getAgentResultAttributes(resultBody)); + return resultBody; + }); +} - /** Gets a list of agents that were previously created. */ - export async function listAgents( - context: Client, - options?: ListAgentsParameters, - ): Promise { - const result = await context +/** Gets a list of agents that were previously created. */ +export async function listAgents( + context: Client, + options?: ListAgentsParameters +): Promise { + const result = await context .path("/assistants") .get(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); } + return result.body; +} - /** Retrieves an existing agent. */ +/** Retrieves an existing agent. */ export async function getAgent( - context: Client, - assistantId: string, - options?: GetAgentParameters, - ): Promise { - const result = await context + context: Client, + assistantId: string, + options?: GetAgentParameters, +): Promise { + const result = await context .path("/assistants/{assistantId}", assistantId) .get(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); } + return result.body; +} - /** Modifies an existing agent. */ +/** Modifies an existing agent. */ export async function updateAgent( - context: Client, - assistantId: string, - options?: UpdateAgentParameters, - ): Promise { - const result = await context + context: Client, + assistantId: string, + options?: UpdateAgentParameters, +): Promise { + const result = await context .path("/assistants/{assistantId}", assistantId) .post(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); } + return result.body; +} - /** Deletes an agent. */ +/** Deletes an agent. */ export async function deleteAgent( - context: Client, - assistantId: string, - options?: DeleteAgentParameters, - ): Promise { - const result = await context + context: Client, + assistantId: string, + options?: DeleteAgentParameters, +): Promise { + const result = await context .path("/assistants/{assistantId}", assistantId) .delete(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; +} + +/** + * Extracts tracing attributes from the agent creation options. + * @param body - The options for creating an agent. + * @returns The tracing attributes. + */ +function getAgentAttributes(body: CreateAgentOptions): TracingAttributeOptions { + return { + operationName: TrackingOperationName.CREATE_AGENT, + name: body.name, + model: body.model, + description: body.description, + instructions: body.instructions, + topP: body.top_p, + responseFormat: formatAgentApiResponse(body.response_format) + }; +} + +/** + * Extracts tracing attributes from the agent output. + * @param output - The output of the agent. + * @returns TracingAttributeOptions The tracing attributes. + */ +function getAgentResultAttributes(output: AgentOutput): TracingAttributeOptions { + return { + operationName: TrackingOperationName.CREATE_AGENT, + agentId: output.id, + }; +} + +/** + * Formats the agent API response. + * @param responseFormat - The response format option. + * @returns The formatted response as a string, or null/undefined. + */ +function formatAgentApiResponse(responseFormat: AgentsApiResponseFormatOption | null | undefined): string | null | undefined { + if (typeof responseFormat === "string" || responseFormat === undefined || responseFormat === null) { + return responseFormat; + } + if ((responseFormat as AgentsApiResponseFormat).type) { + return (responseFormat as AgentsApiResponseFormat).type; } + return undefined; +} diff --git a/sdk/ai/ai-projects/src/tracing.ts b/sdk/ai/ai-projects/src/tracing.ts new file mode 100644 index 000000000000..7c02fdbb2ade --- /dev/null +++ b/sdk/ai/ai-projects/src/tracing.ts @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createTracingClient, TracingSpan } from "@azure/core-tracing"; + +export const tracingClient = createTracingClient({ + namespace: "Microsoft.CognitiveServices", + packageName: "@azure/ai-projects", + packageVersion: "1.0.0-beta.1", +}); + +export enum TracingAttributes { + GEN_AI_MESSAGE_ID = "gen_ai.message.id", + GEN_AI_MESSAGE_STATUS = "gen_ai.message.status", + GEN_AI_THREAD_ID = "gen_ai.thread.id", + GEN_AI_THREAD_RUN_ID = "gen_ai.thread.run.id", + GEN_AI_AGENT_ID = "gen_ai.agent.id", + GEN_AI_AGENT_NAME = "gen_ai.agent.name", + GEN_AI_AGENT_DESCRIPTION = "gen_ai.agent.description", + GEN_AI_OPERATION_NAME = "gen_ai.operation.name", + GEN_AI_THREAD_RUN_STATUS = "gen_ai.thread.run.status", + GEN_AI_REQUEST_MODEL = "gen_ai.request.model", + GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature", + GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p", + GEN_AI_REQUEST_MAX_INPUT_TOKENS = "gen_ai.request.max_input_tokens", + GEN_AI_REQUEST_MAX_OUTPUT_TOKENS = "gen_ai.request.max_output_tokens", + GEN_AI_RESPONSE_MODEL = "gen_ai.response.model", + GEN_AI_SYSTEM = "gen_ai.system", + SERVER_ADDRESS = "server.address", + AZ_AI_AGENT_SYSTEM = "az.ai.agents", + GEN_AI_TOOL_NAME = "gen_ai.tool.name", + GEN_AI_TOOL_CALL_ID = "gen_ai.tool.call.id", + GEN_AI_REQUEST_RESPONSE_FORMAT = "gen_ai.request.response_format", + GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens", + GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens", + GEN_AI_SYSTEM_MESSAGE = "gen_ai.system.message", + GEN_AI_EVENT_CONTENT = "gen_ai.event.content", + ERROR_TYPE = "error.type" +} +export enum TrackingOperationName { + CREATE_AGENT = "create_agent", + CREATE_THREAD = "create_thread", + CREATE_MESSAGE = "create_message", + START_THREAD_RUN = "start_thread_run", + EXECUTE_TOOL = "execute_tool", + LIST_MESSAGES = "list_messages", + SUBMIT_TOOL_OUTPUTS = "submit_tool_outputs", + PROCESS_THREAD_RUN = "process_thread_run", +} + +export interface TracingAttributeOptions { + operationName: string; + name?: string | null; + description?: string | null; + serverAddress?: string | null; + threadId?: string | null; + agentId?: string | null; + instructions?: string | null; + additional_instructions?: string | null; + runId?: string | null; + model?: string | null; + temperature?: number | null; + topP?: number | null; + maxPromptTokens?: number | null; + maxCompletionTokens?: number | null; + responseFormat?: string | null; + genAiSystem?: string | null; +} + +export function setSpanAttributes( + span: Omit, + attributeOptions: TracingAttributeOptions +): void { + if (span) { + const { + operationName, + name, + description, + serverAddress, + threadId, + agentId, + runId, + model, + temperature, + topP, + maxPromptTokens, + maxCompletionTokens, + responseFormat, + genAiSystem = "AZ_AI_AGENT_SYSTEM" + } = attributeOptions; + + if (genAiSystem) { + span.setAttribute(TracingAttributes.GEN_AI_SYSTEM, genAiSystem); + } + + span.setAttribute(TracingAttributes.GEN_AI_OPERATION_NAME, operationName); + + if (name) { + span.setAttribute(TracingAttributes.GEN_AI_AGENT_NAME, name); + } + if (description) { + span.setAttribute(TracingAttributes.GEN_AI_AGENT_DESCRIPTION, description); + } + + if (serverAddress) { + span.setAttribute(TracingAttributes.SERVER_ADDRESS, serverAddress); + } + + if (threadId) { + span.setAttribute(TracingAttributes.GEN_AI_THREAD_ID, threadId); + } + + if (agentId) { + span.setAttribute(TracingAttributes.GEN_AI_AGENT_ID, agentId); + } + + if (runId) { + span.setAttribute(TracingAttributes.GEN_AI_THREAD_RUN_ID, runId); + } + + if (model) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MODEL, model); + } + + if (temperature !== null) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_TEMPERATURE, temperature); + } + + if (topP !== null) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_TOP_P, topP); + } + + if (maxPromptTokens !== null) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MAX_INPUT_TOKENS, maxPromptTokens); + } + + if (maxCompletionTokens !== null) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MAX_OUTPUT_TOKENS, maxCompletionTokens); + } + + if (responseFormat) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_RESPONSE_FORMAT, responseFormat); + } + } + return; +} From f3cea7ceb1009d968a70751fc7a350acd340a0f7 Mon Sep 17 00:00:00 2001 From: Ganesh Bheemarasetty Date: Mon, 2 Dec 2024 09:36:49 -0800 Subject: [PATCH 2/9] Tracing for agents, run, thread and message --- sdk/ai/ai-projects/package.json | 1 + sdk/ai/ai-projects/review/ai-projects.api.md | 8 + .../agents/agents_basics_tracing_console.ts | 3 - sdk/ai/ai-projects/src/agents/assistants.ts | 70 +---- .../ai-projects/src/agents/assistantsTrace.ts | 44 +++ sdk/ai/ai-projects/src/agents/messages.ts | 20 +- .../ai-projects/src/agents/messagesTrace.ts | 18 ++ sdk/ai/ai-projects/src/agents/runTrace.ts | 45 +++ sdk/ai/ai-projects/src/agents/runs.ts | 30 +- sdk/ai/ai-projects/src/agents/threads.ts | 6 +- sdk/ai/ai-projects/src/agents/threadsTrace.ts | 24 ++ sdk/ai/ai-projects/src/agents/traceUtility.ts | 82 ++++++ sdk/ai/ai-projects/src/aiProjectsClient.ts | 5 + sdk/ai/ai-projects/src/index.ts | 1 + sdk/ai/ai-projects/src/telemetry/index.ts | 37 +++ sdk/ai/ai-projects/src/telemetry/telemetry.ts | 51 ++++ sdk/ai/ai-projects/src/tracing.ts | 272 +++++++++++++----- .../test/public/agents/tracing.spec.ts | 119 ++++++++ .../test/public/utils/createClient.ts | 24 +- 19 files changed, 699 insertions(+), 161 deletions(-) create mode 100644 sdk/ai/ai-projects/src/agents/assistantsTrace.ts create mode 100644 sdk/ai/ai-projects/src/agents/messagesTrace.ts create mode 100644 sdk/ai/ai-projects/src/agents/runTrace.ts create mode 100644 sdk/ai/ai-projects/src/agents/threadsTrace.ts create mode 100644 sdk/ai/ai-projects/src/agents/traceUtility.ts create mode 100644 sdk/ai/ai-projects/src/telemetry/index.ts create mode 100644 sdk/ai/ai-projects/src/telemetry/telemetry.ts create mode 100644 sdk/ai/ai-projects/test/public/agents/tracing.spec.ts diff --git a/sdk/ai/ai-projects/package.json b/sdk/ai/ai-projects/package.json index cf59a24c7fd9..800949bc6b6c 100644 --- a/sdk/ai/ai-projects/package.json +++ b/sdk/ai/ai-projects/package.json @@ -72,6 +72,7 @@ "@azure/opentelemetry-instrumentation-azure-sdk": "^1.0.0-beta.7", "@azure-tools/test-credential": "^2.0.0", "@azure-tools/test-recorder": "^4.1.0", + "@azure-tools/test-utils-vitest": "^1.0.0", "@microsoft/api-extractor": "^7.40.3", "@opentelemetry/api": "^1.9.0", "@opentelemetry/instrumentation": "0.53.0", diff --git a/sdk/ai/ai-projects/review/ai-projects.api.md b/sdk/ai/ai-projects/review/ai-projects.api.md index 1a2f7af2f2c6..1fafb83ac10c 100644 --- a/sdk/ai/ai-projects/review/ai-projects.api.md +++ b/sdk/ai/ai-projects/review/ai-projects.api.md @@ -158,6 +158,7 @@ export class AIProjectsClient { readonly agents: AgentsOperations; readonly connections: ConnectionsOperations; static fromConnectionString(connectionString: string, credential: TokenCredential, options?: AIProjectsClientOptions): AIProjectsClient; + readonly telemetry: TelemetryOperations; } // @public (undocumented) @@ -1410,6 +1411,13 @@ export interface SystemDataOutput { readonly lastModifiedAt?: string; } +// @public +export interface TelemetryOperations { + getConnectionString(): Promise; + // Warning: (ae-forgotten-export) The symbol "TelemetryOptions" needs to be exported by the entry point index.d.ts + updateSettings(options: TelemetryOptions): void; +} + // @public export interface ThreadDeletionStatusOutput { deleted: boolean; diff --git a/sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts b/sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts index 75f39031f79a..78c39eea7a6f 100644 --- a/sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts +++ b/sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts @@ -11,7 +11,6 @@ import { SimpleSpanProcessor, } from "@opentelemetry/sdk-trace-node"; - import * as dotenv from "dotenv"; dotenv.config(); @@ -23,8 +22,6 @@ registerInstrumentations({ instrumentations: [createAzureSdkInstrumentation()], }); - - import { AIProjectsClient } from "@azure/ai-projects" import { DefaultAzureCredential } from "@azure/identity"; diff --git a/sdk/ai/ai-projects/src/agents/assistants.ts b/sdk/ai/ai-projects/src/agents/assistants.ts index ef45c13afdef..116f402d9c6a 100644 --- a/sdk/ai/ai-projects/src/agents/assistants.ts +++ b/sdk/ai/ai-projects/src/agents/assistants.ts @@ -2,11 +2,10 @@ // Licensed under the MIT License. import { Client, createRestError } from "@azure-rest/core-client"; -import { AgentsApiResponseFormatOption, CreateAgentOptions, AgentsApiResponseFormat } from "../generated/src/models.js"; import { AgentDeletionStatusOutput, AgentOutput, OpenAIPageableListOfAgentOutput } from "../generated/src/outputModels.js"; import { CreateAgentParameters, DeleteAgentParameters, GetAgentParameters, ListAgentsParameters, UpdateAgentParameters } from "../generated/src/parameters.js"; -import { setSpanAttributes, TracingAttributeOptions, tracingClient, TrackingOperationName } from "../tracing.js"; - +import { TracingUtility} from "../tracing.js"; +import { traceEndCreateAgent, traceStartCreateAgent } from "./assistantsTrace.js"; const expectedStatuses = ["200"]; @@ -15,17 +14,16 @@ export async function createAgent( context: Client, options: CreateAgentParameters ): Promise { - return tracingClient.withSpan("createAgent", options, async (updatedOptions, span) => { - setSpanAttributes(span, getAgentAttributes(options.body)) - const result = await context.path("/assistants").post(updatedOptions); - - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - const resultBody = result.body as AgentOutput; - setSpanAttributes(span, getAgentResultAttributes(resultBody)); - return resultBody; - }); + return TracingUtility.withSpan("CreateAgent", options, + async (updatedOptions) => { + const result = await context.path("/assistants").post(updatedOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body as AgentOutput; + }, + traceStartCreateAgent, traceEndCreateAgent, + ); } /** Gets a list of agents that were previously created. */ @@ -87,47 +85,3 @@ export async function deleteAgent( } return result.body; } - -/** - * Extracts tracing attributes from the agent creation options. - * @param body - The options for creating an agent. - * @returns The tracing attributes. - */ -function getAgentAttributes(body: CreateAgentOptions): TracingAttributeOptions { - return { - operationName: TrackingOperationName.CREATE_AGENT, - name: body.name, - model: body.model, - description: body.description, - instructions: body.instructions, - topP: body.top_p, - responseFormat: formatAgentApiResponse(body.response_format) - }; -} - -/** - * Extracts tracing attributes from the agent output. - * @param output - The output of the agent. - * @returns TracingAttributeOptions The tracing attributes. - */ -function getAgentResultAttributes(output: AgentOutput): TracingAttributeOptions { - return { - operationName: TrackingOperationName.CREATE_AGENT, - agentId: output.id, - }; -} - -/** - * Formats the agent API response. - * @param responseFormat - The response format option. - * @returns The formatted response as a string, or null/undefined. - */ -function formatAgentApiResponse(responseFormat: AgentsApiResponseFormatOption | null | undefined): string | null | undefined { - if (typeof responseFormat === "string" || responseFormat === undefined || responseFormat === null) { - return responseFormat; - } - if ((responseFormat as AgentsApiResponseFormat).type) { - return (responseFormat as AgentsApiResponseFormat).type; - } - return undefined; -} diff --git a/sdk/ai/ai-projects/src/agents/assistantsTrace.ts b/sdk/ai/ai-projects/src/agents/assistantsTrace.ts new file mode 100644 index 000000000000..f6fc66ee3537 --- /dev/null +++ b/sdk/ai/ai-projects/src/agents/assistantsTrace.ts @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { TracingSpan } from "@azure/core-tracing"; +import { AgentOutput } from "../generated/src/outputModels.js"; +import { TracingAttributeOptions, TracingAttributes, TracingUtility, TrackingOperationName } from "../tracing.js"; +import { CreateAgentParameters } from "../generated/src/parameters.js"; +import { addInstructionsEvent, formatAgentApiResponse } from "./traceUtility.js"; + +/** + * Traces the start of creating an agent. + * @param span - The span to trace. + * @param options - The options for creating an agent. + */ +export function traceStartCreateAgent(span: Omit, options: CreateAgentParameters): void { + const attributes: TracingAttributeOptions = { + operationName: TrackingOperationName.CREATE_AGENT, + name: options.body.name, + model: options.body.model, + description: options.body.description, + instructions: options.body.instructions, + topP: options.body.top_p, + temperature: options.body.temperature, + responseFormat: formatAgentApiResponse(options.body.response_format), + genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM + }; + TracingUtility.setSpanAttributes(span,TrackingOperationName.CREATE_AGENT, attributes) + addInstructionsEvent(span, options.body); +} + + +/** + * Traces the end of creating an agent. + * @param span - The span to trace. + * @param _options - The options for creating an agent. + * @param result - The result of creating an agent. + */ +export async function traceEndCreateAgent(span: Omit, _options: CreateAgentParameters, result: Promise): Promise { + const resolvedResult = await result; + const attributes: TracingAttributeOptions = { + agentId: resolvedResult.id, + }; + TracingUtility.updateSpanAttributes(span, attributes); +} diff --git a/sdk/ai/ai-projects/src/agents/messages.ts b/sdk/ai/ai-projects/src/agents/messages.ts index be0b624aa28a..e1a7bb40ab99 100644 --- a/sdk/ai/ai-projects/src/agents/messages.ts +++ b/sdk/ai/ai-projects/src/agents/messages.ts @@ -4,6 +4,8 @@ import { Client, createRestError } from "@azure-rest/core-client"; import { OpenAIPageableListOfThreadMessageOutput, ThreadMessageOutput } from "../generated/src/outputModels.js"; import { CreateMessageParameters, ListMessagesParameters, UpdateMessageParameters } from "../generated/src/parameters.js"; +import { TracingUtility } from "../tracing.js"; +import { traceEndCreateMessage, traceStartCreateMessage } from "./messagesTrace.js"; const expectedStatuses = ["200"]; @@ -13,13 +15,15 @@ export async function createMessage( threadId: string, options: CreateMessageParameters, ): Promise { - const result = await context - .path("/threads/{threadId}/messages", threadId) - .post(options); - if (!expectedStatuses.includes(result.status)) { + return TracingUtility.withSpan("CreateMessage", options, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/messages", threadId) + .post(updateOptions); + if (!expectedStatuses.includes(result.status)) { throw createRestError(result); - } - return result.body; + } + return result.body; + }, (span, updatedOptions) => traceStartCreateMessage(span, threadId, updatedOptions), traceEndCreateMessage); } /** Gets a list of messages that exist on a thread. */ @@ -32,7 +36,7 @@ export async function listMessages( .path("/threads/{threadId}/messages", threadId) .get(options); if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); + throw createRestError(result); } return result.body; } @@ -48,7 +52,7 @@ export async function updateMessage( .path("/threads/{threadId}/messages/{messageId}", threadId, messageId) .post(options); if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); + throw createRestError(result); } return result.body; } diff --git a/sdk/ai/ai-projects/src/agents/messagesTrace.ts b/sdk/ai/ai-projects/src/agents/messagesTrace.ts new file mode 100644 index 000000000000..28258cf52435 --- /dev/null +++ b/sdk/ai/ai-projects/src/agents/messagesTrace.ts @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { TracingSpan } from "@azure/core-tracing"; +import { CreateMessageParameters } from "../generated/src/parameters.js"; +import { TracingAttributes, TracingUtility, TrackingOperationName } from "../tracing.js"; +import { ThreadMessageOutput } from "../generated/src/outputModels.js"; +import { addMessageEvent } from "./traceUtility.js"; + +export function traceStartCreateMessage(span: Omit, threadId: string, options: CreateMessageParameters): void { + TracingUtility.setSpanAttributes(span, TrackingOperationName.CREATE_MESSAGE, { threadId: threadId, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }); + addMessageEvent(span, { ...options.body, thread_id: threadId }); +} + +export async function traceEndCreateMessage(span: Omit, _options: CreateMessageParameters, result: Promise): Promise { + const resolvedResult = await result; + TracingUtility.updateSpanAttributes(span, { messageId: resolvedResult.id }); +} diff --git a/sdk/ai/ai-projects/src/agents/runTrace.ts b/sdk/ai/ai-projects/src/agents/runTrace.ts new file mode 100644 index 000000000000..1a6a19370265 --- /dev/null +++ b/sdk/ai/ai-projects/src/agents/runTrace.ts @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { TracingSpan } from "@azure/core-tracing"; +import { CreateRunParameters, CreateThreadAndRunParameters } from "../generated/src/parameters.js"; +import { ThreadRunOutput } from "../generated/src/outputModels.js"; +import { TracingAttributeOptions, TracingAttributes, TracingUtility, TrackingOperationName } from "../tracing.js"; +import { addInstructionsEvent, addMessageEvent, formatAgentApiResponse } from "./traceUtility.js"; + +export function traceStartCreateRun(span: Omit, options: CreateRunParameters | CreateThreadAndRunParameters, threadId?: string, operationName: string = TrackingOperationName.CREATE_RUN): void { + + const attributes: TracingAttributeOptions = { + threadId: threadId, + agentId: options.body.assistant_id, + model: options.body.model, + instructions: options.body.instructions, + temperature: options.body.temperature, + topP: options.body.top_p, + maxCompletionTokens: options.body.max_completion_tokens, + maxPromptTokens: options.body.max_prompt_tokens, + responseFormat: formatAgentApiResponse(options.body.response_format), + genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM + } + if((options as CreateRunParameters).body.additional_instructions) { + attributes.additional_instructions = (options as CreateRunParameters).body.additional_instructions; + } + TracingUtility.setSpanAttributes(span, operationName, attributes); + setSpanEvents(span, options); +} + +export function traceStartCreateThreadAndRun(span: Omit, options: CreateThreadAndRunParameters): void { + traceStartCreateRun(span, options, undefined, TrackingOperationName.CREATE_THREAD_RUN); +} + +export async function traceEndCreateRun(span: Omit, _options: CreateRunParameters, result: Promise): Promise { + const resolvedResult = await result; + TracingUtility.updateSpanAttributes(span, { runId: resolvedResult.id, runStatus: resolvedResult.status, responseModel: resolvedResult.model, usageCompletionTokens: resolvedResult.usage?.completion_tokens, usagePromptTokens: resolvedResult.usage?.prompt_tokens }); +} + +function setSpanEvents(span: Omit, options: CreateRunParameters): void { + addInstructionsEvent(span, { ...options.body, agentId: options.body.assistant_id }); + options.body.additional_messages?.forEach((message) => { + addMessageEvent(span, message); + }); +} diff --git a/sdk/ai/ai-projects/src/agents/runs.ts b/sdk/ai/ai-projects/src/agents/runs.ts index 0d00ca51d393..549e56ac4424 100644 --- a/sdk/ai/ai-projects/src/agents/runs.ts +++ b/sdk/ai/ai-projects/src/agents/runs.ts @@ -4,6 +4,8 @@ import { Client, createRestError } from "@azure-rest/core-client"; import { CancelRunParameters, CreateRunParameters, CreateThreadAndRunParameters, GetRunParameters, ListRunsParameters, SubmitToolOutputsToRunParameters, UpdateRunParameters } from "../generated/src/parameters.js"; import { OpenAIPageableListOfThreadRunOutput, ThreadRunOutput } from "../generated/src/outputModels.js"; +import { TracingUtility } from "../tracing.js"; +import { traceEndCreateRun, traceStartCreateRun, traceStartCreateThreadAndRun } from "./runTrace.js"; const expectedStatuses = ["200"]; @@ -14,13 +16,15 @@ export async function createRun( options: CreateRunParameters, ): Promise { options.body.stream = false; - const result = await context - .path("/threads/{threadId}/runs", threadId) - .post(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("CreateRun", options, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/runs", threadId) + .post(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartCreateRun(span, updatedOptions, threadId), traceEndCreateRun); } /** Gets a list of runs for a specified thread. */ @@ -109,9 +113,11 @@ export async function createThreadAndRun( options: CreateThreadAndRunParameters, ): Promise { options.body.stream = false; - const result = await context.path("/threads/runs").post(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("CreateThreadAndRun", options, async (updateOptions) => { + const result = await context.path("/threads/runs").post(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, traceStartCreateThreadAndRun, traceEndCreateRun); } diff --git a/sdk/ai/ai-projects/src/agents/threads.ts b/sdk/ai/ai-projects/src/agents/threads.ts index 5a34addb5e6f..604f15819bd6 100644 --- a/sdk/ai/ai-projects/src/agents/threads.ts +++ b/sdk/ai/ai-projects/src/agents/threads.ts @@ -4,6 +4,8 @@ import { Client, createRestError } from "@azure-rest/core-client"; import { CreateThreadParameters, DeleteThreadParameters, GetThreadParameters, UpdateThreadParameters } from "../generated/src/parameters.js"; import { AgentThreadOutput, ThreadDeletionStatusOutput } from "../generated/src/outputModels.js"; +import { TracingUtility } from "../tracing.js"; +import { traceEndCreateThread, traceStartCreateThread } from "./threadsTrace.js"; const expectedStatuses = ["200"]; @@ -12,11 +14,13 @@ export async function createThread( context: Client, options?: CreateThreadParameters, ): Promise { - const result = await context.path("/threads").post(options); + return TracingUtility.withSpan("CreateThread", options || {body: {}}, async (updatedOptions) => { + const result = await context.path("/threads").post(updatedOptions); if (!expectedStatuses.includes(result.status)) { throw createRestError(result); } return result.body; +}, traceStartCreateThread, traceEndCreateThread); } /** Gets information about an existing thread. */ diff --git a/sdk/ai/ai-projects/src/agents/threadsTrace.ts b/sdk/ai/ai-projects/src/agents/threadsTrace.ts new file mode 100644 index 000000000000..845fd254e2f5 --- /dev/null +++ b/sdk/ai/ai-projects/src/agents/threadsTrace.ts @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { TracingSpan } from "@azure/core-tracing"; +import { AgentThreadOutput } from "../generated/src/outputModels.js"; +import { TracingAttributes, TracingUtility, TrackingOperationName } from "../tracing.js"; +import { CreateThreadParameters } from "../generated/src/parameters.js"; +import { addMessageEvent } from "./traceUtility.js"; + +export function traceStartCreateThread(span: Omit, options: CreateThreadParameters): void { + TracingUtility.setSpanAttributes(span, TrackingOperationName.CREATE_THREAD, { genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }); + setSpanEvents(span, options); +} + +export async function traceEndCreateThread(span: Omit, _options: CreateThreadParameters, result: Promise): Promise { + const resolvedResult = await result; + TracingUtility.updateSpanAttributes(span, { threadId: resolvedResult.id }); +} + +function setSpanEvents(span: Omit, options: CreateThreadParameters): void { + options.body.messages?.forEach((message) => { + addMessageEvent(span, message); + }); +} diff --git a/sdk/ai/ai-projects/src/agents/traceUtility.ts b/sdk/ai/ai-projects/src/agents/traceUtility.ts new file mode 100644 index 000000000000..982d73e15575 --- /dev/null +++ b/sdk/ai/ai-projects/src/agents/traceUtility.ts @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { TracingSpan } from "@azure/core-tracing"; +import { AgentsApiResponseFormat, AgentsApiResponseFormatOption, MessageContent, ThreadMessage, ThreadMessageOptions } from "./inputOutputs.js"; +import { TracingAttributes, TracingUtility } from "../tracing.js"; +import { getTelemetryOptions } from "../telemetry/telemetry.js"; + +/** + * Adds a message event to the span. + * @param span - The span to add the event to. + * @param messageAttributes - The attributes of the message event. + */ +export function addMessageEvent(span: Omit, messageAttributes: ThreadMessageOptions | ThreadMessage,): void { + + const eventBody: Record = {}; + const telemetryOptions = getTelemetryOptions() + if (telemetryOptions.enableContentRecording) { + eventBody.content = getMessageContent(messageAttributes.content); + } + eventBody.role = messageAttributes.role; + if (messageAttributes.attachments) { + eventBody.attachments = messageAttributes.attachments.map((attachment) => { + return { + id: attachment.file_id, + tools: attachment.tools.map((tool) => tool.type) + }; + }); + } + const threadId = (messageAttributes as ThreadMessage).thread_id; + const agentId = (messageAttributes as ThreadMessage).assistant_id; + const threadRunId = (messageAttributes as ThreadMessage).run_id; + const messageStatus = (messageAttributes as ThreadMessage).status; + const attributes = { eventContent: JSON.stringify(eventBody), threadId, agentId, threadRunId, messageStatus, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }; + TracingUtility.addSpanEvent(span, `gen_ai.${messageAttributes.role}.message`, attributes); +} + +/** + * Adds an instruction event to the span. + * @param span - The span to add the event to. + * @param instructionAttributes - The attributes of the instruction event. + */ +export function addInstructionsEvent(span: Omit, instructionAttributes: { instructions?: string | null, additional_instructions?: string | null, threadId?: string, agentId?: string }): void { + const eventBody: Record = {}; + eventBody.content = instructionAttributes.instructions && instructionAttributes.additional_instructions ? `${instructionAttributes.instructions} ${instructionAttributes.additional_instructions}` : instructionAttributes.instructions || instructionAttributes.additional_instructions; + const attributes = { eventContent: JSON.stringify(eventBody), threadId: instructionAttributes.threadId, agentId: instructionAttributes.agentId, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }; + TracingUtility.addSpanEvent(span, "gen_ai.system.message", attributes); +} + +/** + * Formats the agent API response. + * @param responseFormat - The response format option. + * @returns The formatted response as a string, or null/undefined. + */ +export function formatAgentApiResponse(responseFormat: AgentsApiResponseFormatOption | null | undefined): string | null | undefined { + if (typeof responseFormat === "string" || responseFormat === undefined || responseFormat === null) { + return responseFormat; + } + if ((responseFormat as AgentsApiResponseFormat).type) { + return (responseFormat as AgentsApiResponseFormat).type; + } + return undefined; +} + + +function getMessageContent(messageContent: string | MessageContent[]): string | {} { + type MessageContentExtended = MessageContent & { [key: string]: any; }; + if (!Array.isArray(messageContent)) { + return messageContent; + } + const contentBody: { [key: string]: any } = {}; + messageContent.forEach(content => { + const typedContent = content.type; + const contentDetails: { value: any, annotations?: string[] } = { value: (content as MessageContentExtended)[typedContent] }; + const annotations = contentDetails.value.annotations; + if (annotations) { + contentDetails.annotations = annotations; + } + contentBody[typedContent] = contentDetails; + }); + return contentBody; +} diff --git a/sdk/ai/ai-projects/src/aiProjectsClient.ts b/sdk/ai/ai-projects/src/aiProjectsClient.ts index 1c0492ff4347..38b09c559f75 100644 --- a/sdk/ai/ai-projects/src/aiProjectsClient.ts +++ b/sdk/ai/ai-projects/src/aiProjectsClient.ts @@ -5,6 +5,7 @@ import createClient, { ProjectsClientOptions } from "./generated/src/projectsCli import { Client } from "@azure-rest/core-client"; import { AgentsOperations, getAgentsOperations } from "./agents/index.js"; import { ConnectionsOperations, getConnectionsOperations } from "./connections/index.js"; +import { getTelemetryOperations, TelemetryOperations } from "./telemetry/index.js"; export interface AIProjectsClientOptions extends ProjectsClientOptions{ } @@ -44,6 +45,7 @@ export class AIProjectsClient { {...options, endpoint:connectionEndPoint}) this.agents = getAgentsOperations(this._client); this.connections = getConnectionsOperations(this._connectionClient); + this.telemetry = getTelemetryOperations(this.connections); } /** @@ -90,4 +92,7 @@ export class AIProjectsClient { /** The operation groups for connections */ public readonly connections: ConnectionsOperations; + + /** The operation groups for telemetry */ + public readonly telemetry: TelemetryOperations; } diff --git a/sdk/ai/ai-projects/src/index.ts b/sdk/ai/ai-projects/src/index.ts index 72bd951fe365..0d5b48cbd826 100644 --- a/sdk/ai/ai-projects/src/index.ts +++ b/sdk/ai/ai-projects/src/index.ts @@ -5,6 +5,7 @@ import { AIProjectsClient, AIProjectsClientOptions } from "./aiProjectsClient.js import { ProjectsClientOptions } from "./generated/src/projectsClient.js" export { AgentsOperations } from "./agents/index.js"; export { ConnectionsOperations } from "./connections/index.js"; +export { TelemetryOperations } from "./telemetry/index.js"; export * from "./agents/inputOutputs.js"; export * from "./connections/inputOutput.js"; diff --git a/sdk/ai/ai-projects/src/telemetry/index.ts b/sdk/ai/ai-projects/src/telemetry/index.ts new file mode 100644 index 000000000000..d9fcd9856eee --- /dev/null +++ b/sdk/ai/ai-projects/src/telemetry/index.ts @@ -0,0 +1,37 @@ + +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + + +import { ConnectionsOperations } from "../connections/index.js"; +import {getConnectionString, TelemetryOptions, updateTelemetryOptions } from "./telemetry.js"; + +/** + * Telemetry operations + **/ +export interface TelemetryOperations { + /** + * Get the appinsights connection string confired in the workspace + * @returns The telemetry connection string + **/ + getConnectionString() : Promise + + /** + * Update the telemetry settings + * @param options - The telemetry options + * @returns void + * */ + updateSettings(options: TelemetryOptions) : void +} + +/** + * Get the telemetry operations + * @param connection - The connections operations + * @returns The telemetry operations + **/ +export function getTelemetryOperations(connection: ConnectionsOperations): TelemetryOperations { + return { + getConnectionString: () => getConnectionString(connection), + updateSettings : (options: TelemetryOptions) => updateTelemetryOptions(options) + } +} diff --git a/sdk/ai/ai-projects/src/telemetry/telemetry.ts b/sdk/ai/ai-projects/src/telemetry/telemetry.ts new file mode 100644 index 000000000000..d9ca3843f560 --- /dev/null +++ b/sdk/ai/ai-projects/src/telemetry/telemetry.ts @@ -0,0 +1,51 @@ + +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { ConnectionsOperations } from "../connections/index.js"; + +/** + * Telemetry options + */ +export interface TelemetryOptions { + enableContentRecording: boolean; +} + +const telemetryOptions: TelemetryOptions & { connectionString: string | unknown } = { + enableContentRecording: false, + connectionString: null +} + +/** + * Update the telemetry settings + * @param options - The telemetry options + */ +export function updateTelemetryOptions(options: TelemetryOptions): void { + telemetryOptions.enableContentRecording = options.enableContentRecording; +}; + +/** + * Get the telemetry options + * @returns The telemetry options + */ +export function getTelemetryOptions(): TelemetryOptions { + return telemetryOptions; +}; + +/** + * Get the appinsights connection string confired in the workspace + * @param connection - get the connection string + * @returns The telemetry connection string + */ +export async function getConnectionString(connection: ConnectionsOperations): Promise { + if (telemetryOptions.connectionString) { + const workspace = await connection.getWorkspace(); + if (workspace.properties.applicationInsights) { + telemetryOptions.connectionString = workspace.properties.applicationInsights + } + else { + throw new Error("Application Insights connection string not found.") + } + } + return telemetryOptions.connectionString as string; +} diff --git a/sdk/ai/ai-projects/src/tracing.ts b/sdk/ai/ai-projects/src/tracing.ts index 7c02fdbb2ade..ecc35bfcd914 100644 --- a/sdk/ai/ai-projects/src/tracing.ts +++ b/sdk/ai/ai-projects/src/tracing.ts @@ -1,13 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { createTracingClient, TracingSpan } from "@azure/core-tracing"; - -export const tracingClient = createTracingClient({ - namespace: "Microsoft.CognitiveServices", - packageName: "@azure/ai-projects", - packageVersion: "1.0.0-beta.1", -}); +import { createTracingClient, OperationTracingOptions, Resolved, TracingSpan } from "@azure/core-tracing"; export enum TracingAttributes { GEN_AI_MESSAGE_ID = "gen_ai.message.id", @@ -41,15 +35,16 @@ export enum TrackingOperationName { CREATE_AGENT = "create_agent", CREATE_THREAD = "create_thread", CREATE_MESSAGE = "create_message", + CREATE_RUN = "create_run", START_THREAD_RUN = "start_thread_run", EXECUTE_TOOL = "execute_tool", LIST_MESSAGES = "list_messages", SUBMIT_TOOL_OUTPUTS = "submit_tool_outputs", - PROCESS_THREAD_RUN = "process_thread_run", + CREATE_THREAD_RUN = "create_thread_run", } export interface TracingAttributeOptions { - operationName: string; + operationName?: string; name?: string | null; description?: string | null; serverAddress?: string | null; @@ -58,6 +53,8 @@ export interface TracingAttributeOptions { instructions?: string | null; additional_instructions?: string | null; runId?: string | null; + runStatus?: string | null; + responseModel?: string | null; model?: string | null; temperature?: number | null; topP?: number | null; @@ -65,82 +62,201 @@ export interface TracingAttributeOptions { maxCompletionTokens?: number | null; responseFormat?: string | null; genAiSystem?: string | null; + messageId?: string | null; + messageStatus?: string | null; + eventContent?: string | null; + usagePromptTokens?: number | null; + usageCompletionTokens?: number | null; } -export function setSpanAttributes( - span: Omit, - attributeOptions: TracingAttributeOptions -): void { - if (span) { - const { - operationName, - name, - description, - serverAddress, - threadId, - agentId, - runId, - model, - temperature, - topP, - maxPromptTokens, - maxCompletionTokens, - responseFormat, - genAiSystem = "AZ_AI_AGENT_SYSTEM" - } = attributeOptions; - - if (genAiSystem) { - span.setAttribute(TracingAttributes.GEN_AI_SYSTEM, genAiSystem); - } - - span.setAttribute(TracingAttributes.GEN_AI_OPERATION_NAME, operationName); - - if (name) { - span.setAttribute(TracingAttributes.GEN_AI_AGENT_NAME, name); - } - if (description) { - span.setAttribute(TracingAttributes.GEN_AI_AGENT_DESCRIPTION, description); - } - - if (serverAddress) { - span.setAttribute(TracingAttributes.SERVER_ADDRESS, serverAddress); - } - - if (threadId) { - span.setAttribute(TracingAttributes.GEN_AI_THREAD_ID, threadId); - } +export class TracingUtility { + private static tracingClient = createTracingClient({ + namespace: "Microsoft.CognitiveServices", + packageName: "@azure/ai-projects", + packageVersion: "1.0.0-beta.1", - if (agentId) { - span.setAttribute(TracingAttributes.GEN_AI_AGENT_ID, agentId); - } - - if (runId) { - span.setAttribute(TracingAttributes.GEN_AI_THREAD_RUN_ID, runId); - } + }); - if (model) { - span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MODEL, model); - } + static async withSpan ReturnType>(name: string, options: Options, request: Request, + startTrace?: (span: Omit, updatedOptions: Options,) => void, + endTrace?: (span: Omit, updatedOptions: Options, result: ReturnType) => void, + ): Promise>> { + return TracingUtility.tracingClient.withSpan(name, options, async (updatedOptions: Options, span: Omit) => { + if (startTrace) { + startTrace(span, updatedOptions); + } + const result = request(updatedOptions); + if (endTrace) { + endTrace(span, updatedOptions, result); + } + return result; + }, { spanKind: "client" }); + } - if (temperature !== null) { - span.setAttribute(TracingAttributes.GEN_AI_REQUEST_TEMPERATURE, temperature); + static updateSpanAttributes(span: Omit, attributeOptions: Omit): void { + TracingUtility.setAttributes(span, attributeOptions); + } + + static setSpanAttributes( + span: Omit, + operationName: string, + attributeOptions: TracingAttributeOptions + ): void { + attributeOptions.operationName = operationName; + TracingUtility.setAttributes(span, attributeOptions); + } + + static setAttributes( + span: Omit, + attributeOptions: TracingAttributeOptions + ): void { + if (span.isRecording()) { + const { + name, + operationName, + description, + serverAddress, + threadId, + agentId, + messageId, + runId, + model, + temperature, + topP, + maxPromptTokens, + maxCompletionTokens, + responseFormat, + runStatus, + responseModel, + usageCompletionTokens, + usagePromptTokens, + genAiSystem = TracingAttributes.AZ_AI_AGENT_SYSTEM + } = attributeOptions; + + if (genAiSystem) { + span.setAttribute(TracingAttributes.GEN_AI_SYSTEM, genAiSystem); + } + if (name) { + span.setAttribute(TracingAttributes.GEN_AI_AGENT_NAME, name); + } + if (description) { + span.setAttribute(TracingAttributes.GEN_AI_AGENT_DESCRIPTION, description); + } + + if (serverAddress) { + span.setAttribute(TracingAttributes.SERVER_ADDRESS, serverAddress); + } + + if (threadId) { + span.setAttribute(TracingAttributes.GEN_AI_THREAD_ID, threadId); + } + + if (agentId) { + span.setAttribute(TracingAttributes.GEN_AI_AGENT_ID, agentId); + } + + if (runId) { + span.setAttribute(TracingAttributes.GEN_AI_THREAD_RUN_ID, runId); + } + + if (messageId) { + span.setAttribute(TracingAttributes.GEN_AI_MESSAGE_ID, messageId); + } + if (model) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MODEL, model); + } + + if (temperature !== null) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_TEMPERATURE, temperature); + } + + if (topP !== null) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_TOP_P, topP); + } + + if (maxPromptTokens !== null) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MAX_INPUT_TOKENS, maxPromptTokens); + } + + if (maxCompletionTokens !== null) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MAX_OUTPUT_TOKENS, maxCompletionTokens); + } + + if (responseFormat) { + span.setAttribute(TracingAttributes.GEN_AI_REQUEST_RESPONSE_FORMAT, responseFormat); + } + + if (runStatus) { + span.setAttribute(TracingAttributes.GEN_AI_THREAD_RUN_STATUS, runStatus); + } + + if (responseModel) { + span.setAttribute(TracingAttributes.GEN_AI_RESPONSE_MODEL, responseModel); + } + + if (usagePromptTokens) { + span.setAttribute(TracingAttributes.GEN_AI_USAGE_INPUT_TOKENS, usagePromptTokens); + } + + if (usageCompletionTokens) { + span.setAttribute(TracingAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, usageCompletionTokens); + } + if(operationName) { + span.setAttribute(TracingAttributes.GEN_AI_OPERATION_NAME, operationName); + } } - - if (topP !== null) { - span.setAttribute(TracingAttributes.GEN_AI_REQUEST_TOP_P, topP); + return; + } + + static addSpanEvent( + span: Omit, + eventName: string, + attributeOptions: Omit + ): void { + if (span.isRecording()) { + const { threadId, agentId, runId, messageId, eventContent, usageCompletionTokens, usagePromptTokens, messageStatus } = attributeOptions; + const attributes: Record = {}; + + if (eventContent) { + attributes[TracingAttributes.GEN_AI_EVENT_CONTENT] = eventContent; + } + + if (threadId) { + attributes[TracingAttributes.GEN_AI_THREAD_ID] = threadId; + } + + if (agentId) { + attributes[TracingAttributes.GEN_AI_AGENT_ID] = agentId; + } + + if (runId) { + attributes[TracingAttributes.GEN_AI_THREAD_RUN_ID] = runId; + } + + if (messageId) { + attributes[TracingAttributes.GEN_AI_MESSAGE_ID] = messageId; + } + if (messageStatus) { + attributes[TracingAttributes.GEN_AI_MESSAGE_STATUS] = messageStatus; + } + + if (usagePromptTokens) { + attributes[TracingAttributes.GEN_AI_USAGE_INPUT_TOKENS] = usagePromptTokens; + } + + if (usageCompletionTokens) { + attributes[TracingAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] = usageCompletionTokens; + } + + if (span.addEvent) { + span.addEvent(eventName, { attributes }); + } } + return; + } + +} - if (maxPromptTokens !== null) { - span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MAX_INPUT_TOKENS, maxPromptTokens); - } - if (maxCompletionTokens !== null) { - span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MAX_OUTPUT_TOKENS, maxCompletionTokens); - } - if (responseFormat) { - span.setAttribute(TracingAttributes.GEN_AI_REQUEST_RESPONSE_FORMAT, responseFormat); - } - } - return; -} diff --git a/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts b/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts new file mode 100644 index 000000000000..8a512e1741f0 --- /dev/null +++ b/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { AgentOutput, AgentsOperations, AgentThreadOutput, AIProjectsClient, ThreadMessageOutput, ThreadRunOutput } from "../../../src/index.js"; +import { createMockProjectsClient } from "../utils/createClient.js"; +import { assert, beforeEach, afterEach, it, describe, vi } from "vitest"; +import { MockInstrumenter, MockTracingSpan } from "@azure-tools/test-utils-vitest"; +import { AddEventOptions, Instrumenter, InstrumenterSpanOptions, TracingContext, TracingSpan, useInstrumenter } from "@azure/core-tracing"; + +interface ExtendedMockTrackingSpan extends MockTracingSpan { + events?: { name: string, attributes: Record }[] + addEvent?(eventName: string, options?: AddEventOptions): void; +} +class ExtendedMockInstrumenter extends MockInstrumenter { + extendSpan(span: any): void { + span.addEvent = (eventName: string, options?: AddEventOptions) => { + if (!span.events) { + span.events = []; + } + span.events.push({ name: eventName, ...options }); + } + } + startSpan(name: string, + spanOptions?: InstrumenterSpanOptions): { span: TracingSpan; tracingContext: TracingContext } { + const { span, tracingContext } = super.startSpan(name, spanOptions); + this.extendSpan(span); + return { span, tracingContext } + } + +} + +describe("Agent Tracing", () => { + let instrumenter: Instrumenter; + let projectsClient: AIProjectsClient; + let agents: AgentsOperations; + let response: any = {}; + let status = 200; + beforeEach(async function () { + instrumenter = new ExtendedMockInstrumenter() + useInstrumenter(instrumenter); + + projectsClient = createMockProjectsClient(() => { return { bodyAsText: JSON.stringify(response), status: status } }); + agents = projectsClient.agents + }); + + afterEach(async function () { + (instrumenter as MockInstrumenter).reset(); + vi.clearAllMocks(); + response = {}; + status = 200; + }); + + it("create agent", async function () { + const agentResponse: Partial = { id: "agentId", object: "assistant" }; + response = agentResponse; + status = 200; + const request = { name: "agentName", instructions: "You are helpful agent", response_format: "json" }; + const model = "gpt-4o"; + await agents.createAgent(model, request); + const mockedInstrumenter = instrumenter as MockInstrumenter; + assert.isAbove(mockedInstrumenter.startedSpans.length, 0); + const span = mockedInstrumenter.startedSpans[0] as ExtendedMockTrackingSpan; + assert.equal(span.attributes["gen_ai.agent.id"], agentResponse.id); + assert.equal(span.attributes["gen_ai.operation.name"], "create_agent"); + assert.equal(span.attributes["gen_ai.agent.name"], request.name); + assert.equal(span.attributes["gen_ai.request.model"], model); + const event = span.events?.find(e => e.name === "gen_ai.system.message"); + assert.isDefined(event); + assert.equal(event?.attributes["gen_ai.event.content"], JSON.stringify({ content: request.instructions })); + + }) + + it("create run", async function () { + const runResponse : Partial = { id: "runId", object: "thread.run", status: "queued", thread_id: "threadId", assistant_id: "agentId" }; + response = runResponse; + status = 200; + await agents.createRun("threadId", "agentId"); + const mockedInstrumenter = instrumenter as MockInstrumenter; + assert.isAbove(mockedInstrumenter.startedSpans.length, 0); + const span = mockedInstrumenter.startedSpans[0]; + assert.equal(span.attributes["gen_ai.thread.id"], runResponse.thread_id); + assert.equal(span.attributes["gen_ai.operation.name"], "create_run"); + assert.equal(span.attributes["gen_ai.agent.id"], runResponse.assistant_id); + assert.equal(span.attributes["gen_ai.thread.run.status"], runResponse.status); + + }) + + it("create Thread", async function () { + const threadResponse: Partial = { id: "threadId", object: "thread" }; + response = threadResponse; + status = 200; + await agents.createThread(); + const mockedInstrumenter = instrumenter as MockInstrumenter; + assert.isAbove(mockedInstrumenter.startedSpans.length, 0); + const span = mockedInstrumenter.startedSpans[0]; + assert.equal(span.attributes["gen_ai.thread.id"], threadResponse.id); + assert.equal(span.attributes["gen_ai.operation.name"], "create_thread"); + }) + + it("create Message", async function () { + const messageResponse: Partial = { id: "messageId", object: "thread.message", thread_id: "threadId" }; + projectsClient.telemetry.updateSettings({ enableContentRecording: true }); + response = messageResponse; + status = 200; + const request = { content: "hello, world!", role: "user" }; + await agents.createMessage("threadId", request); + const mockedInstrumenter = instrumenter as MockInstrumenter; + assert.isAbove(mockedInstrumenter.startedSpans.length, 0); + const span = mockedInstrumenter.startedSpans[0] as ExtendedMockTrackingSpan; + assert.equal(span.attributes["gen_ai.thread.id"], messageResponse.thread_id); + assert.equal(span.attributes["gen_ai.operation.name"], "create_message"); + const event = span.events?.find(e => e.name === "gen_ai.user.message"); + assert.isDefined(event); + assert.equal(event?.attributes["gen_ai.event.content"], JSON.stringify(request)); + assert.equal(event?.attributes["gen_ai.thread.id"], messageResponse.thread_id); + }) + + +}); diff --git a/sdk/ai/ai-projects/test/public/utils/createClient.ts b/sdk/ai/ai-projects/test/public/utils/createClient.ts index d1a8cd58edd6..ec0c5337aa04 100644 --- a/sdk/ai/ai-projects/test/public/utils/createClient.ts +++ b/sdk/ai/ai-projects/test/public/utils/createClient.ts @@ -9,6 +9,7 @@ import { import { createTestCredential } from "@azure-tools/test-credential"; import { AIProjectsClient } from "../../../src/index.js"; import { ClientOptions } from "@azure-rest/core-client"; +import { createHttpHeaders, PipelineRequest, PipelineResponse } from "@azure/core-rest-pipeline"; const replaceableVariables: Record = { SUBSCRIPTION_ID: "azure_subscription_id", @@ -37,5 +38,26 @@ export function createProjectsClient( ): AIProjectsClient { const credential = createTestCredential(); const connectionString = process.env["AZURE_AI_PROJECTS_CONNECTION_STRING"] || ""; - return AIProjectsClient.fromConnectionString(connectionString, credential, recorder?.configureClientOptions(options ?? {})); + return AIProjectsClient.fromConnectionString( + connectionString, + credential, + recorder ? recorder.configureClientOptions(options ?? {}) : options + ); +} + +export function createMockProjectsClient(responseFn: (request:PipelineRequest) => Partial): AIProjectsClient { + const options: ClientOptions = { additionalPolicies: [] }; + options.additionalPolicies?.push({ + policy: { + name: "RequestMockPolicy", + sendRequest:(async (req) => { + const response = responseFn(req); + return { headers: createHttpHeaders(), status: 200, request: req, ...response } as PipelineResponse; + }) + }, + position: "perCall" + }); + const credential = createTestCredential(); + const connectionString = process.env["AZURE_AI_PROJECTS_CONNECTION_STRING"] || ""; + return AIProjectsClient.fromConnectionString(connectionString, credential, options); } From 92c00a5c7edfe1ecc2d2f04766d637a0df42f0c9 Mon Sep 17 00:00:00 2001 From: Ganesh Bheemarasetty Date: Tue, 3 Dec 2024 15:37:32 -0800 Subject: [PATCH 3/9] Tracing for submit tools and list messages --- common/config/rush/pnpm-lock.yaml | 31 ++++++- sdk/ai/ai-projects/package.json | 1 + .../agents/agents_basics_tracing_console.ts | 8 +- sdk/ai/ai-projects/src/agents/messages.ts | 18 ++-- .../ai-projects/src/agents/messagesTrace.ts | 15 +++- sdk/ai/ai-projects/src/agents/runTrace.ts | 19 ++++- sdk/ai/ai-projects/src/agents/runs.ts | 18 ++-- sdk/ai/ai-projects/src/agents/traceUtility.ts | 33 ++++++-- sdk/ai/ai-projects/src/tracing.ts | 82 +++++++++++-------- .../test/public/agents/tracing.spec.ts | 40 ++++++++- 10 files changed, 197 insertions(+), 68 deletions(-) diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index abc331da3c7a..3fb9acbf11aa 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -1618,6 +1618,26 @@ packages: - supports-color dev: false + /@azure/monitor-opentelemetry-exporter@1.0.0-beta.27: + resolution: {integrity: sha512-21iXu9ubtPB7iO3ghnzMMdB0KwHpz7Zl1a9xSBR3Gl8IDUlXOBjMn6OT9+ycj9VZrTyEKiV59T9VTf0IlokPYQ==} + engines: {node: '>=18.0.0'} + dependencies: + '@azure/core-auth': 1.8.0 + '@azure/core-client': 1.9.2 + '@azure/core-rest-pipeline': 1.17.0 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.53.0 + '@opentelemetry/core': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.53.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 1.26.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.27.0 + tslib: 2.7.0 + transitivePeerDependencies: + - supports-color + dev: false + /@azure/msal-browser@3.26.1: resolution: {integrity: sha512-y78sr9g61aCAH9fcLO1um+oHFXc1/5Ap88RIsUSuzkm0BHzFnN+PXGaQeuM1h5Qf5dTnWNOd6JqkskkMPAhh7Q==} engines: {node: '>=0.8.0'} @@ -10804,6 +10824,12 @@ packages: hasBin: true dev: false + /typescript@5.7.2: + resolution: {integrity: sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg==} + engines: {node: '>=14.17'} + hasBin: true + dev: false + /ua-parser-js@0.7.39: resolution: {integrity: sha512-IZ6acm6RhQHNibSt7+c09hhvsKy9WUr4DVbeq9U8o71qxyYtJpQeDxQnMrVqnIFMLcQjHO0I9wgfO2vIahht4w==} hasBin: true @@ -12118,10 +12144,11 @@ packages: dev: false file:projects/ai-projects.tgz: - resolution: {integrity: sha512-AteUpToB0cbZyNZ3+DwqN/EI2BeqnfqbOhLL6W5LeLPHOrKsG84ujxOFjTPedPrqrJSvb08UtwRwvam4BXnBTA==, tarball: file:projects/ai-projects.tgz} + resolution: {integrity: sha512-oRCe+nAcU2VJ9Cg/5+cW6f/NBbxdogZHJUFVNv9o9GlnFjK/Y7UmWyiFjWCMxc+KeL5Mu8EoYAfEJ14ZqttXWQ==, tarball: file:projects/ai-projects.tgz} name: '@rush-temp/ai-projects' version: 0.0.0 dependencies: + '@azure/monitor-opentelemetry-exporter': 1.0.0-beta.27 '@microsoft/api-extractor': 7.47.9(@types/node@18.19.55) '@opentelemetry/api': 1.9.0 '@opentelemetry/instrumentation': 0.53.0(@opentelemetry/api@1.9.0) @@ -21161,7 +21188,7 @@ packages: name: '@rush-temp/dev-tool' version: 0.0.0 dependencies: - '@_ts/max': /typescript@5.6.3 + '@_ts/max': /typescript@5.7.2 '@_ts/min': /typescript@4.2.4 '@azure/identity': 4.4.1 '@eslint/js': 9.12.0 diff --git a/sdk/ai/ai-projects/package.json b/sdk/ai/ai-projects/package.json index 800949bc6b6c..5e2df056522d 100644 --- a/sdk/ai/ai-projects/package.json +++ b/sdk/ai/ai-projects/package.json @@ -70,6 +70,7 @@ "@azure/eslint-plugin-azure-sdk": "^3.0.0", "@azure/identity": "^4.3.0", "@azure/opentelemetry-instrumentation-azure-sdk": "^1.0.0-beta.7", + "@azure/monitor-opentelemetry-exporter": "^1.0.0-beta.27", "@azure-tools/test-credential": "^2.0.0", "@azure-tools/test-recorder": "^4.1.0", "@azure-tools/test-utils-vitest": "^1.0.0", diff --git a/sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts b/sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts index 78c39eea7a6f..6024d82e4051 100644 --- a/sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts +++ b/sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts @@ -5,6 +5,7 @@ import { context } from "@opentelemetry/api"; import { registerInstrumentations } from "@opentelemetry/instrumentation"; import { createAzureSdkInstrumentation } from "@azure/opentelemetry-instrumentation-azure-sdk"; +import { AzureMonitorTraceExporter } from "@azure/monitor-opentelemetry-exporter" import { ConsoleSpanExporter, NodeTracerProvider, @@ -28,9 +29,14 @@ import { DefaultAzureCredential } from "@azure/identity"; const connectionString = process.env["AZURE_AI_PROJECTS_CONNECTION_STRING"] || ">;;;"; export async function main(): Promise { - + const client = AIProjectsClient.fromConnectionString(connectionString || "", new DefaultAzureCredential()); + const appInsightsConnectionString = await client.telemetry.getConnectionString(); + if (appInsightsConnectionString) { + const exporter = new AzureMonitorTraceExporter({ connectionString: appInsightsConnectionString }); + provider.addSpanProcessor(new SimpleSpanProcessor(exporter)); + } const agent = await client.agents.createAgent("gpt-4o", { name: "my-agent", instructions: "You are helpful agent" }, { tracingOptions: { tracingContext: context.active() } }); diff --git a/sdk/ai/ai-projects/src/agents/messages.ts b/sdk/ai/ai-projects/src/agents/messages.ts index e1a7bb40ab99..952446ba1bd6 100644 --- a/sdk/ai/ai-projects/src/agents/messages.ts +++ b/sdk/ai/ai-projects/src/agents/messages.ts @@ -5,7 +5,7 @@ import { Client, createRestError } from "@azure-rest/core-client"; import { OpenAIPageableListOfThreadMessageOutput, ThreadMessageOutput } from "../generated/src/outputModels.js"; import { CreateMessageParameters, ListMessagesParameters, UpdateMessageParameters } from "../generated/src/parameters.js"; import { TracingUtility } from "../tracing.js"; -import { traceEndCreateMessage, traceStartCreateMessage } from "./messagesTrace.js"; +import { traceEndCreateMessage, traceEndListMessages, traceStartCreateMessage, traceStartListMessages } from "./messagesTrace.js"; const expectedStatuses = ["200"]; @@ -32,13 +32,15 @@ export async function listMessages( threadId: string, options?: ListMessagesParameters, ): Promise { - const result = await context - .path("/threads/{threadId}/messages", threadId) - .get(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("ListMessages", options || {}, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/messages", threadId) + .get(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartListMessages(span, threadId, updatedOptions), traceEndListMessages); } /** Modifies an existing message on an existing thread. */ diff --git a/sdk/ai/ai-projects/src/agents/messagesTrace.ts b/sdk/ai/ai-projects/src/agents/messagesTrace.ts index 28258cf52435..afee98d3c7ac 100644 --- a/sdk/ai/ai-projects/src/agents/messagesTrace.ts +++ b/sdk/ai/ai-projects/src/agents/messagesTrace.ts @@ -2,9 +2,9 @@ // Licensed under the MIT License. import { TracingSpan } from "@azure/core-tracing"; -import { CreateMessageParameters } from "../generated/src/parameters.js"; +import { CreateMessageParameters, ListMessagesParameters } from "../generated/src/parameters.js"; import { TracingAttributes, TracingUtility, TrackingOperationName } from "../tracing.js"; -import { ThreadMessageOutput } from "../generated/src/outputModels.js"; +import { OpenAIPageableListOfThreadMessageOutput, ThreadMessageOutput } from "../generated/src/outputModels.js"; import { addMessageEvent } from "./traceUtility.js"; export function traceStartCreateMessage(span: Omit, threadId: string, options: CreateMessageParameters): void { @@ -16,3 +16,14 @@ export async function traceEndCreateMessage(span: Omit, _opt const resolvedResult = await result; TracingUtility.updateSpanAttributes(span, { messageId: resolvedResult.id }); } + +export function traceStartListMessages(span: Omit, threadId: string, _options: ListMessagesParameters) : void { + TracingUtility.setSpanAttributes(span, TrackingOperationName.LIST_MESSAGES, { threadId: threadId, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }); +} + +export async function traceEndListMessages(span: Omit, _options: ListMessagesParameters, result: Promise) : Promise { + const resolvedResult = await result; + resolvedResult.data?.forEach(message => { + addMessageEvent(span, message) + }); +} diff --git a/sdk/ai/ai-projects/src/agents/runTrace.ts b/sdk/ai/ai-projects/src/agents/runTrace.ts index 1a6a19370265..6cdba83495c6 100644 --- a/sdk/ai/ai-projects/src/agents/runTrace.ts +++ b/sdk/ai/ai-projects/src/agents/runTrace.ts @@ -2,10 +2,10 @@ // Licensed under the MIT License. import { TracingSpan } from "@azure/core-tracing"; -import { CreateRunParameters, CreateThreadAndRunParameters } from "../generated/src/parameters.js"; +import { CreateRunParameters, CreateThreadAndRunParameters, SubmitToolOutputsToRunParameters } from "../generated/src/parameters.js"; import { ThreadRunOutput } from "../generated/src/outputModels.js"; import { TracingAttributeOptions, TracingAttributes, TracingUtility, TrackingOperationName } from "../tracing.js"; -import { addInstructionsEvent, addMessageEvent, formatAgentApiResponse } from "./traceUtility.js"; +import { addInstructionsEvent, addMessageEvent, addToolMessagesEvent, formatAgentApiResponse } from "./traceUtility.js"; export function traceStartCreateRun(span: Omit, options: CreateRunParameters | CreateThreadAndRunParameters, threadId?: string, operationName: string = TrackingOperationName.CREATE_RUN): void { @@ -21,7 +21,7 @@ export function traceStartCreateRun(span: Omit, options: Cre responseFormat: formatAgentApiResponse(options.body.response_format), genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM } - if((options as CreateRunParameters).body.additional_instructions) { + if ((options as CreateRunParameters).body.additional_instructions) { attributes.additional_instructions = (options as CreateRunParameters).body.additional_instructions; } TracingUtility.setSpanAttributes(span, operationName, attributes); @@ -37,9 +37,22 @@ export async function traceEndCreateRun(span: Omit, _options TracingUtility.updateSpanAttributes(span, { runId: resolvedResult.id, runStatus: resolvedResult.status, responseModel: resolvedResult.model, usageCompletionTokens: resolvedResult.usage?.completion_tokens, usagePromptTokens: resolvedResult.usage?.prompt_tokens }); } +export function traceStartSubmitToolOutputsToRun(span: Omit, options: SubmitToolOutputsToRunParameters, threadId: string, + runId: string,): void { + const attributes: TracingAttributeOptions = { threadId: threadId, runId: runId } + TracingUtility.setSpanAttributes(span, TrackingOperationName.SUBMIT_TOOL_OUTPUTS, attributes); + addToolMessagesEvent(span, options.body.tool_outputs); +} + +export async function traceEndSubmitToolOutputsToRun(span: Omit, _options: SubmitToolOutputsToRunParameters, result: Promise): Promise { + const resolvedResult = await result; + TracingUtility.updateSpanAttributes(span, { runId: resolvedResult.id, runStatus: resolvedResult.status, responseModel: resolvedResult.model, usageCompletionTokens: resolvedResult.usage?.completion_tokens, usagePromptTokens: resolvedResult.usage?.prompt_tokens }); +} + function setSpanEvents(span: Omit, options: CreateRunParameters): void { addInstructionsEvent(span, { ...options.body, agentId: options.body.assistant_id }); options.body.additional_messages?.forEach((message) => { addMessageEvent(span, message); }); } + diff --git a/sdk/ai/ai-projects/src/agents/runs.ts b/sdk/ai/ai-projects/src/agents/runs.ts index 549e56ac4424..ac83b3a6162f 100644 --- a/sdk/ai/ai-projects/src/agents/runs.ts +++ b/sdk/ai/ai-projects/src/agents/runs.ts @@ -5,7 +5,7 @@ import { Client, createRestError } from "@azure-rest/core-client"; import { CancelRunParameters, CreateRunParameters, CreateThreadAndRunParameters, GetRunParameters, ListRunsParameters, SubmitToolOutputsToRunParameters, UpdateRunParameters } from "../generated/src/parameters.js"; import { OpenAIPageableListOfThreadRunOutput, ThreadRunOutput } from "../generated/src/outputModels.js"; import { TracingUtility } from "../tracing.js"; -import { traceEndCreateRun, traceStartCreateRun, traceStartCreateThreadAndRun } from "./runTrace.js"; +import { traceEndCreateRun, traceEndSubmitToolOutputsToRun, traceStartCreateRun, traceStartCreateThreadAndRun, traceStartSubmitToolOutputsToRun } from "./runTrace.js"; const expectedStatuses = ["200"]; @@ -82,13 +82,15 @@ export async function submitToolOutputsToRun( options: SubmitToolOutputsToRunParameters, ): Promise { options.body.stream = false; - const result = await context - .path("/threads/{threadId}/runs/{runId}/submit_tool_outputs", threadId, runId) - .post(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("SubmitToolOutputsToRun", options, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/runs/{runId}/submit_tool_outputs", threadId, runId) + .post(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartSubmitToolOutputsToRun(span, updatedOptions, threadId, runId), traceEndSubmitToolOutputsToRun); } /** Cancels a run of an in progress thread. */ diff --git a/sdk/ai/ai-projects/src/agents/traceUtility.ts b/sdk/ai/ai-projects/src/agents/traceUtility.ts index 982d73e15575..6b5c723e27f5 100644 --- a/sdk/ai/ai-projects/src/agents/traceUtility.ts +++ b/sdk/ai/ai-projects/src/agents/traceUtility.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import { TracingSpan } from "@azure/core-tracing"; -import { AgentsApiResponseFormat, AgentsApiResponseFormatOption, MessageContent, ThreadMessage, ThreadMessageOptions } from "./inputOutputs.js"; +import { AgentsApiResponseFormat, AgentsApiResponseFormatOption, MessageContent, RunStepCompletionUsageOutput, ThreadMessage, ThreadMessageOptions, ToolOutput } from "./inputOutputs.js"; import { TracingAttributes, TracingUtility } from "../tracing.js"; import { getTelemetryOptions } from "../telemetry/telemetry.js"; @@ -11,7 +11,7 @@ import { getTelemetryOptions } from "../telemetry/telemetry.js"; * @param span - The span to add the event to. * @param messageAttributes - The attributes of the message event. */ -export function addMessageEvent(span: Omit, messageAttributes: ThreadMessageOptions | ThreadMessage,): void { +export function addMessageEvent(span: Omit, messageAttributes: ThreadMessageOptions | ThreadMessage, usage?: RunStepCompletionUsageOutput): void { const eventBody: Record = {}; const telemetryOptions = getTelemetryOptions() @@ -31,7 +31,14 @@ export function addMessageEvent(span: Omit, messageAttribute const agentId = (messageAttributes as ThreadMessage).assistant_id; const threadRunId = (messageAttributes as ThreadMessage).run_id; const messageStatus = (messageAttributes as ThreadMessage).status; - const attributes = { eventContent: JSON.stringify(eventBody), threadId, agentId, threadRunId, messageStatus, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }; + const messageId = (messageAttributes as ThreadMessage).id; + const incompleteDetails = (messageAttributes as ThreadMessage).incomplete_details + if (incompleteDetails) { + eventBody.incomplete_details = incompleteDetails; + } + const usagePromptTokens = usage?.prompt_tokens; + const usageCompletionTokens = usage?.completion_tokens; + const attributes = { eventContent: JSON.stringify(eventBody), threadId, agentId, threadRunId, messageStatus, messageId, usagePromptTokens, usageCompletionTokens, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }; TracingUtility.addSpanEvent(span, `gen_ai.${messageAttributes.role}.message`, attributes); } @@ -62,6 +69,18 @@ export function formatAgentApiResponse(responseFormat: AgentsApiResponseFormatOp return undefined; } +/** + * Adds a tool messages event to the span + * @param span - The span to add the event to. + * @param tool_outputs - List of tool oupts + */ +export function addToolMessagesEvent(span: Omit, tool_outputs: Array): void { + tool_outputs.forEach(tool_output => { + const eventBody = {"content": tool_output.output, "id": tool_output.tool_call_id} + TracingUtility.addSpanEvent(span, "gen_ai.tool.message", {eventContent: JSON.stringify(eventBody), genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM}); + }); + +} function getMessageContent(messageContent: string | MessageContent[]): string | {} { type MessageContentExtended = MessageContent & { [key: string]: any; }; @@ -71,12 +90,8 @@ function getMessageContent(messageContent: string | MessageContent[]): string | const contentBody: { [key: string]: any } = {}; messageContent.forEach(content => { const typedContent = content.type; - const contentDetails: { value: any, annotations?: string[] } = { value: (content as MessageContentExtended)[typedContent] }; - const annotations = contentDetails.value.annotations; - if (annotations) { - contentDetails.annotations = annotations; - } - contentBody[typedContent] = contentDetails; + const {value, annotations} = (content as MessageContentExtended)[typedContent]; + contentBody[typedContent] = {value, annotations}; }); return contentBody; } diff --git a/sdk/ai/ai-projects/src/tracing.ts b/sdk/ai/ai-projects/src/tracing.ts index ecc35bfcd914..ade40ec34b2e 100644 --- a/sdk/ai/ai-projects/src/tracing.ts +++ b/sdk/ai/ai-projects/src/tracing.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { createTracingClient, OperationTracingOptions, Resolved, TracingSpan } from "@azure/core-tracing"; +import { createTracingClient, OperationTracingOptions, Resolved, SpanStatusError, TracingSpan } from "@azure/core-tracing"; export enum TracingAttributes { GEN_AI_MESSAGE_ID = "gen_ai.message.id", @@ -84,11 +84,27 @@ export class TracingUtility { ): Promise>> { return TracingUtility.tracingClient.withSpan(name, options, async (updatedOptions: Options, span: Omit) => { if (startTrace) { - startTrace(span, updatedOptions); + try { + startTrace(span, updatedOptions); + } + catch { /* empty */ } + + } + let result: ReturnType | undefined; + try { + result = await request(updatedOptions); + } catch (error) { + const errorStatus: SpanStatusError = { status: "error" } + if (error instanceof Error) { + errorStatus.error = error; + } + throw error; } - const result = request(updatedOptions); - if (endTrace) { - endTrace(span, updatedOptions, result); + + if (endTrace && result !== undefined) { + try { + endTrace(span, updatedOptions, result); + } catch { /* empty */ } } return result; }, { spanKind: "client" }); @@ -97,7 +113,7 @@ export class TracingUtility { static updateSpanAttributes(span: Omit, attributeOptions: Omit): void { TracingUtility.setAttributes(span, attributeOptions); } - + static setSpanAttributes( span: Omit, operationName: string, @@ -106,7 +122,7 @@ export class TracingUtility { attributeOptions.operationName = operationName; TracingUtility.setAttributes(span, attributeOptions); } - + static setAttributes( span: Omit, attributeOptions: TracingAttributeOptions @@ -133,7 +149,7 @@ export class TracingUtility { usagePromptTokens, genAiSystem = TracingAttributes.AZ_AI_AGENT_SYSTEM } = attributeOptions; - + if (genAiSystem) { span.setAttribute(TracingAttributes.GEN_AI_SYSTEM, genAiSystem); } @@ -143,72 +159,72 @@ export class TracingUtility { if (description) { span.setAttribute(TracingAttributes.GEN_AI_AGENT_DESCRIPTION, description); } - + if (serverAddress) { span.setAttribute(TracingAttributes.SERVER_ADDRESS, serverAddress); } - + if (threadId) { span.setAttribute(TracingAttributes.GEN_AI_THREAD_ID, threadId); } - + if (agentId) { span.setAttribute(TracingAttributes.GEN_AI_AGENT_ID, agentId); } - + if (runId) { span.setAttribute(TracingAttributes.GEN_AI_THREAD_RUN_ID, runId); } - + if (messageId) { span.setAttribute(TracingAttributes.GEN_AI_MESSAGE_ID, messageId); } if (model) { span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MODEL, model); } - + if (temperature !== null) { span.setAttribute(TracingAttributes.GEN_AI_REQUEST_TEMPERATURE, temperature); } - + if (topP !== null) { span.setAttribute(TracingAttributes.GEN_AI_REQUEST_TOP_P, topP); } - + if (maxPromptTokens !== null) { span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MAX_INPUT_TOKENS, maxPromptTokens); } - + if (maxCompletionTokens !== null) { span.setAttribute(TracingAttributes.GEN_AI_REQUEST_MAX_OUTPUT_TOKENS, maxCompletionTokens); } - + if (responseFormat) { span.setAttribute(TracingAttributes.GEN_AI_REQUEST_RESPONSE_FORMAT, responseFormat); } - + if (runStatus) { span.setAttribute(TracingAttributes.GEN_AI_THREAD_RUN_STATUS, runStatus); } - + if (responseModel) { span.setAttribute(TracingAttributes.GEN_AI_RESPONSE_MODEL, responseModel); } - + if (usagePromptTokens) { span.setAttribute(TracingAttributes.GEN_AI_USAGE_INPUT_TOKENS, usagePromptTokens); } - + if (usageCompletionTokens) { span.setAttribute(TracingAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, usageCompletionTokens); } - if(operationName) { + if (operationName) { span.setAttribute(TracingAttributes.GEN_AI_OPERATION_NAME, operationName); } } return; } - + static addSpanEvent( span: Omit, eventName: string, @@ -217,45 +233,45 @@ export class TracingUtility { if (span.isRecording()) { const { threadId, agentId, runId, messageId, eventContent, usageCompletionTokens, usagePromptTokens, messageStatus } = attributeOptions; const attributes: Record = {}; - + if (eventContent) { attributes[TracingAttributes.GEN_AI_EVENT_CONTENT] = eventContent; } - + if (threadId) { attributes[TracingAttributes.GEN_AI_THREAD_ID] = threadId; } - + if (agentId) { attributes[TracingAttributes.GEN_AI_AGENT_ID] = agentId; } - + if (runId) { attributes[TracingAttributes.GEN_AI_THREAD_RUN_ID] = runId; } - + if (messageId) { attributes[TracingAttributes.GEN_AI_MESSAGE_ID] = messageId; } if (messageStatus) { attributes[TracingAttributes.GEN_AI_MESSAGE_STATUS] = messageStatus; } - + if (usagePromptTokens) { attributes[TracingAttributes.GEN_AI_USAGE_INPUT_TOKENS] = usagePromptTokens; } - + if (usageCompletionTokens) { attributes[TracingAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] = usageCompletionTokens; } - + if (span.addEvent) { span.addEvent(eventName, { attributes }); } } return; } - + } diff --git a/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts b/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts index 8a512e1741f0..bd6f1740a325 100644 --- a/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts +++ b/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { AgentOutput, AgentsOperations, AgentThreadOutput, AIProjectsClient, ThreadMessageOutput, ThreadRunOutput } from "../../../src/index.js"; +import { AgentOutput, AgentsOperations, AgentThreadOutput, AIProjectsClient, OpenAIPageableListOfThreadMessageOutput, ThreadMessageOutput, ThreadRunOutput } from "../../../src/index.js"; import { createMockProjectsClient } from "../utils/createClient.js"; import { assert, beforeEach, afterEach, it, describe, vi } from "vitest"; import { MockInstrumenter, MockTracingSpan } from "@azure-tools/test-utils-vitest"; @@ -71,7 +71,7 @@ describe("Agent Tracing", () => { }) it("create run", async function () { - const runResponse : Partial = { id: "runId", object: "thread.run", status: "queued", thread_id: "threadId", assistant_id: "agentId" }; + const runResponse: Partial = { id: "runId", object: "thread.run", status: "queued", thread_id: "threadId", assistant_id: "agentId" }; response = runResponse; status = 200; await agents.createRun("threadId", "agentId"); @@ -113,7 +113,43 @@ describe("Agent Tracing", () => { assert.isDefined(event); assert.equal(event?.attributes["gen_ai.event.content"], JSON.stringify(request)); assert.equal(event?.attributes["gen_ai.thread.id"], messageResponse.thread_id); + assert.equal(event?.name, "gen_ai.user.message"); }) + it("list messages", async function () { + const listMessages = { object: "list", data: [{ id: "messageId", object: "thread.message", thread_id: "threadId", role: "assistant", content: [{ type: "text", text: { value: "You are helpful agent" } }] }, { id: "messageId2", object: "thread.message", thread_id: "threadId", role: "user", content: [{ type: "text", text: { value: "Hello, tell me a joke" } }] }] }; + response = listMessages; + projectsClient.telemetry.updateSettings({ enableContentRecording: true }); + status = 200; + await agents.listMessages("threadId"); + const mockedInstrumenter = instrumenter as MockInstrumenter; + assert.isAbove(mockedInstrumenter.startedSpans.length, 0); + const span = mockedInstrumenter.startedSpans[0] as ExtendedMockTrackingSpan; + assert.isDefined(span.events); + assert.equal(span.events!.length, 2); + assert.equal(span.events![0].attributes["gen_ai.event.content"], JSON.stringify({ content: { text: listMessages.data[0].content[0].text }, role: listMessages.data[0].role })); + assert.equal(span.events![0].name, "gen_ai.assistant.message"); + assert.equal(span.events![1].attributes["gen_ai.event.content"], JSON.stringify({ content: { text: listMessages.data[1].content[0].text }, role: listMessages.data[1].role })); + assert.equal(span.events![1].name, "gen_ai.user.message"); + }) + it("Submit tool outputs to run", async function () { + const submitToolOutputs = { object: "thread.run", id: "runId", status: "queued", thread_id: "threadId", assistant_id: "agentId" }; + response = submitToolOutputs; + status = 200; + const toolOutputs = [{ tool_call_id: "toolcallId1", output: "output1" }, { tool_call_id: "toolcallId2", output: "output2" }]; + await agents.submitToolOutputsToRun("threadId", "runId", toolOutputs); + const mockedInstrumenter = instrumenter as MockInstrumenter; + assert.isAbove(mockedInstrumenter.startedSpans.length, 0); + const span = mockedInstrumenter.startedSpans[0] as ExtendedMockTrackingSpan; + assert.equal(span.attributes["gen_ai.thread.id"], submitToolOutputs.thread_id); + assert.equal(span.attributes["gen_ai.operation.name"], "submit_tool_outputs"); + assert.equal(span.attributes["gen_ai.thread.run.status"], submitToolOutputs.status); + assert.isDefined(span.events); + assert.equal(span.events!.length, 2); + assert.equal(span.events![0].attributes["gen_ai.event.content"], JSON.stringify({ content: toolOutputs[0].output, id: toolOutputs[0].tool_call_id })); + assert.equal(span.events![0].name, "gen_ai.tool.message"); + assert.equal(span.events![1].attributes["gen_ai.event.content"], JSON.stringify({ content: toolOutputs[1].output, id: toolOutputs[1].tool_call_id })); + assert.equal(span.events![1].name, "gen_ai.tool.message"); + }); }); From 8fdf4effc8e3db253546d79894c7e4473546d43c Mon Sep 17 00:00:00 2001 From: Ganesh Bheemarasetty Date: Wed, 4 Dec 2024 15:28:22 -0800 Subject: [PATCH 4/9] address comments --- ...ics_tracing_console.ts => agentCreateWithTracingCconsole.ts} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename sdk/ai/ai-projects/samples-dev/agents/{agents_basics_tracing_console.ts => agentCreateWithTracingCconsole.ts} (97%) diff --git a/sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts b/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingCconsole.ts similarity index 97% rename from sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts rename to sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingCconsole.ts index 6024d82e4051..8032eb3f950b 100644 --- a/sdk/ai/ai-projects/samples-dev/agents/agents_basics_tracing_console.ts +++ b/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingCconsole.ts @@ -43,7 +43,7 @@ export async function main(): Promise { console.log(`Created agent, agent ID : ${agent.id}`); - client.agents.deleteAgent(agent.id); + await client.agents.deleteAgent(agent.id); console.log(`Deleted agent`); } From fdd2d84d58e5b62212ec974e240229d347583450 Mon Sep 17 00:00:00 2001 From: Ganesh Bheemarasetty Date: Thu, 5 Dec 2024 10:41:05 -0800 Subject: [PATCH 5/9] added test and recording for telemetry --- sdk/ai/ai-projects/assets.json | 2 +- sdk/ai/ai-projects/src/telemetry/index.ts | 12 ++++- sdk/ai/ai-projects/src/telemetry/telemetry.ts | 12 ++++- .../test/public/agents/tracing.spec.ts | 2 +- .../test/public/telemetry/telemetry.spec.ts | 48 +++++++++++++++++++ .../test/public/utils/createClient.ts | 6 +++ 6 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 sdk/ai/ai-projects/test/public/telemetry/telemetry.spec.ts diff --git a/sdk/ai/ai-projects/assets.json b/sdk/ai/ai-projects/assets.json index 559ec443b07f..a41ad3fe7b2c 100644 --- a/sdk/ai/ai-projects/assets.json +++ b/sdk/ai/ai-projects/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "js", "TagPrefix": "js/ai/ai-projects", - "Tag": "js/ai/ai-projects_34268fa552" + "Tag": "js/ai/ai-projects_e25d256f48" } diff --git a/sdk/ai/ai-projects/src/telemetry/index.ts b/sdk/ai/ai-projects/src/telemetry/index.ts index d9fcd9856eee..8d10a28050bc 100644 --- a/sdk/ai/ai-projects/src/telemetry/index.ts +++ b/sdk/ai/ai-projects/src/telemetry/index.ts @@ -4,7 +4,7 @@ import { ConnectionsOperations } from "../connections/index.js"; -import {getConnectionString, TelemetryOptions, updateTelemetryOptions } from "./telemetry.js"; +import {getConnectionString, getTelemetryOptions, resetTelemetryOptions, TelemetryOptions, updateTelemetryOptions } from "./telemetry.js"; /** * Telemetry operations @@ -22,6 +22,12 @@ export interface TelemetryOperations { * @returns void * */ updateSettings(options: TelemetryOptions) : void + + /** + * get the telemetry settings + * @returns The telemetry options + * */ + getSettings() : TelemetryOptions } /** @@ -30,8 +36,10 @@ export interface TelemetryOperations { * @returns The telemetry operations **/ export function getTelemetryOperations(connection: ConnectionsOperations): TelemetryOperations { + resetTelemetryOptions(); return { getConnectionString: () => getConnectionString(connection), - updateSettings : (options: TelemetryOptions) => updateTelemetryOptions(options) + updateSettings : (options: TelemetryOptions) => updateTelemetryOptions(options), + getSettings : () => getTelemetryOptions() } } diff --git a/sdk/ai/ai-projects/src/telemetry/telemetry.ts b/sdk/ai/ai-projects/src/telemetry/telemetry.ts index d9ca3843f560..b540bce8007a 100644 --- a/sdk/ai/ai-projects/src/telemetry/telemetry.ts +++ b/sdk/ai/ai-projects/src/telemetry/telemetry.ts @@ -29,16 +29,24 @@ export function updateTelemetryOptions(options: TelemetryOptions): void { * @returns The telemetry options */ export function getTelemetryOptions(): TelemetryOptions { - return telemetryOptions; + return structuredClone(telemetryOptions); }; +/** + * Reset the telemetry options + */ +export function resetTelemetryOptions(): void { + telemetryOptions.connectionString = null; + telemetryOptions.enableContentRecording = false; +} + /** * Get the appinsights connection string confired in the workspace * @param connection - get the connection string * @returns The telemetry connection string */ export async function getConnectionString(connection: ConnectionsOperations): Promise { - if (telemetryOptions.connectionString) { + if (!telemetryOptions.connectionString) { const workspace = await connection.getWorkspace(); if (workspace.properties.applicationInsights) { telemetryOptions.connectionString = workspace.properties.applicationInsights diff --git a/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts b/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts index bd6f1740a325..15037464e840 100644 --- a/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts +++ b/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { AgentOutput, AgentsOperations, AgentThreadOutput, AIProjectsClient, OpenAIPageableListOfThreadMessageOutput, ThreadMessageOutput, ThreadRunOutput } from "../../../src/index.js"; +import { AgentOutput, AgentsOperations, AgentThreadOutput, AIProjectsClient, ThreadMessageOutput, ThreadRunOutput } from "../../../src/index.js"; import { createMockProjectsClient } from "../utils/createClient.js"; import { assert, beforeEach, afterEach, it, describe, vi } from "vitest"; import { MockInstrumenter, MockTracingSpan } from "@azure-tools/test-utils-vitest"; diff --git a/sdk/ai/ai-projects/test/public/telemetry/telemetry.spec.ts b/sdk/ai/ai-projects/test/public/telemetry/telemetry.spec.ts new file mode 100644 index 000000000000..b600fea073c8 --- /dev/null +++ b/sdk/ai/ai-projects/test/public/telemetry/telemetry.spec.ts @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Recorder, VitestTestContext } from "@azure-tools/test-recorder"; +import { AIProjectsClient, TelemetryOperations } from "../../../src/index.js"; +import { createRecorder, createProjectsClient } from "../utils/createClient.js"; +import { assert, beforeEach, afterEach, it, describe } from "vitest"; + +describe("AI Projects - Telemetry", () => { + let recorder: Recorder; + let projectsClient: AIProjectsClient; + let telemetry: TelemetryOperations + + beforeEach(async function (context: VitestTestContext) { + recorder = await createRecorder(context); + projectsClient = createProjectsClient(recorder); + telemetry = projectsClient.telemetry; + }); + + afterEach(async function () { + await recorder.stop(); + }); + it("client and connection operations are accessible", async function () { + assert.isNotNull(projectsClient); + assert.isNotNull(telemetry); + }); + + + + it("update settings", async function () { + assert.equal(telemetry.getSettings().enableContentRecording, false); + telemetry.updateSettings({ enableContentRecording: true }); + assert.equal(telemetry.getSettings().enableContentRecording, true); + }); + + it("get settings", async function () { + const options = telemetry.getSettings(); + assert.equal(options.enableContentRecording, false); + options.enableContentRecording = true; + assert.equal(telemetry.getSettings().enableContentRecording, false); + }); + + it("get app insights connection string", async function () { + const connectionString = await telemetry.getConnectionString(); + assert.isNotEmpty(connectionString); + }); + +}); diff --git a/sdk/ai/ai-projects/test/public/utils/createClient.ts b/sdk/ai/ai-projects/test/public/utils/createClient.ts index f9a45d884670..432c54fdeaaf 100644 --- a/sdk/ai/ai-projects/test/public/utils/createClient.ts +++ b/sdk/ai/ai-projects/test/public/utils/createClient.ts @@ -12,6 +12,8 @@ import { ClientOptions } from "@azure-rest/core-client"; import { createHttpHeaders, PipelineRequest, PipelineResponse } from "@azure/core-rest-pipeline"; const replaceableVariables: Record = { + GENERIC_STRING: "Sanitized", + ENDPOINT: "Sanitized.api.azureml.ms", SUBSCRIPTION_ID: "00000000-0000-0000-0000-000000000000", RESOURCE_GROUP_NAME: "00000", WORKSPACE_NAME: "00000", @@ -29,6 +31,10 @@ const recorderEnvSetup: RecorderStartOptions = { { regex: true, target: "(%2F|/)?subscriptions(%2F|/)([-\\w\\._\\(\\)]+)", value: replaceableVariables.SUBSCRIPTION_ID, groupForReplace: "3" }, { regex: true, target: "(%2F|/)?resource[gG]roups(%2F|/)([-\\w\\._\\(\\)]+)", value: replaceableVariables.RESOURCE_GROUP_NAME, groupForReplace: "3" }, { regex: true, target: "/workspaces/([-\\w\\._\\(\\)]+)", value: replaceableVariables.WORKSPACE_NAME, groupForReplace: "1" }, + { regex: true, target: "/userAssignedIdentities/([-\\w\\._\\(\\)]+)", value: replaceableVariables.GENERIC_STRING, groupForReplace: "1" }, + { regex: true, target: "/components/([-\\w\\._\\(\\)]+)", value: replaceableVariables.GENERIC_STRING, groupForReplace: "1" }, + { regex: true, target: "/vaults/([-\\w\\._\\(\\)]+)", value: replaceableVariables.GENERIC_STRING, groupForReplace: "1" }, + { regex: true, target: "(azureml|http|https):\\/\\/([^\\/]+)", value: replaceableVariables.ENDPOINT, groupForReplace: "2" }, ], bodyKeySanitizers: [ { jsonPath: "properties.ConnectionString", value: "InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://region.applicationinsights.azure.com/;LiveEndpoint=https://region.livediagnostics.monitor.azure.com/;ApplicationId=00000000-0000-0000-0000-000000000000"}, From 50e29b49906e09f8ad74d4d419dcbbf9bb873bec Mon Sep 17 00:00:00 2001 From: Ganesh Bheemarasetty Date: Thu, 5 Dec 2024 11:00:49 -0800 Subject: [PATCH 6/9] fixed an error wrt to playback --- sdk/ai/ai-projects/assets.json | 2 +- sdk/ai/ai-projects/test/public/utils/createClient.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/ai/ai-projects/assets.json b/sdk/ai/ai-projects/assets.json index a41ad3fe7b2c..083f689ef59a 100644 --- a/sdk/ai/ai-projects/assets.json +++ b/sdk/ai/ai-projects/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "js", "TagPrefix": "js/ai/ai-projects", - "Tag": "js/ai/ai-projects_e25d256f48" + "Tag": "js/ai/ai-projects_1b6cffbf79" } diff --git a/sdk/ai/ai-projects/test/public/utils/createClient.ts b/sdk/ai/ai-projects/test/public/utils/createClient.ts index 432c54fdeaaf..328c31f3db96 100644 --- a/sdk/ai/ai-projects/test/public/utils/createClient.ts +++ b/sdk/ai/ai-projects/test/public/utils/createClient.ts @@ -13,7 +13,7 @@ import { createHttpHeaders, PipelineRequest, PipelineResponse } from "@azure/cor const replaceableVariables: Record = { GENERIC_STRING: "Sanitized", - ENDPOINT: "Sanitized.api.azureml.ms", + ENDPOINT: "Sanitized.azure.com", SUBSCRIPTION_ID: "00000000-0000-0000-0000-000000000000", RESOURCE_GROUP_NAME: "00000", WORKSPACE_NAME: "00000", From 80b3186253a92db82749c64754e1ea2e4c645f08 Mon Sep 17 00:00:00 2001 From: Ganesh Bheemarasetty Date: Thu, 5 Dec 2024 13:07:09 -0800 Subject: [PATCH 7/9] Address PR comments --- sdk/ai/ai-projects/package.json | 6 +- sdk/ai/ai-projects/review/ai-projects.api.md | 1 + sdk/ai/ai-projects/src/agents/assistants.ts | 2 +- .../ai-projects/src/agents/assistantsTrace.ts | 21 +++-- .../ai-projects/src/agents/messagesTrace.ts | 15 ++-- sdk/ai/ai-projects/src/agents/runTrace.ts | 33 ++++---- sdk/ai/ai-projects/src/agents/threadsTrace.ts | 11 ++- sdk/ai/ai-projects/src/agents/traceUtility.ts | 17 ++-- sdk/ai/ai-projects/src/constants.ts | 12 +++ sdk/ai/ai-projects/src/logger.ts | 6 ++ sdk/ai/ai-projects/src/telemetry/telemetry.ts | 6 +- sdk/ai/ai-projects/src/tracing.ts | 77 +++++++++++-------- 12 files changed, 117 insertions(+), 90 deletions(-) create mode 100644 sdk/ai/ai-projects/src/constants.ts create mode 100644 sdk/ai/ai-projects/src/logger.ts diff --git a/sdk/ai/ai-projects/package.json b/sdk/ai/ai-projects/package.json index dc0f50b36aa7..2ba8870b77d4 100644 --- a/sdk/ai/ai-projects/package.json +++ b/sdk/ai/ai-projects/package.json @@ -49,8 +49,12 @@ "//metadata": { "constantPaths": [ { - "path": "src/projectsClient.ts", + "path": "src/generated/src/projectsClient.ts", "prefix": "userAgentInfo" + }, + { + "path": "src/constants.ts", + "prefix": "SDK_VERSION" } ] }, diff --git a/sdk/ai/ai-projects/review/ai-projects.api.md b/sdk/ai/ai-projects/review/ai-projects.api.md index b47a35e6f0b2..8b99e39f04df 100644 --- a/sdk/ai/ai-projects/review/ai-projects.api.md +++ b/sdk/ai/ai-projects/review/ai-projects.api.md @@ -1462,6 +1462,7 @@ export interface SystemDataOutput { // @public export interface TelemetryOperations { getConnectionString(): Promise; + getSettings(): TelemetryOptions; // Warning: (ae-forgotten-export) The symbol "TelemetryOptions" needs to be exported by the entry point index.d.ts updateSettings(options: TelemetryOptions): void; } diff --git a/sdk/ai/ai-projects/src/agents/assistants.ts b/sdk/ai/ai-projects/src/agents/assistants.ts index 5ab1f4101659..a1aa8902fba2 100644 --- a/sdk/ai/ai-projects/src/agents/assistants.ts +++ b/sdk/ai/ai-projects/src/agents/assistants.ts @@ -32,7 +32,7 @@ export async function createAgent( if (!expectedStatuses.includes(result.status)) { throw createRestError(result); } - return result.body as AgentOutput; + return result.body; }, traceStartCreateAgent, traceEndCreateAgent, ); diff --git a/sdk/ai/ai-projects/src/agents/assistantsTrace.ts b/sdk/ai/ai-projects/src/agents/assistantsTrace.ts index f6fc66ee3537..e11165d38ac9 100644 --- a/sdk/ai/ai-projects/src/agents/assistantsTrace.ts +++ b/sdk/ai/ai-projects/src/agents/assistantsTrace.ts @@ -1,9 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { TracingSpan } from "@azure/core-tracing"; import { AgentOutput } from "../generated/src/outputModels.js"; -import { TracingAttributeOptions, TracingAttributes, TracingUtility, TrackingOperationName } from "../tracing.js"; +import { TracingAttributeOptions, TracingAttributes, TracingUtility, TracingOperationName, Span } from "../tracing.js"; import { CreateAgentParameters } from "../generated/src/parameters.js"; import { addInstructionsEvent, formatAgentApiResponse } from "./traceUtility.js"; @@ -12,19 +11,19 @@ import { addInstructionsEvent, formatAgentApiResponse } from "./traceUtility.js" * @param span - The span to trace. * @param options - The options for creating an agent. */ -export function traceStartCreateAgent(span: Omit, options: CreateAgentParameters): void { +export function traceStartCreateAgent(span: Span, options: CreateAgentParameters): void { const attributes: TracingAttributeOptions = { - operationName: TrackingOperationName.CREATE_AGENT, - name: options.body.name, + operationName: TracingOperationName.CREATE_AGENT, + name: options.body.name ?? undefined, model: options.body.model, - description: options.body.description, - instructions: options.body.instructions, - topP: options.body.top_p, - temperature: options.body.temperature, + description: options.body.description ?? undefined, + instructions: options.body.instructions ?? undefined, + topP: options.body.top_p ?? undefined, + temperature: options.body.temperature ?? undefined, responseFormat: formatAgentApiResponse(options.body.response_format), genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }; - TracingUtility.setSpanAttributes(span,TrackingOperationName.CREATE_AGENT, attributes) + TracingUtility.setSpanAttributes(span,TracingOperationName.CREATE_AGENT, attributes) addInstructionsEvent(span, options.body); } @@ -35,7 +34,7 @@ export function traceStartCreateAgent(span: Omit, options: C * @param _options - The options for creating an agent. * @param result - The result of creating an agent. */ -export async function traceEndCreateAgent(span: Omit, _options: CreateAgentParameters, result: Promise): Promise { +export async function traceEndCreateAgent(span: Span, _options: CreateAgentParameters, result: Promise): Promise { const resolvedResult = await result; const attributes: TracingAttributeOptions = { agentId: resolvedResult.id, diff --git a/sdk/ai/ai-projects/src/agents/messagesTrace.ts b/sdk/ai/ai-projects/src/agents/messagesTrace.ts index afee98d3c7ac..451b89727150 100644 --- a/sdk/ai/ai-projects/src/agents/messagesTrace.ts +++ b/sdk/ai/ai-projects/src/agents/messagesTrace.ts @@ -1,27 +1,26 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { TracingSpan } from "@azure/core-tracing"; import { CreateMessageParameters, ListMessagesParameters } from "../generated/src/parameters.js"; -import { TracingAttributes, TracingUtility, TrackingOperationName } from "../tracing.js"; +import { TracingAttributes, TracingUtility, TracingOperationName, Span } from "../tracing.js"; import { OpenAIPageableListOfThreadMessageOutput, ThreadMessageOutput } from "../generated/src/outputModels.js"; import { addMessageEvent } from "./traceUtility.js"; -export function traceStartCreateMessage(span: Omit, threadId: string, options: CreateMessageParameters): void { - TracingUtility.setSpanAttributes(span, TrackingOperationName.CREATE_MESSAGE, { threadId: threadId, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }); +export function traceStartCreateMessage(span: Span, threadId: string, options: CreateMessageParameters): void { + TracingUtility.setSpanAttributes(span, TracingOperationName.CREATE_MESSAGE, { threadId: threadId, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }); addMessageEvent(span, { ...options.body, thread_id: threadId }); } -export async function traceEndCreateMessage(span: Omit, _options: CreateMessageParameters, result: Promise): Promise { +export async function traceEndCreateMessage(span: Span, _options: CreateMessageParameters, result: Promise): Promise { const resolvedResult = await result; TracingUtility.updateSpanAttributes(span, { messageId: resolvedResult.id }); } -export function traceStartListMessages(span: Omit, threadId: string, _options: ListMessagesParameters) : void { - TracingUtility.setSpanAttributes(span, TrackingOperationName.LIST_MESSAGES, { threadId: threadId, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }); +export function traceStartListMessages(span: Span, threadId: string, _options: ListMessagesParameters) : void { + TracingUtility.setSpanAttributes(span, TracingOperationName.LIST_MESSAGES, { threadId: threadId, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }); } -export async function traceEndListMessages(span: Omit, _options: ListMessagesParameters, result: Promise) : Promise { +export async function traceEndListMessages(span: Span, _options: ListMessagesParameters, result: Promise) : Promise { const resolvedResult = await result; resolvedResult.data?.forEach(message => { addMessageEvent(span, message) diff --git a/sdk/ai/ai-projects/src/agents/runTrace.ts b/sdk/ai/ai-projects/src/agents/runTrace.ts index 6cdba83495c6..9a0e4ce6d000 100644 --- a/sdk/ai/ai-projects/src/agents/runTrace.ts +++ b/sdk/ai/ai-projects/src/agents/runTrace.ts @@ -1,55 +1,54 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { TracingSpan } from "@azure/core-tracing"; import { CreateRunParameters, CreateThreadAndRunParameters, SubmitToolOutputsToRunParameters } from "../generated/src/parameters.js"; import { ThreadRunOutput } from "../generated/src/outputModels.js"; -import { TracingAttributeOptions, TracingAttributes, TracingUtility, TrackingOperationName } from "../tracing.js"; +import { TracingAttributeOptions, TracingAttributes, TracingUtility, TracingOperationName, Span } from "../tracing.js"; import { addInstructionsEvent, addMessageEvent, addToolMessagesEvent, formatAgentApiResponse } from "./traceUtility.js"; -export function traceStartCreateRun(span: Omit, options: CreateRunParameters | CreateThreadAndRunParameters, threadId?: string, operationName: string = TrackingOperationName.CREATE_RUN): void { +export function traceStartCreateRun(span: Span, options: CreateRunParameters | CreateThreadAndRunParameters, threadId?: string, operationName: string = TracingOperationName.CREATE_RUN): void { const attributes: TracingAttributeOptions = { threadId: threadId, agentId: options.body.assistant_id, - model: options.body.model, - instructions: options.body.instructions, - temperature: options.body.temperature, - topP: options.body.top_p, - maxCompletionTokens: options.body.max_completion_tokens, - maxPromptTokens: options.body.max_prompt_tokens, + model: options.body.model ?? undefined, + instructions: options.body.instructions ?? undefined, + temperature: options.body.temperature ?? undefined, + topP: options.body.top_p ?? undefined, + maxCompletionTokens: options.body.max_completion_tokens ?? undefined, + maxPromptTokens: options.body.max_prompt_tokens ?? undefined, responseFormat: formatAgentApiResponse(options.body.response_format), genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM } if ((options as CreateRunParameters).body.additional_instructions) { - attributes.additional_instructions = (options as CreateRunParameters).body.additional_instructions; + attributes.additional_instructions = (options as CreateRunParameters).body.additional_instructions ?? undefined; } TracingUtility.setSpanAttributes(span, operationName, attributes); setSpanEvents(span, options); } -export function traceStartCreateThreadAndRun(span: Omit, options: CreateThreadAndRunParameters): void { - traceStartCreateRun(span, options, undefined, TrackingOperationName.CREATE_THREAD_RUN); +export function traceStartCreateThreadAndRun(span: Span, options: CreateThreadAndRunParameters): void { + traceStartCreateRun(span, options, undefined, TracingOperationName.CREATE_THREAD_RUN); } -export async function traceEndCreateRun(span: Omit, _options: CreateRunParameters, result: Promise): Promise { +export async function traceEndCreateRun(span: Span, _options: CreateRunParameters, result: Promise): Promise { const resolvedResult = await result; TracingUtility.updateSpanAttributes(span, { runId: resolvedResult.id, runStatus: resolvedResult.status, responseModel: resolvedResult.model, usageCompletionTokens: resolvedResult.usage?.completion_tokens, usagePromptTokens: resolvedResult.usage?.prompt_tokens }); } -export function traceStartSubmitToolOutputsToRun(span: Omit, options: SubmitToolOutputsToRunParameters, threadId: string, +export function traceStartSubmitToolOutputsToRun(span: Span, options: SubmitToolOutputsToRunParameters, threadId: string, runId: string,): void { const attributes: TracingAttributeOptions = { threadId: threadId, runId: runId } - TracingUtility.setSpanAttributes(span, TrackingOperationName.SUBMIT_TOOL_OUTPUTS, attributes); + TracingUtility.setSpanAttributes(span, TracingOperationName.SUBMIT_TOOL_OUTPUTS, attributes); addToolMessagesEvent(span, options.body.tool_outputs); } -export async function traceEndSubmitToolOutputsToRun(span: Omit, _options: SubmitToolOutputsToRunParameters, result: Promise): Promise { +export async function traceEndSubmitToolOutputsToRun(span: Span, _options: SubmitToolOutputsToRunParameters, result: Promise): Promise { const resolvedResult = await result; TracingUtility.updateSpanAttributes(span, { runId: resolvedResult.id, runStatus: resolvedResult.status, responseModel: resolvedResult.model, usageCompletionTokens: resolvedResult.usage?.completion_tokens, usagePromptTokens: resolvedResult.usage?.prompt_tokens }); } -function setSpanEvents(span: Omit, options: CreateRunParameters): void { +function setSpanEvents(span: Span, options: CreateRunParameters): void { addInstructionsEvent(span, { ...options.body, agentId: options.body.assistant_id }); options.body.additional_messages?.forEach((message) => { addMessageEvent(span, message); diff --git a/sdk/ai/ai-projects/src/agents/threadsTrace.ts b/sdk/ai/ai-projects/src/agents/threadsTrace.ts index 845fd254e2f5..5c9df8b4d142 100644 --- a/sdk/ai/ai-projects/src/agents/threadsTrace.ts +++ b/sdk/ai/ai-projects/src/agents/threadsTrace.ts @@ -1,23 +1,22 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { TracingSpan } from "@azure/core-tracing"; import { AgentThreadOutput } from "../generated/src/outputModels.js"; -import { TracingAttributes, TracingUtility, TrackingOperationName } from "../tracing.js"; +import { TracingAttributes, TracingUtility, TracingOperationName, Span } from "../tracing.js"; import { CreateThreadParameters } from "../generated/src/parameters.js"; import { addMessageEvent } from "./traceUtility.js"; -export function traceStartCreateThread(span: Omit, options: CreateThreadParameters): void { - TracingUtility.setSpanAttributes(span, TrackingOperationName.CREATE_THREAD, { genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }); +export function traceStartCreateThread(span: Span, options: CreateThreadParameters): void { + TracingUtility.setSpanAttributes(span, TracingOperationName.CREATE_THREAD, { genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }); setSpanEvents(span, options); } -export async function traceEndCreateThread(span: Omit, _options: CreateThreadParameters, result: Promise): Promise { +export async function traceEndCreateThread(span: Span, _options: CreateThreadParameters, result: Promise): Promise { const resolvedResult = await result; TracingUtility.updateSpanAttributes(span, { threadId: resolvedResult.id }); } -function setSpanEvents(span: Omit, options: CreateThreadParameters): void { +function setSpanEvents(span: Span, options: CreateThreadParameters): void { options.body.messages?.forEach((message) => { addMessageEvent(span, message); }); diff --git a/sdk/ai/ai-projects/src/agents/traceUtility.ts b/sdk/ai/ai-projects/src/agents/traceUtility.ts index 6b5c723e27f5..6590da35e1d3 100644 --- a/sdk/ai/ai-projects/src/agents/traceUtility.ts +++ b/sdk/ai/ai-projects/src/agents/traceUtility.ts @@ -1,9 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { TracingSpan } from "@azure/core-tracing"; import { AgentsApiResponseFormat, AgentsApiResponseFormatOption, MessageContent, RunStepCompletionUsageOutput, ThreadMessage, ThreadMessageOptions, ToolOutput } from "./inputOutputs.js"; -import { TracingAttributes, TracingUtility } from "../tracing.js"; +import { Span, TracingAttributes, TracingUtility } from "../tracing.js"; import { getTelemetryOptions } from "../telemetry/telemetry.js"; /** @@ -11,7 +10,7 @@ import { getTelemetryOptions } from "../telemetry/telemetry.js"; * @param span - The span to add the event to. * @param messageAttributes - The attributes of the message event. */ -export function addMessageEvent(span: Omit, messageAttributes: ThreadMessageOptions | ThreadMessage, usage?: RunStepCompletionUsageOutput): void { +export function addMessageEvent(span: Span, messageAttributes: ThreadMessageOptions | ThreadMessage, usage?: RunStepCompletionUsageOutput): void { const eventBody: Record = {}; const telemetryOptions = getTelemetryOptions() @@ -28,7 +27,7 @@ export function addMessageEvent(span: Omit, messageAttribute }); } const threadId = (messageAttributes as ThreadMessage).thread_id; - const agentId = (messageAttributes as ThreadMessage).assistant_id; + const agentId = (messageAttributes as ThreadMessage).assistant_id ?? undefined; const threadRunId = (messageAttributes as ThreadMessage).run_id; const messageStatus = (messageAttributes as ThreadMessage).status; const messageId = (messageAttributes as ThreadMessage).id; @@ -47,7 +46,7 @@ export function addMessageEvent(span: Omit, messageAttribute * @param span - The span to add the event to. * @param instructionAttributes - The attributes of the instruction event. */ -export function addInstructionsEvent(span: Omit, instructionAttributes: { instructions?: string | null, additional_instructions?: string | null, threadId?: string, agentId?: string }): void { +export function addInstructionsEvent(span: Span, instructionAttributes: { instructions?: string | null, additional_instructions?: string | null, threadId?: string, agentId?: string }): void { const eventBody: Record = {}; eventBody.content = instructionAttributes.instructions && instructionAttributes.additional_instructions ? `${instructionAttributes.instructions} ${instructionAttributes.additional_instructions}` : instructionAttributes.instructions || instructionAttributes.additional_instructions; const attributes = { eventContent: JSON.stringify(eventBody), threadId: instructionAttributes.threadId, agentId: instructionAttributes.agentId, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }; @@ -59,12 +58,12 @@ export function addInstructionsEvent(span: Omit, instruction * @param responseFormat - The response format option. * @returns The formatted response as a string, or null/undefined. */ -export function formatAgentApiResponse(responseFormat: AgentsApiResponseFormatOption | null | undefined): string | null | undefined { +export function formatAgentApiResponse(responseFormat: AgentsApiResponseFormatOption | null | undefined): string | undefined { if (typeof responseFormat === "string" || responseFormat === undefined || responseFormat === null) { - return responseFormat; + return responseFormat ?? undefined; } if ((responseFormat as AgentsApiResponseFormat).type) { - return (responseFormat as AgentsApiResponseFormat).type; + return (responseFormat as AgentsApiResponseFormat).type ?? undefined; } return undefined; } @@ -74,7 +73,7 @@ export function formatAgentApiResponse(responseFormat: AgentsApiResponseFormatOp * @param span - The span to add the event to. * @param tool_outputs - List of tool oupts */ -export function addToolMessagesEvent(span: Omit, tool_outputs: Array): void { +export function addToolMessagesEvent(span: Span, tool_outputs: Array): void { tool_outputs.forEach(tool_output => { const eventBody = {"content": tool_output.output, "id": tool_output.tool_call_id} TracingUtility.addSpanEvent(span, "gen_ai.tool.message", {eventContent: JSON.stringify(eventBody), genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM}); diff --git a/sdk/ai/ai-projects/src/constants.ts b/sdk/ai/ai-projects/src/constants.ts new file mode 100644 index 000000000000..fe3152c1805e --- /dev/null +++ b/sdk/ai/ai-projects/src/constants.ts @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Current version of the `@azure/ai-projects` package. + */ +export const SDK_VERSION = `1.0.0-beta.1`; + +/** + * The package name of the `@azure/ai-projects` package. + */ +export const PACKAGE_NAME = "@azure/ai-projects"; diff --git a/sdk/ai/ai-projects/src/logger.ts b/sdk/ai/ai-projects/src/logger.ts new file mode 100644 index 000000000000..9f092c1689f3 --- /dev/null +++ b/sdk/ai/ai-projects/src/logger.ts @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createClientLogger } from "@azure/logger"; +import { PACKAGE_NAME } from "./constants.js"; +export const logger = createClientLogger(PACKAGE_NAME); diff --git a/sdk/ai/ai-projects/src/telemetry/telemetry.ts b/sdk/ai/ai-projects/src/telemetry/telemetry.ts index b540bce8007a..d0acb961156f 100644 --- a/sdk/ai/ai-projects/src/telemetry/telemetry.ts +++ b/sdk/ai/ai-projects/src/telemetry/telemetry.ts @@ -11,9 +11,9 @@ export interface TelemetryOptions { enableContentRecording: boolean; } -const telemetryOptions: TelemetryOptions & { connectionString: string | unknown } = { +const telemetryOptions: TelemetryOptions & { connectionString: string | undefined } = { enableContentRecording: false, - connectionString: null + connectionString: undefined } /** @@ -36,7 +36,7 @@ export function getTelemetryOptions(): TelemetryOptions { * Reset the telemetry options */ export function resetTelemetryOptions(): void { - telemetryOptions.connectionString = null; + telemetryOptions.connectionString = undefined; telemetryOptions.enableContentRecording = false; } diff --git a/sdk/ai/ai-projects/src/tracing.ts b/sdk/ai/ai-projects/src/tracing.ts index ade40ec34b2e..4ac15c66c1b7 100644 --- a/sdk/ai/ai-projects/src/tracing.ts +++ b/sdk/ai/ai-projects/src/tracing.ts @@ -2,6 +2,9 @@ // Licensed under the MIT License. import { createTracingClient, OperationTracingOptions, Resolved, SpanStatusError, TracingSpan } from "@azure/core-tracing"; +import { PACKAGE_NAME, SDK_VERSION } from "./constants.js"; +import { getErrorMessage } from "@azure/core-util"; +import { logger } from "./logger.js"; export enum TracingAttributes { GEN_AI_MESSAGE_ID = "gen_ai.message.id", @@ -31,7 +34,7 @@ export enum TracingAttributes { GEN_AI_EVENT_CONTENT = "gen_ai.event.content", ERROR_TYPE = "error.type" } -export enum TrackingOperationName { +export enum TracingOperationName { CREATE_AGENT = "create_agent", CREATE_THREAD = "create_thread", CREATE_MESSAGE = "create_message", @@ -45,49 +48,52 @@ export enum TrackingOperationName { export interface TracingAttributeOptions { operationName?: string; - name?: string | null; - description?: string | null; - serverAddress?: string | null; - threadId?: string | null; - agentId?: string | null; - instructions?: string | null; - additional_instructions?: string | null; - runId?: string | null; - runStatus?: string | null; - responseModel?: string | null; - model?: string | null; - temperature?: number | null; - topP?: number | null; - maxPromptTokens?: number | null; - maxCompletionTokens?: number | null; - responseFormat?: string | null; - genAiSystem?: string | null; - messageId?: string | null; - messageStatus?: string | null; - eventContent?: string | null; - usagePromptTokens?: number | null; - usageCompletionTokens?: number | null; + name?: string; + description?: string; + serverAddress?: string; + threadId?: string; + agentId?: string; + instructions?: string; + additional_instructions?: string; + runId?: string; + runStatus?: string; + responseModel?: string; + model?: string; + temperature?: number; + topP?: number; + maxPromptTokens?: number; + maxCompletionTokens?: number; + responseFormat?: string; + genAiSystem?: string; + messageId?: string; + messageStatus?: string; + eventContent?: string; + usagePromptTokens?: number; + usageCompletionTokens?: number; } +export type Span = Omit; export class TracingUtility { private static tracingClient = createTracingClient({ namespace: "Microsoft.CognitiveServices", - packageName: "@azure/ai-projects", - packageVersion: "1.0.0-beta.1", + packageName: PACKAGE_NAME, + packageVersion: SDK_VERSION, }); static async withSpan ReturnType>(name: string, options: Options, request: Request, - startTrace?: (span: Omit, updatedOptions: Options,) => void, - endTrace?: (span: Omit, updatedOptions: Options, result: ReturnType) => void, + startTrace?: (span: Span, updatedOptions: Options,) => void, + endTrace?: (span: Span, updatedOptions: Options, result: ReturnType) => void, ): Promise>> { - return TracingUtility.tracingClient.withSpan(name, options, async (updatedOptions: Options, span: Omit) => { + return TracingUtility.tracingClient.withSpan(name, options, async (updatedOptions: Options, span: Span) => { if (startTrace) { try { startTrace(span, updatedOptions); } - catch { /* empty */ } + catch (e) { + logger.warning(`Skipping updating span before request execution due to an error: ${getErrorMessage(e)}`); + } } let result: ReturnType | undefined; @@ -104,18 +110,21 @@ export class TracingUtility { if (endTrace && result !== undefined) { try { endTrace(span, updatedOptions, result); - } catch { /* empty */ } + } + catch (e) { + logger.warning(`Skipping updating span after request execution due to an error: ${getErrorMessage(e)}`); + } } return result; }, { spanKind: "client" }); } - static updateSpanAttributes(span: Omit, attributeOptions: Omit): void { + static updateSpanAttributes(span: Span, attributeOptions: Omit): void { TracingUtility.setAttributes(span, attributeOptions); } static setSpanAttributes( - span: Omit, + span: Span, operationName: string, attributeOptions: TracingAttributeOptions ): void { @@ -124,7 +133,7 @@ export class TracingUtility { } static setAttributes( - span: Omit, + span: Span, attributeOptions: TracingAttributeOptions ): void { if (span.isRecording()) { @@ -226,7 +235,7 @@ export class TracingUtility { } static addSpanEvent( - span: Omit, + span: Span, eventName: string, attributeOptions: Omit ): void { From 5e3ded25fc85f2cc0f8fb5f046321238c6cc1617 Mon Sep 17 00:00:00 2001 From: Ganesh Bheemarasetty Date: Sun, 8 Dec 2024 16:00:26 -0800 Subject: [PATCH 8/9] Add tracing for more functions --- sdk/ai/ai-projects/package.json | 7 ++ .../agents/agentCreateWithTracingCconsole.ts | 53 ----------- .../agents/agentCreateWithTracingConsole.ts | 89 +++++++++++++++++++ sdk/ai/ai-projects/src/agents/assistants.ts | 85 ++++++++++-------- .../ai-projects/src/agents/assistantsTrace.ts | 19 ++-- sdk/ai/ai-projects/src/agents/messages.ts | 23 ++--- sdk/ai/ai-projects/src/agents/runTrace.ts | 13 ++- sdk/ai/ai-projects/src/agents/runs.ts | 71 ++++++++------- sdk/ai/ai-projects/src/agents/threads.ts | 59 ++++++------ sdk/ai/ai-projects/src/agents/threadsTrace.ts | 6 +- sdk/ai/ai-projects/src/agents/traceUtility.ts | 29 ++++-- sdk/ai/ai-projects/src/aiProjectsClient.ts | 18 +++- sdk/ai/ai-projects/src/telemetry/index.ts | 17 ++-- sdk/ai/ai-projects/src/telemetry/telemetry.ts | 12 ++- sdk/ai/ai-projects/src/tracing.ts | 6 +- .../test/public/agents/tracing.spec.ts | 10 +-- 16 files changed, 315 insertions(+), 202 deletions(-) delete mode 100644 sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingCconsole.ts create mode 100644 sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingConsole.ts diff --git a/sdk/ai/ai-projects/package.json b/sdk/ai/ai-projects/package.json index 2ba8870b77d4..9119393164bf 100644 --- a/sdk/ai/ai-projects/package.json +++ b/sdk/ai/ai-projects/package.json @@ -146,6 +146,13 @@ } } }, + "//sampleConfiguration": { + "productName": "Azure AI Projects", + "productSlugs": [ + "azure" + ], + "apiRefLink": "https://learn.microsoft.com/javascript/api/@azure/ai-projects" + }, "main": "./dist/commonjs/index.js", "types": "./dist/commonjs/index.d.ts", "module": "./dist/esm/index.js" diff --git a/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingCconsole.ts b/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingCconsole.ts deleted file mode 100644 index 8032eb3f950b..000000000000 --- a/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingCconsole.ts +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - - -import { context } from "@opentelemetry/api"; -import { registerInstrumentations } from "@opentelemetry/instrumentation"; -import { createAzureSdkInstrumentation } from "@azure/opentelemetry-instrumentation-azure-sdk"; -import { AzureMonitorTraceExporter } from "@azure/monitor-opentelemetry-exporter" -import { - ConsoleSpanExporter, - NodeTracerProvider, - SimpleSpanProcessor, -} from "@opentelemetry/sdk-trace-node"; - -import * as dotenv from "dotenv"; -dotenv.config(); - -const provider = new NodeTracerProvider(); -provider.addSpanProcessor(new SimpleSpanProcessor(new ConsoleSpanExporter())); -provider.register(); - -registerInstrumentations({ - instrumentations: [createAzureSdkInstrumentation()], -}); - -import { AIProjectsClient } from "@azure/ai-projects" -import { DefaultAzureCredential } from "@azure/identity"; - -const connectionString = process.env["AZURE_AI_PROJECTS_CONNECTION_STRING"] || ">;;;"; - -export async function main(): Promise { - - const client = AIProjectsClient.fromConnectionString(connectionString || "", new DefaultAzureCredential()); - - const appInsightsConnectionString = await client.telemetry.getConnectionString(); - if (appInsightsConnectionString) { - const exporter = new AzureMonitorTraceExporter({ connectionString: appInsightsConnectionString }); - provider.addSpanProcessor(new SimpleSpanProcessor(exporter)); - } - - const agent = await client.agents.createAgent("gpt-4o", { name: "my-agent", instructions: "You are helpful agent" }, { tracingOptions: { tracingContext: context.active() } }); - - - console.log(`Created agent, agent ID : ${agent.id}`); - - await client.agents.deleteAgent(agent.id); - - console.log(`Deleted agent`); -} - -main().catch((err) => { - console.error("The sample encountered an error:", err); -}); diff --git a/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingConsole.ts b/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingConsole.ts new file mode 100644 index 000000000000..43dac37bcfe6 --- /dev/null +++ b/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingConsole.ts @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Demonstrates How to instrument and get tracing using open telemetry. + * + * @summary Create Agent and instrument using open telemetry. + */ + +import { trace, context } from "@opentelemetry/api"; +import { registerInstrumentations } from "@opentelemetry/instrumentation"; +import { createAzureSdkInstrumentation } from "@azure/opentelemetry-instrumentation-azure-sdk"; +import { AzureMonitorTraceExporter } from "@azure/monitor-opentelemetry-exporter" +import { + ConsoleSpanExporter, + NodeTracerProvider, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-node"; + +import * as dotenv from "dotenv"; +dotenv.config(); + +const provider = new NodeTracerProvider(); +provider.addSpanProcessor(new SimpleSpanProcessor(new ConsoleSpanExporter())); +provider.register(); + +registerInstrumentations({ + instrumentations: [createAzureSdkInstrumentation()], +}); + +import { AIProjectsClient } from "@azure/ai-projects" +import { delay } from "@azure/core-util"; +import { DefaultAzureCredential } from "@azure/identity"; + +const connectionString = process.env["AZURE_AI_PROJECTS_CONNECTION_STRING"] || ">;;;"; +let appInsightsConnectionString = process.env["APPLICATIONINSIGHTS_CONNECTION_STRING"] + +export async function main(): Promise { + + const tracer = trace.getTracer("Agents Sample", "1.0.0"); + + const client = AIProjectsClient.fromConnectionString(connectionString || "", new DefaultAzureCredential()); + + if (!appInsightsConnectionString) { + appInsightsConnectionString = await client.telemetry.getConnectionString(); + } + + if (appInsightsConnectionString) { + const exporter = new AzureMonitorTraceExporter({ connectionString: appInsightsConnectionString }); + provider.addSpanProcessor(new SimpleSpanProcessor(exporter)); + } + + await tracer.startActiveSpan("main", async (span) => { + + client.telemetry.updateSettings({enableContentRecording: true}) + + const agent = await client.agents.createAgent("gpt-4o", { name: "my-agent", instructions: "You are helpful agent" }, { tracingOptions: { tracingContext: context.active() } }); + + + console.log(`Created agent, agent ID : ${agent.id}`); + + const thread = await client.agents.createThread(); + console.log(`Created Thread, thread ID: ${thread.id}`); + + // Create message + const message = await client.agents.createMessage(thread.id, { role: "user", content: "What's the weather like in my favorite city?" }) + console.log(`Created message, message ID ${message.id}`); + + // Create run + let run = await client.agents.createRun(thread.id, agent.id); + console.log(`Created Run, Run ID: ${run.id}`); + + while (["queued", "in_progress", "requires_action"].includes(run.status)) { + await delay(1000); + run = await client.agents.getRun(thread.id, run.id); + console.log(`Current Run status - ${run.status}, run ID: ${run.id}`); + } + + await client.agents.deleteAgent(agent.id); + + console.log(`Deleted agent`); + + span.end(); + }); +} + +main().catch((err) => { + console.error("The sample encountered an error:", err); +}); diff --git a/sdk/ai/ai-projects/src/agents/assistants.ts b/sdk/ai/ai-projects/src/agents/assistants.ts index a1aa8902fba2..3c833e4ddda5 100644 --- a/sdk/ai/ai-projects/src/agents/assistants.ts +++ b/sdk/ai/ai-projects/src/agents/assistants.ts @@ -6,7 +6,8 @@ import { AgentDeletionStatusOutput, AgentOutput, OpenAIPageableListOfAgentOutput import { CreateAgentParameters, DeleteAgentParameters, GetAgentParameters, ListAgentsParameters, UpdateAgentParameters } from "../generated/src/parameters.js"; import { validateLimit, validateMetadata, validateOrder, validateVectorStoreDataType } from "./inputValidations.js"; import { TracingUtility } from "../tracing.js"; -import { traceEndCreateAgent, traceStartCreateAgent } from "./assistantsTrace.js"; +import { traceEndCreateOrUpdateAgent, traceStartCreateOrUpdateAgent } from "./assistantsTrace.js"; +import { traceEndAgentGeneric, traceStartAgentGeneric } from "./traceUtility.js"; const expectedStatuses = ["200"]; @@ -34,55 +35,61 @@ export async function createAgent( } return result.body; }, - traceStartCreateAgent, traceEndCreateAgent, + traceStartCreateOrUpdateAgent, traceEndCreateOrUpdateAgent, ); } /** Gets a list of agents that were previously created. */ export async function listAgents( context: Client, - options?: ListAgentsParameters, + options: ListAgentsParameters = {}, ): Promise { validateListAgentsParameters(options); - const result = await context - .path("/assistants") - .get(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("ListAgents", options || {}, async (updateOptions) => { + const result = await context + .path("/assistants") + .get(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }); } /** Retrieves an existing agent. */ export async function getAgent( context: Client, assistantId: string, - options?: GetAgentParameters, + options: GetAgentParameters = {}, ): Promise { validateAssistantId(assistantId); - const result = await context - .path("/assistants/{assistantId}", assistantId) - .get(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("GetAgent", options || {}, async (updateOptions) => { + const result = await context + .path("/assistants/{assistantId}", assistantId) + .get(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { agentId: assistantId } })); } /** Modifies an existing agent. */ export async function updateAgent( context: Client, assistantId: string, - options?: UpdateAgentParameters, + options: UpdateAgentParameters = { body: {} }, ): Promise { validateUpdateAgentParameters(assistantId, options); - const result = await context - .path("/assistants/{assistantId}", assistantId) - .post(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("UpdateAgent", options, async (updateOptions) => { + const result = await context + .path("/assistants/{assistantId}", assistantId) + .post(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartCreateOrUpdateAgent(span, updatedOptions, assistantId), traceEndCreateOrUpdateAgent,); } @@ -90,16 +97,18 @@ export async function updateAgent( export async function deleteAgent( context: Client, assistantId: string, - options?: DeleteAgentParameters, + options: DeleteAgentParameters = {}, ): Promise { validateAssistantId(assistantId); - const result = await context - .path("/assistants/{assistantId}", assistantId) - .delete(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("DeleteAgent", options, async (updateOptions) => { + const result = await context + .path("/assistants/{assistantId}", assistantId) + .delete(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, traceStartAgentGeneric, traceEndAgentGeneric); } function validateCreateAgentParameters(options: CreateAgentParameters | UpdateAgentParameters): void { @@ -125,10 +134,10 @@ function validateCreateAgentParameters(options: CreateAgentParameters | UpdateAg throw new Error("Only one vector store ID is allowed"); } if (options.body.tool_resources.file_search.vector_stores) { - if (options.body.tool_resources.file_search.vector_stores.length > 1) { - throw new Error("Only one vector store is allowed"); - } - validateVectorStoreDataType(options.body.tool_resources.file_search.vector_stores[0]?.configuration.data_sources); + if (options.body.tool_resources.file_search.vector_stores.length > 1) { + throw new Error("Only one vector store is allowed"); + } + validateVectorStoreDataType(options.body.tool_resources.file_search.vector_stores[0]?.configuration.data_sources); } } if (options.body.tool_resources.azure_ai_search) { diff --git a/sdk/ai/ai-projects/src/agents/assistantsTrace.ts b/sdk/ai/ai-projects/src/agents/assistantsTrace.ts index e11165d38ac9..111e03ac5655 100644 --- a/sdk/ai/ai-projects/src/agents/assistantsTrace.ts +++ b/sdk/ai/ai-projects/src/agents/assistantsTrace.ts @@ -2,16 +2,16 @@ // Licensed under the MIT License. import { AgentOutput } from "../generated/src/outputModels.js"; -import { TracingAttributeOptions, TracingAttributes, TracingUtility, TracingOperationName, Span } from "../tracing.js"; -import { CreateAgentParameters } from "../generated/src/parameters.js"; -import { addInstructionsEvent, formatAgentApiResponse } from "./traceUtility.js"; +import { TracingAttributeOptions, TracingUtility, TracingOperationName, Span } from "../tracing.js"; +import { CreateAgentParameters, UpdateAgentParameters } from "../generated/src/parameters.js"; +import { addInstructionsEvent, formatAgentApiResponse, UpdateWithAgentAttributes } from "./traceUtility.js"; /** - * Traces the start of creating an agent. + * Traces the start of creating or updating an agent. * @param span - The span to trace. * @param options - The options for creating an agent. */ -export function traceStartCreateAgent(span: Span, options: CreateAgentParameters): void { +export function traceStartCreateOrUpdateAgent(span: Span, options: CreateAgentParameters | UpdateAgentParameters, agentId?:string): void { const attributes: TracingAttributeOptions = { operationName: TracingOperationName.CREATE_AGENT, name: options.body.name ?? undefined, @@ -21,9 +21,12 @@ export function traceStartCreateAgent(span: Span, options: CreateAgentParameters topP: options.body.top_p ?? undefined, temperature: options.body.temperature ?? undefined, responseFormat: formatAgentApiResponse(options.body.response_format), - genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }; - TracingUtility.setSpanAttributes(span,TracingOperationName.CREATE_AGENT, attributes) + if(agentId){ + attributes.operationName = TracingOperationName.CREATE_UPDATE_AGENT + attributes.agentId = agentId + } + TracingUtility.setSpanAttributes(span, TracingOperationName.CREATE_AGENT, UpdateWithAgentAttributes(attributes)) addInstructionsEvent(span, options.body); } @@ -34,7 +37,7 @@ export function traceStartCreateAgent(span: Span, options: CreateAgentParameters * @param _options - The options for creating an agent. * @param result - The result of creating an agent. */ -export async function traceEndCreateAgent(span: Span, _options: CreateAgentParameters, result: Promise): Promise { +export async function traceEndCreateOrUpdateAgent(span: Span, _options: CreateAgentParameters | UpdateAgentParameters, result: Promise): Promise { const resolvedResult = await result; const attributes: TracingAttributeOptions = { agentId: resolvedResult.id, diff --git a/sdk/ai/ai-projects/src/agents/messages.ts b/sdk/ai/ai-projects/src/agents/messages.ts index 0746352f829b..e32e1d0b82c8 100644 --- a/sdk/ai/ai-projects/src/agents/messages.ts +++ b/sdk/ai/ai-projects/src/agents/messages.ts @@ -7,6 +7,7 @@ import { CreateMessageParameters, ListMessagesParameters, UpdateMessageParameter import { validateMetadata, validateVectorStoreDataType } from "./inputValidations.js"; import { TracingUtility } from "../tracing.js"; import { traceEndCreateMessage, traceEndListMessages, traceStartCreateMessage, traceStartListMessages } from "./messagesTrace.js"; +import { traceStartAgentGeneric } from "./traceUtility.js"; const expectedStatuses = ["200"]; @@ -33,11 +34,11 @@ export async function createMessage( export async function listMessages( context: Client, threadId: string, - options?: ListMessagesParameters, + options: ListMessagesParameters = {}, ): Promise { validateThreadId(threadId); validateListMessagesParameters(options); - return TracingUtility.withSpan("ListMessages", options || {}, async (updateOptions) => { + return TracingUtility.withSpan("ListMessages", options, async (updateOptions) => { const result = await context .path("/threads/{threadId}/messages", threadId) .get(updateOptions); @@ -53,17 +54,19 @@ export async function updateMessage( context: Client, threadId: string, messageId: string, - options?: UpdateMessageParameters, + options: UpdateMessageParameters = { body: {} }, ): Promise { validateThreadId(threadId); validateMessageId(messageId); - const result = await context - .path("/threads/{threadId}/messages/{messageId}", threadId, messageId) - .post(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("UpdateMessage", options, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/messages/{messageId}", threadId, messageId) + .post(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { threadId: threadId, messageId: messageId } })); } function validateThreadId(threadId: string): void { diff --git a/sdk/ai/ai-projects/src/agents/runTrace.ts b/sdk/ai/ai-projects/src/agents/runTrace.ts index 9a0e4ce6d000..953257c108d9 100644 --- a/sdk/ai/ai-projects/src/agents/runTrace.ts +++ b/sdk/ai/ai-projects/src/agents/runTrace.ts @@ -1,10 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { CreateRunParameters, CreateThreadAndRunParameters, SubmitToolOutputsToRunParameters } from "../generated/src/parameters.js"; +import { CreateRunParameters, CreateThreadAndRunParameters, SubmitToolOutputsToRunParameters, UpdateRunParameters } from "../generated/src/parameters.js"; import { ThreadRunOutput } from "../generated/src/outputModels.js"; -import { TracingAttributeOptions, TracingAttributes, TracingUtility, TracingOperationName, Span } from "../tracing.js"; -import { addInstructionsEvent, addMessageEvent, addToolMessagesEvent, formatAgentApiResponse } from "./traceUtility.js"; +import { TracingAttributeOptions, TracingUtility, TracingOperationName, Span } from "../tracing.js"; +import { addInstructionsEvent, addMessageEvent, addToolMessagesEvent, formatAgentApiResponse, UpdateWithAgentAttributes } from "./traceUtility.js"; export function traceStartCreateRun(span: Span, options: CreateRunParameters | CreateThreadAndRunParameters, threadId?: string, operationName: string = TracingOperationName.CREATE_RUN): void { @@ -18,12 +18,11 @@ export function traceStartCreateRun(span: Span, options: CreateRunParameters | C maxCompletionTokens: options.body.max_completion_tokens ?? undefined, maxPromptTokens: options.body.max_prompt_tokens ?? undefined, responseFormat: formatAgentApiResponse(options.body.response_format), - genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM } if ((options as CreateRunParameters).body.additional_instructions) { attributes.additional_instructions = (options as CreateRunParameters).body.additional_instructions ?? undefined; } - TracingUtility.setSpanAttributes(span, operationName, attributes); + TracingUtility.setSpanAttributes(span, operationName, UpdateWithAgentAttributes(attributes)); setSpanEvents(span, options); } @@ -31,7 +30,7 @@ export function traceStartCreateThreadAndRun(span: Span, options: CreateThreadAn traceStartCreateRun(span, options, undefined, TracingOperationName.CREATE_THREAD_RUN); } -export async function traceEndCreateRun(span: Span, _options: CreateRunParameters, result: Promise): Promise { +export async function traceEndCreateOrUpdateRun(span: Span, _options: CreateRunParameters | UpdateRunParameters, result: Promise): Promise { const resolvedResult = await result; TracingUtility.updateSpanAttributes(span, { runId: resolvedResult.id, runStatus: resolvedResult.status, responseModel: resolvedResult.model, usageCompletionTokens: resolvedResult.usage?.completion_tokens, usagePromptTokens: resolvedResult.usage?.prompt_tokens }); } @@ -39,7 +38,7 @@ export async function traceEndCreateRun(span: Span, _options: CreateRunParameter export function traceStartSubmitToolOutputsToRun(span: Span, options: SubmitToolOutputsToRunParameters, threadId: string, runId: string,): void { const attributes: TracingAttributeOptions = { threadId: threadId, runId: runId } - TracingUtility.setSpanAttributes(span, TracingOperationName.SUBMIT_TOOL_OUTPUTS, attributes); + TracingUtility.setSpanAttributes(span, TracingOperationName.SUBMIT_TOOL_OUTPUTS, UpdateWithAgentAttributes(attributes)); addToolMessagesEvent(span, options.body.tool_outputs); } diff --git a/sdk/ai/ai-projects/src/agents/runs.ts b/sdk/ai/ai-projects/src/agents/runs.ts index b5da4c4676c0..45997a69f411 100644 --- a/sdk/ai/ai-projects/src/agents/runs.ts +++ b/sdk/ai/ai-projects/src/agents/runs.ts @@ -6,7 +6,8 @@ import { CancelRunParameters, CreateRunParameters, CreateThreadAndRunParameters, import { OpenAIPageableListOfThreadRunOutput, ThreadRunOutput } from "../generated/src/outputModels.js"; import { validateLimit, validateMessages, validateMetadata, validateOrder, validateRunId, validateThreadId, validateTools, validateTruncationStrategy } from "./inputValidations.js"; import { TracingUtility } from "../tracing.js"; -import { traceEndCreateRun, traceEndSubmitToolOutputsToRun, traceStartCreateRun, traceStartCreateThreadAndRun, traceStartSubmitToolOutputsToRun } from "./runTrace.js"; +import { traceEndCreateOrUpdateRun, traceEndSubmitToolOutputsToRun, traceStartCreateRun, traceStartCreateThreadAndRun, traceStartSubmitToolOutputsToRun } from "./runTrace.js"; +import { traceStartAgentGeneric } from "./traceUtility.js"; const expectedStatuses = ["200"]; @@ -27,7 +28,7 @@ export async function createRun( throw createRestError(result); } return result.body; - }, (span, updatedOptions) => traceStartCreateRun(span, updatedOptions, threadId), traceEndCreateRun); + }, (span, updatedOptions) => traceStartCreateRun(span, updatedOptions, threadId), traceEndCreateOrUpdateRun); } /** Gets a list of runs for a specified thread. */ @@ -37,13 +38,15 @@ export async function listRuns( options?: ListRunsParameters, ): Promise { validateListRunsParameters(threadId, options); - const result = await context - .path("/threads/{threadId}/runs", threadId) - .get(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("ListRuns", options || {}, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/runs", threadId) + .get(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { threadId: threadId } })); } /** Gets an existing run from an existing thread. */ @@ -55,13 +58,15 @@ export async function getRun( ): Promise { validateThreadId(threadId); validateRunId(runId); - const result = await context - .path("/threads/{threadId}/runs/{runId}", threadId, runId) - .get(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("GetRun", options || {}, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/runs/{runId}", threadId, runId) + .get(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { threadId: threadId, runId: runId } })); } /** Modifies an existing thread run. */ @@ -72,13 +77,15 @@ export async function updateRun( options?: UpdateRunParameters, ): Promise { validateUpdateRunParameters(threadId, runId, options); - const result = await context - .path("/threads/{threadId}/runs/{runId}", threadId, runId) - .post(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("UpdateRun", options || { body: {} }, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/runs/{runId}", threadId, runId) + .post(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { threadId: threadId, runId: runId } }), traceEndCreateOrUpdateRun); } /** Submits outputs from tools as requested by tool calls in a run. Runs that need submitted tool outputs will have a status of 'requires_action' with a required_action.type of 'submit_tool_outputs'. */ @@ -111,13 +118,15 @@ export async function cancelRun( ): Promise { validateThreadId(threadId); validateRunId(runId); - const result = await context - .path("/threads/{threadId}/runs/{runId}/cancel", threadId, runId) - .post(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("CancelRun", options || {}, async (updateOptions) => { + const result = await context + .path("/threads/{threadId}/runs/{runId}/cancel", threadId, runId) + .post(updateOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }); } /** Creates a new thread and immediately starts a run of that thread. */ @@ -133,7 +142,7 @@ export async function createThreadAndRun( throw createRestError(result); } return result.body; - }, traceStartCreateThreadAndRun, traceEndCreateRun); + }, traceStartCreateThreadAndRun, traceEndCreateOrUpdateRun); } function validateListRunsParameters(thread_id: string, options?: ListRunsParameters): void { diff --git a/sdk/ai/ai-projects/src/agents/threads.ts b/sdk/ai/ai-projects/src/agents/threads.ts index 7caa13939514..de9a8edef529 100644 --- a/sdk/ai/ai-projects/src/agents/threads.ts +++ b/sdk/ai/ai-projects/src/agents/threads.ts @@ -7,16 +7,17 @@ import { AgentThreadOutput, ThreadDeletionStatusOutput } from "../generated/src/ import { TracingUtility } from "../tracing.js"; import { traceEndCreateThread, traceStartCreateThread } from "./threadsTrace.js"; import { validateMessages, validateMetadata, validateThreadId, validateToolResources } from "./inputValidations.js"; +import { traceStartAgentGeneric } from "./traceUtility.js"; const expectedStatuses = ["200"]; /** Creates a new thread. Threads contain messages and can be run by agents. */ export async function createThread( context: Client, - options?: CreateThreadParameters, + options: CreateThreadParameters = { body: {} }, ): Promise { validateCreateThreadParameters(options); - return TracingUtility.withSpan("CreateThread", options || { body: {} }, async (updatedOptions) => { + return TracingUtility.withSpan("CreateThread", options, async (updatedOptions) => { const result = await context.path("/threads").post(updatedOptions); if (!expectedStatuses.includes(result.status)) { throw createRestError(result); @@ -29,48 +30,54 @@ export async function createThread( export async function getThread( context: Client, threadId: string, - options?: GetThreadParameters, + options: GetThreadParameters = {}, ): Promise { validateThreadId(threadId); - const result = await context - .path("/threads/{threadId}", threadId) - .get(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("GetThread", options, async (updatedOptions) => { + const result = await context + .path("/threads/{threadId}", threadId) + .get(updatedOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { threadId: threadId } })); } /** Modifies an existing thread. */ export async function updateThread( context: Client, threadId: string, - options?: UpdateThreadParameters, + options: UpdateThreadParameters = { body: {} }, ): Promise { validateUpdateThreadParameters(threadId, options); - const result = await context - .path("/threads/{threadId}", threadId) - .post(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("UpdateThread", options, async (updatedOptions) => { + const result = await context + .path("/threads/{threadId}", threadId) + .post(updatedOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { threadId: threadId } })); } /** Deletes an existing thread. */ export async function deleteThread( context: Client, threadId: string, - options?: DeleteThreadParameters, + options: DeleteThreadParameters = {}, ): Promise { validateThreadId(threadId); - const result = await context - .path("/threads/{threadId}", threadId) - .delete(options); - if (!expectedStatuses.includes(result.status)) { - throw createRestError(result); - } - return result.body; + return TracingUtility.withSpan("DeleteThread", options, async (updatedOptions) => { + const result = await context + .path("/threads/{threadId}", threadId) + .delete(updatedOptions); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + return result.body; + }, (span, updatedOptions) => traceStartAgentGeneric(span, { ...updatedOptions, tracingAttributeOptions: { threadId: threadId } })); } diff --git a/sdk/ai/ai-projects/src/agents/threadsTrace.ts b/sdk/ai/ai-projects/src/agents/threadsTrace.ts index 5c9df8b4d142..859bb8bcc3b2 100644 --- a/sdk/ai/ai-projects/src/agents/threadsTrace.ts +++ b/sdk/ai/ai-projects/src/agents/threadsTrace.ts @@ -2,12 +2,12 @@ // Licensed under the MIT License. import { AgentThreadOutput } from "../generated/src/outputModels.js"; -import { TracingAttributes, TracingUtility, TracingOperationName, Span } from "../tracing.js"; +import { TracingUtility, TracingOperationName, Span } from "../tracing.js"; import { CreateThreadParameters } from "../generated/src/parameters.js"; -import { addMessageEvent } from "./traceUtility.js"; +import { addMessageEvent, UpdateWithAgentAttributes } from "./traceUtility.js"; export function traceStartCreateThread(span: Span, options: CreateThreadParameters): void { - TracingUtility.setSpanAttributes(span, TracingOperationName.CREATE_THREAD, { genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }); + TracingUtility.setSpanAttributes(span, TracingOperationName.CREATE_THREAD, UpdateWithAgentAttributes({})); setSpanEvents(span, options); } diff --git a/sdk/ai/ai-projects/src/agents/traceUtility.ts b/sdk/ai/ai-projects/src/agents/traceUtility.ts index 6590da35e1d3..75655514c55a 100644 --- a/sdk/ai/ai-projects/src/agents/traceUtility.ts +++ b/sdk/ai/ai-projects/src/agents/traceUtility.ts @@ -2,9 +2,23 @@ // Licensed under the MIT License. import { AgentsApiResponseFormat, AgentsApiResponseFormatOption, MessageContent, RunStepCompletionUsageOutput, ThreadMessage, ThreadMessageOptions, ToolOutput } from "./inputOutputs.js"; -import { Span, TracingAttributes, TracingUtility } from "../tracing.js"; +import { OptionsWithTracing, Span, TracingAttributeOptions, TracingAttributes, TracingUtility } from "../tracing.js"; import { getTelemetryOptions } from "../telemetry/telemetry.js"; +export function traceStartAgentGeneric (span: Span, options: Options): void { + const attributeOptions = options.tracingAttributeOptions || {} ; + TracingUtility.setSpanAttributes(span, options.tracingAttributeOptions?.operationName || "Agent_Operation" , UpdateWithAgentAttributes(attributeOptions)) +} +export function traceEndAgentGeneric (span: Span, _options: Options): void { + const attributeOptions = {}; + TracingUtility.updateSpanAttributes(span, UpdateWithAgentAttributes(attributeOptions)) +} + +export function UpdateWithAgentAttributes(attributeOptions: Omit): Omit { + attributeOptions.genAiSystem = TracingAttributes.AZ_AI_AGENT_SYSTEM + return attributeOptions; +} + /** * Adds a message event to the span. * @param span - The span to add the event to. @@ -48,9 +62,12 @@ export function addMessageEvent(span: Span, messageAttributes: ThreadMessageOpti */ export function addInstructionsEvent(span: Span, instructionAttributes: { instructions?: string | null, additional_instructions?: string | null, threadId?: string, agentId?: string }): void { const eventBody: Record = {}; - eventBody.content = instructionAttributes.instructions && instructionAttributes.additional_instructions ? `${instructionAttributes.instructions} ${instructionAttributes.additional_instructions}` : instructionAttributes.instructions || instructionAttributes.additional_instructions; + if (instructionAttributes.instructions || instructionAttributes.additional_instructions) { + eventBody.content = instructionAttributes.instructions && instructionAttributes.additional_instructions ? `${instructionAttributes.instructions} ${instructionAttributes.additional_instructions}` : instructionAttributes.instructions || instructionAttributes.additional_instructions; + } const attributes = { eventContent: JSON.stringify(eventBody), threadId: instructionAttributes.threadId, agentId: instructionAttributes.agentId, genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }; TracingUtility.addSpanEvent(span, "gen_ai.system.message", attributes); + } /** @@ -75,8 +92,8 @@ export function formatAgentApiResponse(responseFormat: AgentsApiResponseFormatOp */ export function addToolMessagesEvent(span: Span, tool_outputs: Array): void { tool_outputs.forEach(tool_output => { - const eventBody = {"content": tool_output.output, "id": tool_output.tool_call_id} - TracingUtility.addSpanEvent(span, "gen_ai.tool.message", {eventContent: JSON.stringify(eventBody), genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM}); + const eventBody = { "content": tool_output.output, "id": tool_output.tool_call_id } + TracingUtility.addSpanEvent(span, "gen_ai.tool.message", { eventContent: JSON.stringify(eventBody), genAiSystem: TracingAttributes.AZ_AI_AGENT_SYSTEM }); }); } @@ -89,8 +106,8 @@ function getMessageContent(messageContent: string | MessageContent[]): string | const contentBody: { [key: string]: any } = {}; messageContent.forEach(content => { const typedContent = content.type; - const {value, annotations} = (content as MessageContentExtended)[typedContent]; - contentBody[typedContent] = {value, annotations}; + const { value, annotations } = (content as MessageContentExtended)[typedContent]; + contentBody[typedContent] = { value, annotations }; }); return contentBody; } diff --git a/sdk/ai/ai-projects/src/aiProjectsClient.ts b/sdk/ai/ai-projects/src/aiProjectsClient.ts index 38b09c559f75..b1edc5576199 100644 --- a/sdk/ai/ai-projects/src/aiProjectsClient.ts +++ b/sdk/ai/ai-projects/src/aiProjectsClient.ts @@ -2,17 +2,18 @@ // Licensed under the MIT License. import { TokenCredential } from "@azure/core-auth"; import createClient, { ProjectsClientOptions } from "./generated/src/projectsClient.js"; -import { Client } from "@azure-rest/core-client"; import { AgentsOperations, getAgentsOperations } from "./agents/index.js"; import { ConnectionsOperations, getConnectionsOperations } from "./connections/index.js"; import { getTelemetryOperations, TelemetryOperations } from "./telemetry/index.js"; +import { Client } from "@azure-rest/core-client"; -export interface AIProjectsClientOptions extends ProjectsClientOptions{ +export interface AIProjectsClientOptions extends ProjectsClientOptions { } export class AIProjectsClient { private _client: Client; private _connectionClient: Client; + private _telemetryClient: Client; /* * @param endpointParam - The Azure AI Studio project endpoint, in the form `https://.api.azureml.ms` or `https://..api.azureml.ms`, where is the Azure region where the project is deployed (e.g. westus) and is the GUID of the Enterprise private link. @@ -38,14 +39,23 @@ export class AIProjectsClient { credential, options, ); + this._connectionClient = createClient(endpointParam, subscriptionId, resourceGroupName, projectName, credential, - {...options, endpoint:connectionEndPoint}) + { ...options, endpoint: connectionEndPoint }) + + this._telemetryClient = createClient(endpointParam, subscriptionId, + resourceGroupName, + projectName, + credential, + { ...options, apiVersion: "2020-02-02", endpoint: 'https://management.azure.com' }) + this.agents = getAgentsOperations(this._client); this.connections = getConnectionsOperations(this._connectionClient); - this.telemetry = getTelemetryOperations(this.connections); + this.telemetry = getTelemetryOperations(this._telemetryClient, this.connections); + } /** diff --git a/sdk/ai/ai-projects/src/telemetry/index.ts b/sdk/ai/ai-projects/src/telemetry/index.ts index 8d10a28050bc..9857b7acfda3 100644 --- a/sdk/ai/ai-projects/src/telemetry/index.ts +++ b/sdk/ai/ai-projects/src/telemetry/index.ts @@ -3,8 +3,9 @@ // Licensed under the MIT License. +import { Client } from "@azure-rest/core-client"; import { ConnectionsOperations } from "../connections/index.js"; -import {getConnectionString, getTelemetryOptions, resetTelemetryOptions, TelemetryOptions, updateTelemetryOptions } from "./telemetry.js"; +import { getConnectionString, getTelemetryOptions, resetTelemetryOptions, TelemetryOptions, updateTelemetryOptions } from "./telemetry.js"; /** * Telemetry operations @@ -14,20 +15,20 @@ export interface TelemetryOperations { * Get the appinsights connection string confired in the workspace * @returns The telemetry connection string **/ - getConnectionString() : Promise + getConnectionString(): Promise /** * Update the telemetry settings * @param options - The telemetry options * @returns void * */ - updateSettings(options: TelemetryOptions) : void + updateSettings(options: TelemetryOptions): void /** * get the telemetry settings * @returns The telemetry options * */ - getSettings() : TelemetryOptions + getSettings(): TelemetryOptions } /** @@ -35,11 +36,11 @@ export interface TelemetryOperations { * @param connection - The connections operations * @returns The telemetry operations **/ -export function getTelemetryOperations(connection: ConnectionsOperations): TelemetryOperations { +export function getTelemetryOperations(context: Client, connection: ConnectionsOperations): TelemetryOperations { resetTelemetryOptions(); return { - getConnectionString: () => getConnectionString(connection), - updateSettings : (options: TelemetryOptions) => updateTelemetryOptions(options), - getSettings : () => getTelemetryOptions() + getConnectionString: () => getConnectionString(context, connection), + updateSettings: (options: TelemetryOptions) => updateTelemetryOptions(options), + getSettings: () => getTelemetryOptions() } } diff --git a/sdk/ai/ai-projects/src/telemetry/telemetry.ts b/sdk/ai/ai-projects/src/telemetry/telemetry.ts index d0acb961156f..a69b809f28a2 100644 --- a/sdk/ai/ai-projects/src/telemetry/telemetry.ts +++ b/sdk/ai/ai-projects/src/telemetry/telemetry.ts @@ -3,6 +3,10 @@ // Licensed under the MIT License. import { ConnectionsOperations } from "../connections/index.js"; +import { GetAppInsightsResponseOutput } from "../agents/inputOutputs.js"; +import { Client, createRestError } from "@azure-rest/core-client"; + +const expectedStatuses = ["200"]; /** * Telemetry options @@ -45,11 +49,15 @@ export function resetTelemetryOptions(): void { * @param connection - get the connection string * @returns The telemetry connection string */ -export async function getConnectionString(connection: ConnectionsOperations): Promise { +export async function getConnectionString(context: Client, connection: ConnectionsOperations): Promise { if (!telemetryOptions.connectionString) { const workspace = await connection.getWorkspace(); if (workspace.properties.applicationInsights) { - telemetryOptions.connectionString = workspace.properties.applicationInsights + const result = await context.path("/{appInsightsResourceUrl}", workspace.properties.applicationInsights).get(); + if (!expectedStatuses.includes(result.status)) { + throw createRestError(result); + } + telemetryOptions.connectionString = (result.body as GetAppInsightsResponseOutput).properties.ConnectionString; } else { throw new Error("Application Insights connection string not found.") diff --git a/sdk/ai/ai-projects/src/tracing.ts b/sdk/ai/ai-projects/src/tracing.ts index 4ac15c66c1b7..7bf17dcbb566 100644 --- a/sdk/ai/ai-projects/src/tracing.ts +++ b/sdk/ai/ai-projects/src/tracing.ts @@ -36,6 +36,7 @@ export enum TracingAttributes { } export enum TracingOperationName { CREATE_AGENT = "create_agent", + CREATE_UPDATE_AGENT = "update_agent", CREATE_THREAD = "create_thread", CREATE_MESSAGE = "create_message", CREATE_RUN = "create_run", @@ -73,6 +74,8 @@ export interface TracingAttributeOptions { } export type Span = Omit; +export type OptionsWithTracing = { tracingOptions?: OperationTracingOptions, tracingAttributeOptions?: TracingAttributeOptions}; + export class TracingUtility { private static tracingClient = createTracingClient({ namespace: "Microsoft.CognitiveServices", @@ -81,7 +84,7 @@ export class TracingUtility { }); - static async withSpan ReturnType>(name: string, options: Options, request: Request, startTrace?: (span: Span, updatedOptions: Options,) => void, endTrace?: (span: Span, updatedOptions: Options, result: ReturnType) => void, @@ -89,6 +92,7 @@ export class TracingUtility { return TracingUtility.tracingClient.withSpan(name, options, async (updatedOptions: Options, span: Span) => { if (startTrace) { try { + updatedOptions.tracingAttributeOptions = {...updatedOptions.tracingAttributeOptions, operationName: name}; startTrace(span, updatedOptions); } catch (e) { diff --git a/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts b/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts index 15037464e840..36ba7bbc96e1 100644 --- a/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts +++ b/sdk/ai/ai-projects/test/public/agents/tracing.spec.ts @@ -13,10 +13,10 @@ interface ExtendedMockTrackingSpan extends MockTracingSpan { } class ExtendedMockInstrumenter extends MockInstrumenter { extendSpan(span: any): void { + if (!span.events) { + span.events = []; + } span.addEvent = (eventName: string, options?: AddEventOptions) => { - if (!span.events) { - span.events = []; - } span.events.push({ name: eventName, ...options }); } } @@ -77,12 +77,12 @@ describe("Agent Tracing", () => { await agents.createRun("threadId", "agentId"); const mockedInstrumenter = instrumenter as MockInstrumenter; assert.isAbove(mockedInstrumenter.startedSpans.length, 0); - const span = mockedInstrumenter.startedSpans[0]; + const span = mockedInstrumenter.startedSpans[0] as ExtendedMockTrackingSpan; assert.equal(span.attributes["gen_ai.thread.id"], runResponse.thread_id); assert.equal(span.attributes["gen_ai.operation.name"], "create_run"); assert.equal(span.attributes["gen_ai.agent.id"], runResponse.assistant_id); assert.equal(span.attributes["gen_ai.thread.run.status"], runResponse.status); - + assert.equal(span.events!.length, 1); }) it("create Thread", async function () { From 3262c60cd93239d3d850c1b17e238cb03dda1f34 Mon Sep 17 00:00:00 2001 From: Ganesh Bheemarasetty Date: Sun, 8 Dec 2024 20:42:39 -0800 Subject: [PATCH 9/9] Address test failures in playback mode --- sdk/ai/ai-projects/assets.json | 2 +- .../samples-dev/agents/agentCreateWithTracingConsole.ts | 4 +++- sdk/ai/ai-projects/src/telemetry/telemetry.ts | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/sdk/ai/ai-projects/assets.json b/sdk/ai/ai-projects/assets.json index 083f689ef59a..1e6703b1d480 100644 --- a/sdk/ai/ai-projects/assets.json +++ b/sdk/ai/ai-projects/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "js", "TagPrefix": "js/ai/ai-projects", - "Tag": "js/ai/ai-projects_1b6cffbf79" + "Tag": "js/ai/ai-projects_57b254f3af" } diff --git a/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingConsole.ts b/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingConsole.ts index 43dac37bcfe6..e12df5fa5fad 100644 --- a/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingConsole.ts +++ b/sdk/ai/ai-projects/samples-dev/agents/agentCreateWithTracingConsole.ts @@ -63,7 +63,7 @@ export async function main(): Promise { console.log(`Created Thread, thread ID: ${thread.id}`); // Create message - const message = await client.agents.createMessage(thread.id, { role: "user", content: "What's the weather like in my favorite city?" }) + const message = await client.agents.createMessage(thread.id, { role: "user", content: "Hello, tell me a joke" }) console.log(`Created message, message ID ${message.id}`); // Create run @@ -80,6 +80,8 @@ export async function main(): Promise { console.log(`Deleted agent`); + await client.agents.listMessages(thread.id) + span.end(); }); } diff --git a/sdk/ai/ai-projects/src/telemetry/telemetry.ts b/sdk/ai/ai-projects/src/telemetry/telemetry.ts index a69b809f28a2..f5f1e1f25261 100644 --- a/sdk/ai/ai-projects/src/telemetry/telemetry.ts +++ b/sdk/ai/ai-projects/src/telemetry/telemetry.ts @@ -53,7 +53,7 @@ export async function getConnectionString(context: Client, connection: Connectio if (!telemetryOptions.connectionString) { const workspace = await connection.getWorkspace(); if (workspace.properties.applicationInsights) { - const result = await context.path("/{appInsightsResourceUrl}", workspace.properties.applicationInsights).get(); + const result = await context.path("/{appInsightsResourceUrl}", workspace.properties.applicationInsights).get({skipUrlEncoding:true}); if (!expectedStatuses.includes(result.status)) { throw createRestError(result); }