Allow running tests with recordings (#31935)

Serialize tests, define sanitization rules and connect to test recordings. --------- Co-authored-by: Grace Brigham <[email protected]>
Azure · Nov 25, 2024 · 658b933 · 658b933
1 parent 074f6d9
commit 658b933
Show file tree

Hide file tree

Showing 11 changed files with 64 additions and 44 deletions.
diff --git a/sdk/ai/ai-projects/assets.json b/sdk/ai/ai-projects/assets.json
@@ -0,0 +1,6 @@
+{
+  "AssetsRepo": "Azure/azure-sdk-assets",
+  "AssetsRepoPrefixPath": "js",
+  "TagPrefix": "js/ai/ai-projects",
+  "Tag": "js/ai/ai-projects_dd39266c76"
+}
diff --git a/sdk/ai/ai-projects/test/public/agents/assistants.spec.ts b/sdk/ai/ai-projects/test/public/agents/assistants.spec.ts
@@ -49,7 +49,7 @@ describe("Agents - assistants", () => {
     console.log(`Listed agents, agents count: ${assistants.data.length}`);
 
     // Delete agent
-    agents.deleteAgent(agent.id);
+    await agents.deleteAgent(agent.id);
     console.log(`Deleted agent, agent ID: ${agent.id}`);
   });
 
@@ -61,7 +61,7 @@ describe("Agents - assistants", () => {
     assert.isNotNull(agent.id);
 
     // Delete agent
-    agents.deleteAgent(agent.id);
+    await agents.deleteAgent(agent.id);
     console.log(`Deleted agent, agent ID: ${agent.id}`);
   })
 
@@ -77,7 +77,7 @@ describe("Agents - assistants", () => {
     console.log(`Updated agent name to ${updated.name}, agent ID: ${agent.id}`);
 
     // Delete agent
-    agents.deleteAgent(agent.id);
+    await agents.deleteAgent(agent.id);
     console.log(`Deleted agent, agent ID: ${agent.id}`);
   });
 

diff --git a/sdk/ai/ai-projects/test/public/agents/functionTool.spec.ts b/sdk/ai/ai-projects/test/public/agents/functionTool.spec.ts
@@ -24,15 +24,12 @@ describe("Agents - function tool", () => {
         await recorder.stop();
     });
 
-
 function getCurrentDateTime(): {} {
-    const current = Date.now();
-    return { "currentDateTime": current };
+    return { "currentDateTime": "2024-10-10 12:30:19" };
 }
 
 it("should create agent with function tool", async function () {
     // Create agent
-
     const agent = await agents.createAgent("gpt-4o", { name: "my-agent", instructions: "You are a helpful agent", tools: [getCurrentDateTimeTool] })
     console.log(`Created agent, agent ID: ${agent.id}`);
     assert.isNotNull(agent);
@@ -41,13 +38,12 @@ it("should create agent with function tool", async function () {
     assert.equal((agent.tools[0] as FunctionToolDefinition).function.name, "getCurrentDateTime");
 
     // Delete agent
-    agents.deleteAgent(agent.id);
+    await agents.deleteAgent(agent.id);
     console.log(`Deleted agent, agent ID: ${agent.id}`);
 })
 
 it("should create agent with run function tool", async function () {
     // Create agent
-
     const agent = await agents.createAgent("gpt-4o", { name: "my-agent", instructions: "You are a helpful agent", tools: [getCurrentDateTimeTool] })
     console.log(`Created agent, agent ID: ${agent.id}`);
     assert.isNotNull(agent);
@@ -109,10 +105,9 @@ it("should create agent with run function tool", async function () {
             }
         });
     });
+
     // Delete agent
-    agents.deleteAgent(agent.id);
+    await agents.deleteAgent(agent.id);
     console.log(`Deleted agent, agent ID: ${agent.id}`);
 })
 });
-
-
diff --git a/sdk/ai/ai-projects/test/public/agents/messages.spec.ts b/sdk/ai/ai-projects/test/public/agents/messages.spec.ts
@@ -38,7 +38,7 @@ describe("Agents - messages", () => {
     assert.isNotNull(message.id);
 
     // Delete thread
-    agents.deleteThread(thread.id);
+    await agents.deleteThread(thread.id);
     console.log(`Deleted thread, thread ID: ${thread.id}`);
   });
 
@@ -61,7 +61,7 @@ describe("Agents - messages", () => {
     console.log(`Listed ${messages.data.length} messages, thread ID: ${thread.id}`);
 
     // Delete thread
-    agents.deleteThread(thread.id);
+    await agents.deleteThread(thread.id);
     console.log(`Deleted thread, thread ID: ${thread.id}`);
   });
 
@@ -82,7 +82,7 @@ describe("Agents - messages", () => {
     console.log(`Updated message to have metadata "key":"${updatedMessage.metadata?.key}", message ID: ${updatedMessage.id}`);
 
     // Delete thread
-    agents.deleteThread(thread.id);
+    await agents.deleteThread(thread.id);
     console.log(`Deleted thread, thread ID: ${thread.id}`);
   });
 });
diff --git a/sdk/ai/ai-projects/test/public/agents/runSteps.spec.ts b/sdk/ai/ai-projects/test/public/agents/runSteps.spec.ts
@@ -52,9 +52,9 @@ describe("Agents - run steps", () => {
     assert.isNotNull(runSteps.data);
 
     // Clean up
-    agents.deleteThread(thread.id);
+    await agents.deleteThread(thread.id);
     console.log(`Deleted thread, thread ID: ${thread.id}`);
-    agents.deleteAgent(agent.id);
+    await agents.deleteAgent(agent.id);
     console.log(`Deleted agent, agent ID: ${agent.id}`);
   });
 
@@ -99,9 +99,9 @@ describe("Agents - run steps", () => {
     assert.equal(step.id, stepId);
 
     // Clean up
-    agents.deleteThread(thread.id);
+    await agents.deleteThread(thread.id);
     console.log(`Deleted thread, thread ID: ${thread.id}`);
-    agents.deleteAgent(agent.id);
+    await agents.deleteAgent(agent.id);
     console.log(`Deleted agent, agent ID: ${agent.id}`);
   });
 

diff --git a/sdk/ai/ai-projects/test/public/agents/runs.spec.ts b/sdk/ai/ai-projects/test/public/agents/runs.spec.ts
@@ -41,9 +41,9 @@ describe("Agents - Run", () => {
     console.log(`Created Run, Run ID:  ${run.id}`);
 
     // Delete agent and thread
-    agents.deleteAgent(agent.id);
+    await agents.deleteAgent(agent.id);
     console.log(`Deleted agent, agent ID:  ${agent.id}`);
-    agents.deleteThread(thread.id);
+    await agents.deleteThread(thread.id);
     console.log(`Deleted Thread, thread ID:  ${thread.id}`);
   })
 
@@ -74,9 +74,9 @@ describe("Agents - Run", () => {
     console.log(`Retrieved run, Run ID:  ${runDetails.id}`);
 
     // Delete agent and thread
-    agents.deleteAgent(agent.id);
+    await agents.deleteAgent(agent.id);
     console.log(`Deleted agent, agent ID:  ${agent.id}`);
-    agents.deleteThread(thread.id);
+    await agents.deleteThread(thread.id);
     console.log(`Deleted Thread, thread ID:  ${thread.id}`);
   })
 
@@ -131,9 +131,9 @@ describe("Agents - Run", () => {
     console.log(`Run status - ${runDetails.status}, run ID: ${runDetails.id}`);
 
     // Delete agent and thread
-    agents.deleteAgent(agent.id);
+    await agents.deleteAgent(agent.id);
     console.log(`Deleted agent, agent ID:  ${agent.id}`);
-    agents.deleteThread(thread.id);
+    await agents.deleteThread(thread.id);
     console.log(`Deleted Thread, thread ID:  ${thread.id}`);
   });
 

diff --git a/sdk/ai/ai-projects/test/public/agents/streaming.spec.ts b/sdk/ai/ai-projects/test/public/agents/streaming.spec.ts
@@ -59,9 +59,9 @@ describe("Agents - streaming", () => {
     assert.isNotNull(streamEventMessages);
 
     // Delete agent and thread
-    agents.deleteAgent(agent.id);
+    await agents.deleteAgent(agent.id);
     console.log(`Deleted agent, agent ID:  ${agent.id}`);
-    agents.deleteThread(thread.id);
+    await agents.deleteThread(thread.id);
     console.log(`Deleted Thread, thread ID:  ${thread.id}`);
   });
 
@@ -97,7 +97,7 @@ describe("Agents - streaming", () => {
     assert.isNotNull(streamEventMessages);
 
     // Delete agent
-    agents.deleteAgent(agent.id);
+    await agents.deleteAgent(agent.id);
     console.log(`Deleted agent, agent ID:  ${agent.id}`);
   });
 });
diff --git a/sdk/ai/ai-projects/test/public/agents/threads.spec.ts b/sdk/ai/ai-projects/test/public/agents/threads.spec.ts
@@ -34,7 +34,7 @@ describe("Agents - threads", () => {
     assert.isNotNull(thread.id);
 
     // Delete thread
-    agents.deleteThread(thread.id);
+    await agents.deleteThread(thread.id);
     console.log(`Deleted thread, thread ID: ${thread.id}`);
   });
 
@@ -50,7 +50,7 @@ describe("Agents - threads", () => {
     console.log(`Retrieved thread, thread ID: ${_thread.id}`);
 
     // Delete thread
-    agents.deleteThread(thread.id);
+    await agents.deleteThread(thread.id);
     console.log(`Deleted thread, thread ID: ${thread.id}`);
   });
 
@@ -68,7 +68,7 @@ describe("Agents - threads", () => {
     console.log(`Updated thread to have metadata "key":"${_thread.metadata?.key}", thread ID: ${_thread.id}`);
 
     // Delete thread
-    agents.deleteThread(thread.id);
+    await agents.deleteThread(thread.id);
     console.log(`Deleted thread, thread ID: ${thread.id}`);
   });
 

diff --git a/sdk/ai/ai-projects/test/public/agents/vectorStores.spec.ts b/sdk/ai/ai-projects/test/public/agents/vectorStores.spec.ts
@@ -33,7 +33,7 @@ describe("Agents - vector stores", () => {
     console.log(`Created vector store, vector store ID: ${vectorStore.id}`);
 
     // Delete vector store
-    agents.deleteVectorStore(vectorStore.id);
+    await agents.deleteVectorStore(vectorStore.id);
     console.log(`Deleted vector store, vector store ID: ${vectorStore.id}`);
   });
 
@@ -59,7 +59,7 @@ describe("Agents - vector stores", () => {
     console.log(`Retrieved vector store, vector store ID: ${_vectorStore.id}`);
 
     // Delete vector store
-    agents.deleteVectorStore(vectorStore.id);
+    await agents.deleteVectorStore(vectorStore.id);
     console.log(`Deleted vector store, vector store ID: ${vectorStore.id}`);
   });
 
@@ -75,7 +75,7 @@ describe("Agents - vector stores", () => {
     console.log(`Listed ${vectorStores.data.length} vector stores`);
 
     // Delete vector store
-    agents.deleteVectorStore(vectorStore.id);
+    await agents.deleteVectorStore(vectorStore.id);
     console.log(`Deleted vector store, vector store ID: ${vectorStore.id}`);
   });
 
@@ -91,7 +91,7 @@ describe("Agents - vector stores", () => {
     console.log(`Updated vector store name to ${updatedVectorStore.name}, vector store ID: ${updatedVectorStore.id}`);
 
     // Delete vector store
-    agents.deleteVectorStore(vectorStore.id);
+    await agents.deleteVectorStore(vectorStore.id);
     console.log(`Deleted vector store, vector store ID: ${vectorStore.id}`);
   });
 

diff --git a/sdk/ai/ai-projects/test/public/connections/connections.spec.ts b/sdk/ai/ai-projects/test/public/connections/connections.spec.ts
@@ -49,13 +49,13 @@ describe("Agents - assistants", () => {
     assert.isAtLeast(connectionsList.value.length, 1);
 
     // Retrieve one connection
-    connectionsList.value.forEach(async (i) => {
-      const connectionName = i.name;
+    for (const _connection of connectionsList.value) {
+      const connectionName = _connection.name;
       const connection = await connections.getConnection(connectionName);
       assert.isNotNull(connection);
       assert.equal(connection.name, connectionName);
-      console.log(`Retrieved connection, connection name: ${i.name}`);
-    });
+      console.log(`Retrieved connection, connection name: ${connection.name}`);
+    }
   });
 
   it("should retrieve a connection with secrets", async function () {
@@ -65,13 +65,13 @@ describe("Agents - assistants", () => {
     assert.isAtLeast(connectionsList.value.length, 1);
 
     // Retrieve one connection with secrets
-    connectionsList.value.forEach(async (i) => {
-      const connectionName = i.name;
+    for (const _connection of connectionsList.value) {
+      const connectionName = _connection.name;
       const connection = await connections.getConnectionWithSecrets(connectionName);
       assert.isNotNull(connection);
       assert.equal(connection.name, connectionName);
-      console.log(`Retrieved connection with secrets, connection name: ${i.name}`);
-    });
+      console.log(`Retrieved connection with secrets, connection name: ${connection.name}`);
+    }
   });
 
 });
diff --git a/sdk/ai/ai-projects/test/public/utils/createClient.ts b/sdk/ai/ai-projects/test/public/utils/createClient.ts
@@ -11,11 +11,30 @@ import { AIProjectsClient } from "../../../src/index.js";
 import { ClientOptions } from "@azure-rest/core-client";
 
 const replaceableVariables: Record<string, string> = {
-  SUBSCRIPTION_ID: "azure_subscription_id",
+  SUBSCRIPTION_ID: "00000000-0000-0000-0000-000000000000",
+  RESOURCE_GROUP_NAME: "00000",
+  WORKSPACE_NAME: "00000",
+  DATASET_NAME: "00000",
+  TENANT_ID: "00000000-0000-0000-0000-000000000000",
+  USER_OBJECT_ID: "00000000-0000-0000-0000-000000000000",
+  API_KEY: "00000000000000000000000000000000000000000000000000000000000000000000",
+  AZURE_AI_PROJECTS_CONNECTION_STRING: `Sanitized.api.azureml.ms;00000000-0000-0000-0000-000000000000;00000;00000`
 };
 
 const recorderEnvSetup: RecorderStartOptions = {
   envSetupForPlayback: replaceableVariables,
+  sanitizerOptions: {
+    generalSanitizers: [
+      { regex: true, target: "(%2F|/)?subscriptions(%2F|/)([-\\w\\._\\(\\)]+)", value: replaceableVariables.SUBSCRIPTION_ID, groupForReplace: "3" },
+      { regex: true, target: "(%2F|/)?resource[gG]roups(%2F|/)([-\\w\\._\\(\\)]+)", value: replaceableVariables.RESOURCE_GROUP_NAME, groupForReplace: "3"  },
+      { regex: true, target: "/workspaces/([-\\w\\._\\(\\)]+)", value: replaceableVariables.WORKSPACE_NAME, groupForReplace: "1"  },
+    ],
+    bodyKeySanitizers: [
+      { jsonPath: "properties.ConnectionString", value: "InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://region.applicationinsights.azure.com/;LiveEndpoint=https://region.livediagnostics.monitor.azure.com/;ApplicationId=00000000-0000-0000-0000-000000000000"},
+      { jsonPath: "properties.credentials.key", value: replaceableVariables.API_KEY },
+    ]
+  },
+  removeCentralSanitizers: ["AZSDK3430", "AZSDK3493"]
 };
 
 /**