From e63079d3d3af18a3af27914e842d915f307b38b3 Mon Sep 17 00:00:00 2001 From: chungjac Date: Wed, 11 Dec 2024 12:59:40 -0800 Subject: [PATCH] telemetry(amazonq): cleanup amazonq_utgGenerateTests logic #6212 ## Problem - Repetitive code when emitting amazonq_utgGenerateTests telemetry events - Metric has already been migrated to aws-toolkit-common ## Solution - Created sendTestGenerationToolkitEvent helper to extract common fields - Removed metric override in this repo - Tested and can confirm that metrics are same as previous method --- package-lock.json | 8 +- package.json | 2 +- .../amazonqTest/chat/controller/controller.ts | 110 +++++++------- .../chat/controller/messenger/messenger.ts | 38 +++-- .../src/codewhisperer/util/telemetryHelper.ts | 45 ++++++ .../src/shared/telemetry/vscodeTelemetry.json | 136 ------------------ 6 files changed, 119 insertions(+), 220 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1a84022b32f..db823c798b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,7 @@ "vscode-nls-dev": "^4.0.4" }, "devDependencies": { - "@aws-toolkits/telemetry": "^1.0.284", + "@aws-toolkits/telemetry": "^1.0.287", "@playwright/browser-chromium": "^1.43.1", "@stylistic/eslint-plugin": "^2.11.0", "@types/he": "^1.2.3", @@ -5136,9 +5136,9 @@ } }, "node_modules/@aws-toolkits/telemetry": { - "version": "1.0.285", - "resolved": "https://registry.npmjs.org/@aws-toolkits/telemetry/-/telemetry-1.0.285.tgz", - "integrity": "sha512-O5/kbCE9cXF8scL5XmeDjMX9ojmCLvXg6cwcBayTS4URypI6XFat6drmaIF/QoDqxAfnHLHs0zypOdqSWCDr8w==", + "version": "1.0.287", + "resolved": "https://registry.npmjs.org/@aws-toolkits/telemetry/-/telemetry-1.0.287.tgz", + "integrity": "sha512-qK2l8Fv5Cvs865ap2evf4ikBREg33/jGw0lgxolqZLdHwm5zm/DkR9vNyqwhDlqDRlSgSlros3Z8zaiSBVRYVQ==", "dev": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index 5e5a67af5ed..20d53676e49 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "generateNonCodeFiles": "npm run generateNonCodeFiles -w packages/ --if-present" }, "devDependencies": { - "@aws-toolkits/telemetry": "^1.0.284", + "@aws-toolkits/telemetry": "^1.0.287", "@playwright/browser-chromium": "^1.43.1", "@stylistic/eslint-plugin": "^2.11.0", "@types/he": "^1.2.3", diff --git a/packages/core/src/amazonqTest/chat/controller/controller.ts b/packages/core/src/amazonqTest/chat/controller/controller.ts index 88490315e45..8eb75e38294 100644 --- a/packages/core/src/amazonqTest/chat/controller/controller.ts +++ b/packages/core/src/amazonqTest/chat/controller/controller.ts @@ -242,22 +242,20 @@ export class TestController { this.messenger.sendUpdatePromptProgress(data.tabID, null) const session = this.sessionStorage.getSession() const isCancel = data.error.message === unitTestGenerationCancelMessage - telemetry.amazonq_utgGenerateTests.emit({ - cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext', - jobId: session.listOfTestGenerationJobId[0], // For RIV, UTG does only one StartTestGeneration API call - jobGroup: session.testGenerationJobGroupName, - requestId: session.startTestGenerationRequestId, - hasUserPromptSupplied: session.hasUserPromptSupplied, - isCodeBlockSelected: session.isCodeBlockSelected, - buildPayloadBytes: session.srcPayloadSize, - buildZipFileBytes: session.srcZipFileSize, - artifactsUploadDuration: session.artifactsUploadDuration, - perfClientLatency: performance.now() - session.testGenerationStartTime, - result: isCancel ? 'Cancelled' : 'Failed', - reasonDesc: getTelemetryReasonDesc(data.error), - isSupportedLanguage: true, - credentialStartUrl: AuthUtil.instance.startUrl, - }) + + TelemetryHelper.instance.sendTestGenerationToolkitEvent( + session, + true, + isCancel ? 'Cancelled' : 'Failed', + session.startTestGenerationRequestId, + performance.now() - session.testGenerationStartTime, + getTelemetryReasonDesc(data.error), + session.isCodeBlockSelected, + session.artifactsUploadDuration, + session.srcPayloadSize, + session.srcZipFileSize + ) + if (session.stopIteration) { // Error from Science this.messenger.sendMessage(data.error.message.replaceAll('```', ''), data.tabID, 'answer') @@ -716,27 +714,25 @@ export class TestController { // TODO: send the message once again once build is enabled // this.messenger.sendMessage('Accepted', message.tabID, 'prompt') telemetry.ui_click.emit({ elementId: 'unitTestGeneration_acceptDiff' }) - telemetry.amazonq_utgGenerateTests.emit({ - generatedCount: session.numberOfTestsGenerated, - acceptedCount: session.numberOfTestsGenerated, - generatedCharactersCount: session.charsOfCodeGenerated, - acceptedCharactersCount: session.charsOfCodeAccepted, - generatedLinesCount: session.linesOfCodeGenerated, - acceptedLinesCount: session.linesOfCodeAccepted, - cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext', - jobId: session.listOfTestGenerationJobId[0], // For RIV, UTG does only one StartTestGeneration API call so jobId = session.listOfTestGenerationJobId[0] - jobGroup: session.testGenerationJobGroupName, - requestId: session.startTestGenerationRequestId, - buildPayloadBytes: session.srcPayloadSize, - buildZipFileBytes: session.srcZipFileSize, - artifactsUploadDuration: session.artifactsUploadDuration, - hasUserPromptSupplied: session.hasUserPromptSupplied, - isCodeBlockSelected: session.isCodeBlockSelected, - perfClientLatency: session.latencyOfTestGeneration, - isSupportedLanguage: true, - credentialStartUrl: AuthUtil.instance.startUrl, - result: 'Succeeded', - }) + + TelemetryHelper.instance.sendTestGenerationToolkitEvent( + session, + true, + 'Succeeded', + session.startTestGenerationRequestId, + session.latencyOfTestGeneration, + undefined, + session.isCodeBlockSelected, + session.artifactsUploadDuration, + session.srcPayloadSize, + session.srcZipFileSize, + session.charsOfCodeAccepted, + session.numberOfTestsGenerated, + session.linesOfCodeAccepted, + session.charsOfCodeGenerated, + session.numberOfTestsGenerated, + session.linesOfCodeGenerated + ) await this.endSession(message, FollowUpTypes.SkipBuildAndFinish) await this.sessionCleanUp() @@ -840,27 +836,25 @@ export class TestController { private async endSession(data: any, step: FollowUpTypes) { const session = this.sessionStorage.getSession() if (step === FollowUpTypes.RejectCode) { - telemetry.amazonq_utgGenerateTests.emit({ - generatedCount: session.numberOfTestsGenerated, - acceptedCount: 0, - generatedCharactersCount: session.charsOfCodeGenerated, - acceptedCharactersCount: 0, - generatedLinesCount: session.linesOfCodeGenerated, - acceptedLinesCount: 0, - cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext', - jobId: session.listOfTestGenerationJobId[0], // For RIV, UTG does only one StartTestGeneration API call so jobId = session.listOfTestGenerationJobId[0] - jobGroup: session.testGenerationJobGroupName, - requestId: session.startTestGenerationRequestId, - buildPayloadBytes: session.srcPayloadSize, - buildZipFileBytes: session.srcZipFileSize, - artifactsUploadDuration: session.artifactsUploadDuration, - hasUserPromptSupplied: session.hasUserPromptSupplied, - isCodeBlockSelected: session.isCodeBlockSelected, - perfClientLatency: session.latencyOfTestGeneration, - isSupportedLanguage: true, - credentialStartUrl: AuthUtil.instance.startUrl, - result: 'Succeeded', - }) + TelemetryHelper.instance.sendTestGenerationToolkitEvent( + session, + true, + 'Succeeded', + session.startTestGenerationRequestId, + session.latencyOfTestGeneration, + undefined, + session.isCodeBlockSelected, + session.artifactsUploadDuration, + session.srcPayloadSize, + session.srcZipFileSize, + 0, + 0, + 0, + session.charsOfCodeGenerated, + session.numberOfTestsGenerated, + session.linesOfCodeGenerated + ) + telemetry.ui_click.emit({ elementId: 'unitTestGeneration_rejectDiff' }) } diff --git a/packages/core/src/amazonqTest/chat/controller/messenger/messenger.ts b/packages/core/src/amazonqTest/chat/controller/messenger/messenger.ts index 745cc233154..10b496b69d3 100644 --- a/packages/core/src/amazonqTest/chat/controller/messenger/messenger.ts +++ b/packages/core/src/amazonqTest/chat/controller/messenger/messenger.ts @@ -36,9 +36,8 @@ import { CodeReference } from '../../../../amazonq/webview/ui/apps/amazonqCommon import { getHttpStatusCode, getRequestId, getTelemetryReasonDesc, ToolkitError } from '../../../../shared/errors' import { sleep, waitUntil } from '../../../../shared/utilities/timeoutUtils' import { keys } from '../../../../shared/utilities/tsUtils' -import { AuthUtil, testGenState } from '../../../../codewhisperer' +import { TelemetryHelper, testGenState } from '../../../../codewhisperer' import { cancellingProgressField, testGenCompletedField } from '../../../models/constants' -import { telemetry } from '../../../../shared/telemetry/telemetry' export type UnrecoverableErrorType = 'no-project-found' | 'no-open-file-found' | 'invalid-file-type' @@ -275,31 +274,28 @@ export class Messenger { .finally(async () => { if (testGenState.isCancelling()) { this.sendMessage(CodeWhispererConstants.unitTestGenerationCancelMessage, tabID, 'answer') - telemetry.amazonq_utgGenerateTests.emit({ - cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext', - hasUserPromptSupplied: session.hasUserPromptSupplied, - perfClientLatency: performance.now() - session.testGenerationStartTime, - result: 'Cancelled', - reasonDesc: getTelemetryReasonDesc(CodeWhispererConstants.unitTestGenerationCancelMessage), - isSupportedLanguage: false, - credentialStartUrl: AuthUtil.instance.startUrl, - requestId: messageId, - }) + TelemetryHelper.instance.sendTestGenerationToolkitEvent( + session, + false, + 'Cancelled', + messageId, + performance.now() - session.testGenerationStartTime, + getTelemetryReasonDesc(CodeWhispererConstants.unitTestGenerationCancelMessage) + ) this.dispatcher.sendUpdatePromptProgress( new UpdatePromptProgressMessage(tabID, cancellingProgressField) ) await sleep(500) } else { - telemetry.amazonq_utgGenerateTests.emit({ - cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext', - hasUserPromptSupplied: session.hasUserPromptSupplied, - perfClientLatency: performance.now() - session.testGenerationStartTime, - result: 'Succeeded', - isSupportedLanguage: false, - credentialStartUrl: AuthUtil.instance.startUrl, - requestId: messageId, - }) + TelemetryHelper.instance.sendTestGenerationToolkitEvent( + session, + false, + 'Succeeded', + messageId, + performance.now() - session.testGenerationStartTime + ) + this.dispatcher.sendUpdatePromptProgress( new UpdatePromptProgressMessage(tabID, testGenCompletedField) ) diff --git a/packages/core/src/codewhisperer/util/telemetryHelper.ts b/packages/core/src/codewhisperer/util/telemetryHelper.ts index bdb63b45727..9518aa610fc 100644 --- a/packages/core/src/codewhisperer/util/telemetryHelper.ts +++ b/packages/core/src/codewhisperer/util/telemetryHelper.ts @@ -27,6 +27,7 @@ import { CodeWhispererSupplementalContext } from '../models/model' import { FeatureConfigProvider } from '../../shared/featureConfig' import { CodeScanRemediationsEventType } from '../client/codewhispereruserclient' import { CodeAnalysisScope as CodeAnalysisScopeClientSide } from '../models/constants' +import { Session } from '../../amazonqTest/chat/session/session' export class TelemetryHelper { // Some variables for client component latency @@ -57,6 +58,50 @@ export class TelemetryHelper { return (this.#instance ??= new this()) } + public sendTestGenerationToolkitEvent( + session: Session, + isSupportedLanguage: boolean, + result: 'Succeeded' | 'Failed' | 'Cancelled', + requestId?: string, + perfClientLatency?: number, + reasonDesc?: string, + isCodeBlockSelected?: boolean, + artifactsUploadDuration?: number, + buildPayloadBytes?: number, + buildZipFileBytes?: number, + acceptedCharactersCount?: number, + acceptedCount?: number, + acceptedLinesCount?: number, + generatedCharactersCount?: number, + generatedCount?: number, + generatedLinesCount?: number, + reason?: string + ) { + telemetry.amazonq_utgGenerateTests.emit({ + cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext', + hasUserPromptSupplied: session.hasUserPromptSupplied, + isSupportedLanguage: isSupportedLanguage, + result: result, + artifactsUploadDuration: artifactsUploadDuration, + buildPayloadBytes: buildPayloadBytes, + buildZipFileBytes: buildZipFileBytes, + credentialStartUrl: AuthUtil.instance.startUrl, + acceptedCharactersCount: acceptedCharactersCount, + acceptedCount: acceptedCount, + acceptedLinesCount: acceptedLinesCount, + generatedCharactersCount: generatedCharactersCount, + generatedCount: generatedCount, + generatedLinesCount: generatedLinesCount, + isCodeBlockSelected: isCodeBlockSelected, + jobGroup: session.testGenerationJobGroupName, + jobId: session.listOfTestGenerationJobId[0], + perfClientLatency: perfClientLatency, + requestId: requestId, + reasonDesc: reasonDesc, + reason: reason, + }) + } + public recordServiceInvocationTelemetry( requestId: string, sessionId: string, diff --git a/packages/core/src/shared/telemetry/vscodeTelemetry.json b/packages/core/src/shared/telemetry/vscodeTelemetry.json index 48d5ab88f4c..5e32b249d4f 100644 --- a/packages/core/src/shared/telemetry/vscodeTelemetry.json +++ b/packages/core/src/shared/telemetry/vscodeTelemetry.json @@ -1,20 +1,5 @@ { "types": [ - { - "name": "acceptedCharactersCount", - "type": "int", - "description": "The number of accepted characters" - }, - { - "name": "acceptedCount", - "type": "int", - "description": "The number of accepted cases" - }, - { - "name": "acceptedLinesCount", - "type": "int", - "description": "The number of accepted lines of code" - }, { "name": "amazonGenerateApproachLatency", "type": "double", @@ -296,16 +281,6 @@ "type": "string", "description": "An AWS region." }, - { - "name": "buildPayloadBytes", - "type": "int", - "description": "The uncompressed payload size in bytes of the source files in customer project context" - }, - { - "name": "buildZipFileBytes", - "type": "int", - "description": "The compressed payload size of source files in bytes of customer project context sent" - }, { "name": "connectionState", "type": "string", @@ -380,46 +355,6 @@ "name": "executedCount", "type": "int", "description": "The number of executed operations" - }, - { - "name": "generatedCharactersCount", - "type": "int", - "description": "Number of characters of code generated" - }, - { - "name": "generatedCount", - "type": "int", - "description": "The number of generated cases" - }, - { - "name": "generatedLinesCount", - "type": "int", - "description": "The number of generated lines of code" - }, - { - "name": "hasUserPromptSupplied", - "type": "boolean", - "description": "True if user supplied prompt message as input else false" - }, - { - "name": "isCodeBlockSelected", - "type": "boolean", - "description": "True if user selected code snippet as input else false" - }, - { - "name": "isSupportedLanguage", - "type": "boolean", - "description": "Indicate if the language is supported" - }, - { - "name": "jobGroup", - "type": "string", - "description": "Job group name used in the operation" - }, - { - "name": "jobId", - "type": "string", - "description": "Job id used in the operation" } ], "metrics": [ @@ -1138,77 +1073,6 @@ } ] }, - { - "name": "amazonq_utgGenerateTests", - "description": "Client side invocation of the AmazonQ Unit Test Generation", - "metadata": [ - { - "type": "acceptedCharactersCount", - "required": false - }, - { - "type": "acceptedCount", - "required": false - }, - { - "type": "acceptedLinesCount", - "required": false - }, - { - "type": "artifactsUploadDuration", - "required": false - }, - { - "type": "buildPayloadBytes", - "required": false - }, - { - "type": "buildZipFileBytes", - "required": false - }, - { - "type": "credentialStartUrl", - "required": false - }, - { - "type": "cwsprChatProgrammingLanguage" - }, - { - "type": "generatedCharactersCount", - "required": false - }, - { - "type": "generatedCount", - "required": false - }, - { - "type": "generatedLinesCount", - "required": false - }, - { - "type": "hasUserPromptSupplied" - }, - { - "type": "isCodeBlockSelected", - "required": false - }, - { - "type": "isSupportedLanguage" - }, - { - "type": "jobGroup", - "required": false - }, - { - "type": "jobId", - "required": false - }, - { - "type": "perfClientLatency", - "required": false - } - ] - }, { "name": "ide_editCodeFile", "description": "User opened a code file with the given file extension. Client should DEDUPLICATE this metric (ideally hourly/daily). AWS-specific files should (also) emit `file_editAwsFile`.",