From 086d3f7b07d351d5e634728cd33dc70fed346d3d Mon Sep 17 00:00:00 2001 From: Panagiota Mitsopoulou Date: Wed, 22 Jan 2025 16:19:25 +0200 Subject: [PATCH] temp --- .../evaluate/scenarios/rca/checkout.spec.ts | 151 + .../data.json.gz | Bin 0 -> 1681 bytes .../mappings.json | 50776 ++++++++++++++++ 3 files changed, 50927 insertions(+) create mode 100644 x-pack/solutions/observability/plugins/investigate_app/scripts/evaluate/scenarios/rca/checkout.spec.ts create mode 100644 x-pack/solutions/observability/plugins/investigate_app/scripts/load/fixtures/custom_threshold_alerts_checkout/data.json.gz create mode 100644 x-pack/solutions/observability/plugins/investigate_app/scripts/load/fixtures/custom_threshold_alerts_checkout/mappings.json diff --git a/x-pack/solutions/observability/plugins/investigate_app/scripts/evaluate/scenarios/rca/checkout.spec.ts b/x-pack/solutions/observability/plugins/investigate_app/scripts/evaluate/scenarios/rca/checkout.spec.ts new file mode 100644 index 0000000000000..e5523f1bb8bb5 --- /dev/null +++ b/x-pack/solutions/observability/plugins/investigate_app/scripts/evaluate/scenarios/rca/checkout.spec.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/// + +import type { + RootCauseAnalysisEvent, + EndProcessToolMessage, + InvestigateEntityToolMessage, + ObservationToolMessage, + ToolErrorMessage, +} from '@kbn/observability-ai-server/root_cause_analysis'; +import { + chatClient, + kibanaClient, + logger, +} from '@kbn/observability-ai-assistant-app-plugin/scripts/evaluation/services'; +import { RCAClient } from '../../rca_client'; + +type ToolCallMessage = + | EndProcessToolMessage + | InvestigateEntityToolMessage + | ObservationToolMessage + | ToolErrorMessage; + +const ALERT_FIXTURE_ID = '88591c00-cce5-4495-9a1b-cbe96db8ed86'; + +describe('Root cause analysis - checkout alert', () => { + const investigations: string[] = []; + const rcaChatClient = new RCAClient(kibanaClient, logger); + function countEntities(entities: InvestigateEntityToolMessage[]) { + const entityCount: Record = {}; + entities.forEach((entity) => { + const name = entity.response.entity['service.name']; + entityCount[name] = (entityCount[name] || 0) + 1; + }); + return entityCount; + } + + // TODO move it in a common file later on + function categorizeEvents(events: RootCauseAnalysisEvent[]) { + const report: EndProcessToolMessage[] = []; + const observations: ObservationToolMessage[] = []; + const errors: ToolErrorMessage[] = []; + const entities: InvestigateEntityToolMessage[] = []; + const other: RootCauseAnalysisEvent[] = []; + const toolCallEvents = events.filter((event): event is ToolCallMessage => { + const maybeToolEvent = event as EndProcessToolMessage; + return ( + maybeToolEvent?.name === 'endProcessAndWriteReport' || + maybeToolEvent?.name === 'observe' || + maybeToolEvent?.name === 'error' || + maybeToolEvent?.name === 'investigateEntity' + ); + }); + toolCallEvents.forEach((event) => { + if (event.name) { + switch (event.name) { + case 'endProcessAndWriteReport': + report.push(event as EndProcessToolMessage); + break; + case 'observe': + observations.push(event as ObservationToolMessage); + break; + case 'error': + errors.push(event as ToolErrorMessage); + break; + case 'investigateEntity': + entities.push(event as InvestigateEntityToolMessage); + break; + default: + other.push(event); + } + } + }); + if (report.length > 1) { + throw new Error('More than one final report found'); + } + if (report.length === 0) { + throw new Error('No final report found'); + } + return { report: report[0], observations, errors, entities, other }; + } + + it('can accurately pinpoint the root cause of cartservice bad entrypoint failure - checkout alert', async () => { + const alert = await rcaChatClient.getAlert(ALERT_FIXTURE_ID); + const connectorId = chatClient.getConnectorId(); + const { from, to } = await rcaChatClient.getTimeRange({ + fromOffset: 'now-15m', // time the alert was triggered (now) + toOffset: 'now+15m', + alert, + }); + const investigationId = await rcaChatClient.createInvestigation({ + alertId: ALERT_FIXTURE_ID, + from, + to, + }); + investigations.push(investigationId); + const events = await rcaChatClient.rootCauseAnalysis({ + investigationId, + from: new Date(from).toISOString(), + to: new Date(to).toISOString(), + alert, + connectorId, + }); + const { report, entities, errors } = categorizeEvents(events); + const prompt = ` + An investigation was performed by the Observability AI Assistant to identify the root cause of an alert for the controller service. Here is the alert: + + ${JSON.stringify(alert)} + + The following entities were analyzed during the investigation. + ${Object.entries(countEntities(entities)) + .map(([name, count]) => { + return ` - ${name} (analyzed ${count} times)`; + }) + .join('\n')} + + During the course of the investigation, the Observability AI Assistant encountered ${ + errors.length + } errors when attempting to analyze the entities.${ + errors.length + ? ' These errors were failures to retrieve data from the entities and do not reflect issues in the system being evaluated' + : '' + }. + + A report was written by the Observability AI Assistant detailing issues throughout the system, including the controller service and it's dependencies. The report includes a hypothesis about the underlying root cause of the system failure. Here is the report: + + ${report.response.report} + `; + + const conversation = await chatClient.complete({ messages: prompt }); + + await chatClient.evaluate(conversation, [ + 'Effectively reflects the actual root cause in the report. The actual root cause of the system failure was a misconfiguration related to the `cartservice`. A bad container entrypoint was configured for the cart service, causing it to fail to start', + 'Analyzes the cartservice during the course of the investigation.', + 'Analyzes each entity only once.', + 'The Observability AI Assistant encountered 0 errors when attempting to analyze the system failure.', + ]); + }); + + after(async () => { + for (const investigationId of investigations) { + await rcaChatClient.deleteInvestigation({ investigationId }); + } + }); +}); diff --git a/x-pack/solutions/observability/plugins/investigate_app/scripts/load/fixtures/custom_threshold_alerts_checkout/data.json.gz b/x-pack/solutions/observability/plugins/investigate_app/scripts/load/fixtures/custom_threshold_alerts_checkout/data.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..b87fc7520f409a01854ef16d4467d6ba828bf63e GIT binary patch literal 1681 zcmV;C25$KuiwFP!000026YZN@Z`(K)fZy{g2tBn$sFJ!%zz(*T#bAIz_hDy&#av)e z00AuXoWn!vT=<2^{S`uXasz;O1^9i=!Vrk zt3jI<`_UYgMQD-XVm%zZglv;9&ksX?4j=BjCdcXsmGk{P~$)+Ke57IR0?`gxyZp z{)(B!37fCkO1|byn*)>63zAK1Si`~Jl%Ty+*Lz#znoDLYPh*v5x3UrHb|%GPM4y;( zrC}{q*UtX0G}pk|O`(YiLI2QN=@!Y>CebxwrCI|r3;Vk@hE#>9k6!E&zCaW$Wpj-L zGYqXI+!ja~sAsjBx3m7s7jHVFmdt=aP{OQAgP)x|=QdUf}B^c1{L0>}1>qY+A{Al08d)czp3Yb2w ziF>&Z>OC`TWSnbhKub1A=V9VmDYdkY7}an84}-+BZSBuZLAQf1-oAeG61}o9cq~@4 z66#^J9*Zo&Xvy0nZ$9;Cv$pyCm+hH?tFAFe#i{njwkwKJ{+ z6tBh&Dx=BM&Be#bdP}HP-^ueDO{-m7H&@+8|L5Pl?0QHk{?p4*HsO ze;l&Sykqlzvu}Elqt);9yqm1;oo`ocyNm+;?#X7Zp>1(9m(9oDJ@n?od$e0nf%=O1 zYJAUjpB|=)gI1#Bn;;sTc&93g?L6&fo9L8SSHsL6*=L)wH#||>(bhk1R#Vd&HXo+7 zCt#_(!%|dB%r)3v<4t$Z-r%kt=Om7UfN-2uCB;#dSGa_fW5U7^%AjNwk54$(8r#{A zIzv#zK|!gXo$d^q6dgO@55#O{%rtA}Fs$8k1Cx_Zzxmy|F94-u6Sqp=RJFRD?H8g} z(0n4@$MVsErhjogJ)L|>YtY*O^Kzf~`wBM9^5OCCQRc4&%GEAs97t>Z5Z zC=J5P=yz=K-x>Xm+(sW`r4t<|5`aW6kPM~kds)|4rGdjYdK^qsi*O@M|KCz$4( z;wVo-oF!q3D@Lj?BWai=!B>I4B(t$ZqHq4@H&3D<3@$_8vBf*|byvBLXC2ReJ3RYj z6+6MR8HX%MnZS_897l`;F3X7HIE}I_37LRAa6IdH*72<4S;w>AJ