From 2b44e52151d002c3c51b0e0ad3fadf880ab05e1b Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 9 Jan 2025 11:16:38 -0500 Subject: [PATCH] Evaluate payload body in prologues and epilogues. (#772) Signed-off-by: dblock --- CHANGELOG.md | 1 + tools/src/tester/ChapterEvaluator.ts | 22 ++------- tools/src/tester/ResponsePayloadEvaluator.ts | 39 +++++++++++++++ .../tester/SupplementalChapterEvaluator.ts | 30 ++++++------ tools/src/tester/types/story.types.ts | 38 ++++++++------- .../tester/ResponsePayloadEvaluator.test.ts | 39 +++++++++++++++ .../SupplementalChapterEvaluator.test.ts | 48 +++++++++++++++++++ 7 files changed, 167 insertions(+), 50 deletions(-) create mode 100644 tools/src/tester/ResponsePayloadEvaluator.ts create mode 100644 tools/tests/tester/ResponsePayloadEvaluator.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 67fc0dc14..d9172f985 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added the ability to skip an individual chapter test ([#765](https://github.com/opensearch-project/opensearch-api-specification/pull/765)) - Added uploading of test spec logs ([#767](https://github.com/opensearch-project/opensearch-api-specification/pull/767)) - Added `POST /_plugins/_ml/memory`, `POST /_plugins/_ml/memory/_search`, `{memory_id}/_search`, `{memory_id}/messages`, `PUT /_plugins/_ml/memory/{memory_id}`, `message/{message_id}`, `GET /_plugins/_ml/memory`, `GET /_plugins/_ml/memory/{memory_id}`, `_search`, `message/{message_id}`, `{memory_id}/messages`, `{memory_id}/_search`, `message/{message_id}/traces`, and `DELETE /_plugins/_ml/memory/{memory_id}` ([#771](https://github.com/opensearch-project/opensearch-api-specification/pull/771)) +- Added support for evaluating response payloads in prologues and epilogues ([#772](https://github.com/opensearch-project/opensearch-api-specification/pull/772)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index 73ce1045e..e31047581 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -16,12 +16,12 @@ import type OperationLocator from './OperationLocator' import type SchemaValidator from './SchemaValidator' import { type StoryOutputs } from './StoryOutputs' import { ChapterOutput } from './ChapterOutput' -import { Operation, atomizeChangeset, diff } from 'json-diff-ts' import _ from 'lodash' import { Logger } from 'Logger' import { sleep, to_json } from '../helpers' -import { APPLICATION_JSON } from "./MimeTypes"; +import { APPLICATION_JSON } from "./MimeTypes" import { ParsedChapter } from './types/parsed_story.types' +import ResponsePayloadEvaluator from './ResponsePayloadEvaluator' export default class ChapterEvaluator { private readonly logger: Logger @@ -70,7 +70,7 @@ export default class ChapterEvaluator { const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(chapter, response, operation) : { result: Result.SKIPPED } const output_values_evaluation: EvaluationWithOutput = status.result === Result.PASSED ? ChapterOutput.extract_output_values(response, chapter.output) : { evaluation: { result: Result.SKIPPED } } const response_payload: Payload | undefined = status.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload - const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, response_payload) : { result: Result.SKIPPED } + const payload_body_evaluation = status.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload) : { result: Result.SKIPPED } if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`) @@ -151,22 +151,6 @@ export default class ChapterEvaluator { return result } - #evaluate_payload_body(response: ActualResponse, expected_payload?: Payload): Evaluation { - if (expected_payload == null) return { result: Result.PASSED } - const payload = response.payload - this.logger.info(`${to_json(payload)}`) - const delta = atomizeChangeset(diff(expected_payload, payload)) - const messages: string[] = _.compact(delta.map((value, _index, _array) => { - switch (value.type) { - case Operation.UPDATE: - return `expected ${value.path.replace('$.', '')}='${value.oldValue}', got '${value.value}'` - case Operation.REMOVE: - return `missing ${value.path.replace('$.', '')}='${value.value}'` - } - })) - return messages.length > 0 ? { result: Result.FAILED, message: _.join(messages, ', ') } : { result: Result.PASSED } - } - #evaluate_payload_schema(chapter: ParsedChapter, response: ActualResponse, operation: ParsedOperation): Evaluation { const content_type = chapter.response?.content_type ?? APPLICATION_JSON diff --git a/tools/src/tester/ResponsePayloadEvaluator.ts b/tools/src/tester/ResponsePayloadEvaluator.ts new file mode 100644 index 000000000..a3ca8ad7c --- /dev/null +++ b/tools/src/tester/ResponsePayloadEvaluator.ts @@ -0,0 +1,39 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import _ from "lodash" +import { Evaluation, Result } from './types/eval.types' +import { Logger } from "../Logger" +import { to_json } from "../helpers" +import { ActualResponse, Payload } from "./types/story.types" +import { atomizeChangeset, diff, Operation } from "json-diff-ts" + +export default class ResponsePayloadEvaluator { + private readonly logger: Logger + + constructor(logger: Logger) { + this.logger = logger + } + + evaluate(response: ActualResponse, expected_payload?: Payload): Evaluation { + if (expected_payload == null) return { result: Result.PASSED } + const payload = response.payload + this.logger.info(`${to_json(payload)}`) + const delta = atomizeChangeset(diff(expected_payload, payload)) + const messages: string[] = _.compact(delta.map((value, _index, _array) => { + switch (value.type) { + case Operation.UPDATE: + return `expected ${value.path.replace('$.', '')}='${value.oldValue}', got '${value.value}'` + case Operation.REMOVE: + return `missing ${value.path.replace('$.', '')}='${value.value}'` + } + })) + return messages.length > 0 ? { result: Result.FAILED, message: _.join(messages, ', ') } : { result: Result.PASSED } + } +} \ No newline at end of file diff --git a/tools/src/tester/SupplementalChapterEvaluator.ts b/tools/src/tester/SupplementalChapterEvaluator.ts index 9abfe19e4..fd8d4ce2b 100644 --- a/tools/src/tester/SupplementalChapterEvaluator.ts +++ b/tools/src/tester/SupplementalChapterEvaluator.ts @@ -7,22 +7,23 @@ * compatible open source license. */ -import _ from "lodash"; -import { ChapterOutput } from "./ChapterOutput"; -import ChapterReader from "./ChapterReader"; -import { StoryOutputs } from "./StoryOutputs"; -import { overall_result } from "./helpers"; -import { ChapterEvaluation, EvaluationWithOutput, Result } from './types/eval.types'; -import { Logger } from "../Logger"; -import { sleep, to_json } from "../helpers"; -import { SupplementalChapter } from "./types/story.types"; +import _ from "lodash" +import { ChapterOutput } from "./ChapterOutput" +import ChapterReader from "./ChapterReader" +import { StoryOutputs } from "./StoryOutputs" +import { overall_result } from "./helpers" +import { ChapterEvaluation, EvaluationWithOutput, Result } from './types/eval.types' +import { Logger } from "../Logger" +import { sleep, to_json } from "../helpers" +import { Payload, SupplementalChapter } from "./types/story.types" +import ResponsePayloadEvaluator from './ResponsePayloadEvaluator' export default class SupplementalChapterEvaluator { - private readonly _chapter_reader: ChapterReader; - private readonly logger: Logger; + private readonly _chapter_reader: ChapterReader + private readonly logger: Logger constructor(chapter_reader: ChapterReader, logger: Logger) { - this._chapter_reader = chapter_reader; + this._chapter_reader = chapter_reader this.logger = logger } @@ -49,10 +50,11 @@ export default class SupplementalChapterEvaluator { const response = await this._chapter_reader.read(chapter, story_outputs) const output_values_evaluation = ChapterOutput.extract_output_values(response, chapter.output) if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`) - const status = chapter.status ?? [200, 201] const overall = status.includes(response.status) ? { result: Result.PASSED } : { result: Result.ERROR, message: response.message, error: response.error as Error } - const result: Result = overall_result(_.compact([overall, output_values_evaluation.evaluation])) + const response_payload: Payload | undefined = overall.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload + const payload_body_evaluation = overall.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload) : { result: Result.SKIPPED } + const result: Result = overall_result(_.compact([overall, payload_body_evaluation, output_values_evaluation.evaluation])) var evaluation_result: EvaluationWithOutput = { evaluation: { result } } if (output_values_evaluation.output) { evaluation_result.output = output_values_evaluation.output } diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index 23b6d6a0f..2e48e38af 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -63,23 +63,6 @@ export type Version = string; * via the `definition` "DistributionsList". */ export type DistributionsList = string[]; -/** - * Number of times to retry on error. - * - * - * This interface was referenced by `Story`'s JSON-Schema - * via the `definition` "Retry". - */ -export type Retry = { - /** - * Number of retries. - */ - count: number; - /** - * Number of milliseconds to wait before retrying. - */ - wait?: number; -}; /** * This interface was referenced by `Story`'s JSON-Schema * via the `definition` "Chapter". @@ -90,6 +73,9 @@ export type Chapter = ChapterRequest & { * A brief description of the chapter. */ synopsis: string; + /** + * An explanation is provided to clarify why it has been skipped. + */ pending?: string; response?: ExpectedResponse; warnings?: Warnings; @@ -131,6 +117,7 @@ export interface ChapterRequest { version?: Version; distributions?: Distributions; retry?: Retry; + response?: ExpectedResponse; } /** * This interface was referenced by `Story`'s JSON-Schema @@ -177,6 +164,23 @@ export interface Distributions { included?: DistributionsList; excluded?: DistributionsList; } +/** + * Number of times to retry on error. + * + * + * This interface was referenced by `Story`'s JSON-Schema + * via the `definition` "Retry". + */ +export interface Retry { + /** + * Number of retries. + */ + count: number; + /** + * Number of milliseconds to wait before retrying. + */ + wait?: number; +} /** * This interface was referenced by `Story`'s JSON-Schema * via the `definition` "ExpectedResponse". diff --git a/tools/tests/tester/ResponsePayloadEvaluator.test.ts b/tools/tests/tester/ResponsePayloadEvaluator.test.ts new file mode 100644 index 000000000..d8d904337 --- /dev/null +++ b/tools/tests/tester/ResponsePayloadEvaluator.test.ts @@ -0,0 +1,39 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import { Result } from "tester/types/eval.types"; +import ResponsePayloadEvaluator from "tester/ResponsePayloadEvaluator"; +import { Logger } from "Logger"; +import { ActualResponse } from "tester/types/story.types"; + +function create_response(payload: any): ActualResponse { + return { + status: 200, + content_type: 'application/json', + payload + } +} + +describe('ResponsePayloadEvaluator', () => { + const evaluator = new ResponsePayloadEvaluator(new Logger()) + + describe('evaluate', () => { + test('succeeds without an expected payload', () => { + expect(evaluator.evaluate(create_response({}), undefined)).toEqual({ result: Result.PASSED }) + }) + + test('fails with a non-matching payload', () => { + expect(evaluator.evaluate(create_response({}), { x: 1 })).toEqual({ result: Result.FAILED, message: "missing x='1'" }) + }) + + test('succeeds with a matching payload', () => { + expect(evaluator.evaluate(create_response({ x: 1 }), { x: 1 })).toEqual({ result: Result.PASSED }) + }) + }) +}) diff --git a/tools/tests/tester/SupplementalChapterEvaluator.test.ts b/tools/tests/tester/SupplementalChapterEvaluator.test.ts index 56bb01eab..de9798b8d 100644 --- a/tools/tests/tester/SupplementalChapterEvaluator.test.ts +++ b/tools/tests/tester/SupplementalChapterEvaluator.test.ts @@ -86,5 +86,53 @@ describe('SupplementalChapterEvaluator', () => { expect(result.overall.result).toEqual(Result.ERROR) expect(count).toEqual(5) }) + + test('a valid response payload', async () => { + mock.onAny().reply(200, '{"acknowledged":true}', { "content-type": "application/json" }) + + expect( + await supplemental_chapter_evaluator.evaluate({ + path: '/test', + method: 'PUT', + request: { + payload: {} + }, + response: { + status: 200, + payload: { + acknowledged: true + } + } + }, story_outputs)).toEqual({ + title: 'PUT /test', + overall: { + result: Result.PASSED + } + }) + }) + + test('an invalid response payload', async () => { + mock.onAny().reply(200, '{"acknowledged":false}', { "content-type": "application/json" }) + + expect( + await supplemental_chapter_evaluator.evaluate({ + path: '/test', + method: 'PUT', + request: { + payload: {} + }, + response: { + status: 200, + payload: { + acknowledged: true + } + } + }, story_outputs)).toEqual({ + title: 'PUT /test', + overall: { + result: Result.FAILED + } + }) + }) }) })