From 9eef52ccf9edd2fa61bcb9b57dde85b1e8c09378 Mon Sep 17 00:00:00 2001 From: dblock Date: Wed, 19 Jun 2024 16:01:45 -0400 Subject: [PATCH] WIP: Check response payload. Signed-off-by: dblock --- package-lock.json | 10 ++++++++ package.json | 1 + tests/_core/info.yaml | 5 ++++ tools/src/tester/ChapterEvaluator.ts | 37 ++++++++++++++++++++++++---- tools/src/tester/ResultLogger.ts | 12 ++++++--- tools/src/tester/types/eval.types.ts | 3 ++- 6 files changed, 59 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index f0ca22c86..efb49a279 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,6 +39,7 @@ "eslint-plugin-yml": "^1.14.0", "globals": "^15.0.0", "jest": "^29.7.0", + "json-diff-ts": "^4.0.1", "json-schema-to-typescript": "^14.0.4", "ts-jest": "^29.1.2" } @@ -6109,6 +6110,15 @@ "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", "dev": true }, + "node_modules/json-diff-ts": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/json-diff-ts/-/json-diff-ts-4.0.1.tgz", + "integrity": "sha512-FEuq+gv4DXI+nL7oF/trBQIBQYDVcJe5Kqe3mYUZAkDtHBY/cdWmVZpV9BLZFp+wThMClajYp0fmNVmB81FsmA==", + "dev": true, + "dependencies": { + "lodash": "4.x" + } + }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", diff --git a/package.json b/package.json index b0d5f6ae4..ddcd4a510 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "eslint-plugin-yml": "^1.14.0", "globals": "^15.0.0", "jest": "^29.7.0", + "json-diff-ts": "^4.0.1", "json-schema-to-typescript": "^14.0.4", "ts-jest": "^29.1.2" } diff --git a/tests/_core/info.yaml b/tests/_core/info.yaml index 5a8e14f07..bfde4c6e2 100644 --- a/tests/_core/info.yaml +++ b/tests/_core/info.yaml @@ -16,3 +16,8 @@ chapters: pretty: false response: status: 200 + payload: + version: + distribution: opensearchX + tagline: 'The OpenSearch Project: https://opensearch.org/.' + x: y \ No newline at end of file diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index fefa0202f..2737a0cf6 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -7,7 +7,7 @@ * compatible open source license. */ -import { type Chapter, type ActualResponse } from './types/story.types' +import { type Chapter, type ActualResponse, type Payload } from './types/story.types' import { type ChapterEvaluation, type Evaluation, Result } from './types/eval.types' import { type ParsedOperation } from './types/spec.types' import { overall_result } from './helpers' @@ -16,6 +16,9 @@ import type OperationLocator from './OperationLocator' import type SchemaValidator from './SchemaValidator' import { type StoryOutputs } from './StoryOutputs' import { ChapterOutput } from './ChapterOutput' +import { to_json } from '../helpers' +import { Operation, atomizeChangeset, diff } from 'json-diff-ts' +import _ from 'lodash' export default class ChapterEvaluator { private readonly _operation_locator: OperationLocator @@ -36,13 +39,20 @@ export default class ChapterEvaluator { const params = this.#evaluate_parameters(chapter, operation) const request_body = this.#evaluate_request_body(chapter, operation) const status = this.#evaluate_status(chapter, response) - const payload = status.result === Result.PASSED ? this.#evaluate_payload(response, operation) : { result: Result.SKIPPED } + const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, chapter.response?.payload) : { result: Result.SKIPPED } + const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(response, operation) : { result: Result.SKIPPED } const output_values = ChapterOutput.extract_output_values(response, chapter.output) return { title: chapter.synopsis, - overall: { result: overall_result(Object.values(params).concat([request_body, status, payload]).concat(output_values ? [output_values] : [])) }, + overall: { result: overall_result(Object.values(params).concat([ + request_body, status, payload_body_evaluation, payload_schema_evaluation + ]).concat(output_values ? [output_values] : [])) }, request: { parameters: params, request_body }, - response: { status, payload }, + response: { + status: status, + payload_body: payload_body_evaluation, + payload_schema: payload_schema_evaluation + }, ...(output_values ? { output_values } : {}) } } @@ -74,7 +84,24 @@ export default class ChapterEvaluator { } } - #evaluate_payload(response: ActualResponse, operation: ParsedOperation): Evaluation { + #evaluate_payload_body(response: ActualResponse, expected_payload?: Payload): Evaluation { + if (expected_payload == null) return { result: Result.PASSED } + console.log(`${to_json(response.payload)} vs. ${to_json(expected_payload)}`) + const delta = atomizeChangeset(diff(expected_payload, response.payload)) + var messages: string[] = [] + delta.forEach((value, index, array) => { + console.log(value) + switch(value.type) { + case Operation.UPDATE: + return messages.push(`expected ${value.path.replace('$.', '')}='${value.oldValue}', got '${value.value}'`) + case Operation.REMOVE: + return messages.push(`missing ${value.path.replace('$.', '')}=${value.value}`) + } + }) + return messages.length > 0 ? { result: Result.FAILED, message: _.join(messages, ', ')} : { result: Result.PASSED } + } + + #evaluate_payload_schema(response: ActualResponse, operation: ParsedOperation): Evaluation { const content_type = response.content_type ?? 'application/json' const content = operation.responses[response.status]?.content[content_type] const schema = content?.schema diff --git a/tools/src/tester/ResultLogger.ts b/tools/src/tester/ResultLogger.ts index 1707eab58..2a1aef7fe 100644 --- a/tools/src/tester/ResultLogger.ts +++ b/tools/src/tester/ResultLogger.ts @@ -53,7 +53,8 @@ export class ConsoleResultLogger implements ResultLogger { this.#log_parameters(chapter.request?.parameters ?? {}) this.#log_request_body(chapter.request?.request_body) this.#log_status(chapter.response?.status) - this.#log_payload(chapter.response?.payload) + this.#log_payload_body(chapter.response?.payload_body) + this.#log_payload_schema(chapter.response?.payload_schema) } #log_parameters (parameters: Record): void { @@ -75,9 +76,14 @@ export class ConsoleResultLogger implements ResultLogger { this.#log_evaluation(evaluation, 'RESPONSE STATUS', this._tab_width * 3) } - #log_payload (evaluation: Evaluation | undefined): void { + #log_payload_body (evaluation: Evaluation | undefined): void { if (evaluation == null) return - this.#log_evaluation(evaluation, 'RESPONSE PAYLOAD', this._tab_width * 3) + this.#log_evaluation(evaluation, 'RESPONSE PAYLOAD BODY', this._tab_width * 3) + } + + #log_payload_schema (evaluation: Evaluation | undefined): void { + if (evaluation == null) return + this.#log_evaluation(evaluation, 'RESPONSE PAYLOAD SCHEMA', this._tab_width * 3) } #log_evaluation (evaluation: Evaluation, title: string, prefix: number = 0): void { diff --git a/tools/src/tester/types/eval.types.ts b/tools/src/tester/types/eval.types.ts index e4c4fb160..d42e39912 100644 --- a/tools/src/tester/types/eval.types.ts +++ b/tools/src/tester/types/eval.types.ts @@ -32,7 +32,8 @@ export interface ChapterEvaluation { } response?: { status: Evaluation - payload: Evaluation + payload_body: Evaluation, + payload_schema: Evaluation } output_values?: EvaluationWithOutput }