From ace2bb40ca77f20cb3ea6b2874b180dcc444b07e Mon Sep 17 00:00:00 2001 From: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> Date: Mon, 6 Jan 2025 17:46:30 +0500 Subject: [PATCH] Add support for pending tests (#765) * ignored parameter Signed-off-by: Tokesh * chore: fix lint Signed-off-by: Tokesh * chore: lint fix and operation structure Signed-off-by: Tokesh * chore: deleting not important files Signed-off-by: Tokesh * adding tests Signed-off-by: Tokesh * refactor: added pending statement to main if statement Signed-off-by: Tokesh * chore: renaming ignored test files to make standart Signed-off-by: Tokesh --------- Signed-off-by: Tokesh Signed-off-by: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> --- CHANGELOG.md | 1 + TESTING_GUIDE.md | 5 +++++ json_schemas/test_story.schema.yaml | 3 +++ tools/src/tester/StoryEvaluator.ts | 4 ++++ tools/src/tester/types/eval.types.ts | 2 ++ tools/src/tester/types/story.types.ts | 1 + .../tester/fixtures/evals/ignored/ignored.yaml | 15 +++++++++++++++ .../tester/fixtures/stories/ignored/ignored.yaml | 14 ++++++++++++++ tools/tests/tester/integ/StoryEvaluator.test.ts | 6 ++++++ tools/tests/tester/integ/TestRunner.test.ts | 1 + 10 files changed, 52 insertions(+) create mode 100644 tools/tests/tester/fixtures/evals/ignored/ignored.yaml create mode 100644 tools/tests/tester/fixtures/stories/ignored/ignored.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 42c3ff268..cd9b356bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `POST /_plugins/_ml/_train/{algorithm_name}`, `_predict/{algorithm_name}/{model_id}`, and `_train_predict/{algorithm_name}` ([#755](https://github.com/opensearch-project/opensearch-api-specification/pull/755)) - Added `PUT /_plugins/_ml/model_groups/{model_group_id}`, `GET /_plugins/_ml/model_groups/_search`, and `POST /_plugins/_ml/model_groups/_search` ([#760](https://github.com/opensearch-project/opensearch-api-specification/pull/760)) - Added `GET /_plugins/_ml/connectors/{connector_id}`, `_search`, `POST /_plugins/_ml/connectors/_search`, and `PUT /_plugins/_ml/connectors/{connector_id}` ([#764](https://github.com/opensearch-project/opensearch-api-specification/pull/764)) +- Added the ability to skip an individual chapter test ([#765](https://github.com/opensearch-project/opensearch-api-specification/pull/765)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index c4590bd0a..df2a2526f 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -193,6 +193,11 @@ chapters: - synopsis: Delete the `books` index. path: /{index} method: DELETE + pending: | # This test is included but marked as pending, meaning it will not be executed. An explanation is provided to clarify why it has been skipped. + SSL needs to be configured to trust itself. + For this to work, will need to: + 1. set reindex.remote.allowlist=localhost:9200 in docker-compose.yml + 2. setup SSL in a way that the docker instance trusts its own cert parameters: index: books ``` diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index 6792caad8..333f24987 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -44,6 +44,9 @@ definitions: type: string description: A brief description of the chapter. pattern: ^\p{Lu}[\s\S]*\.$|^\p{Lu}[\s\S]*\. \[(GET|PUT|POST|DELETE|PATCH|HEAD|OPTIONS)\]$ + pending: + type: string + description: An explanation is provided to clarify why it has been skipped. response: $ref: '#/definitions/ExpectedResponse' warnings: diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index eb0bbf2f7..a032f9369 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -111,6 +111,7 @@ export default class StoryEvaluator { const evaluations: ChapterEvaluation[] = [] for (const chapter of chapters) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` + if (dry_run) { evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run' } }) } else if (distribution != undefined && chapter.distributions?.included !== undefined && chapter.distributions?.included.length > 0 && !chapter.distributions.included.includes(distribution)) { @@ -119,6 +120,9 @@ export default class StoryEvaluator { evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is ${chapter.distributions.excluded.length > 1 ? 'one of ' : ''}${chapter.distributions.excluded.join(', ')}.` } }) } else if (version != undefined && chapter.version !== undefined && !semver.satisfies(version, chapter.version)) { evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.` } }) + } else if (chapter.pending != null && chapter.pending) { + evaluations.push({ title, overall: { result: Result.IGNORED, message: chapter.pending } }) + continue } else { const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs) has_errors = has_errors || evaluation.overall.result === Result.ERROR diff --git a/tools/src/tester/types/eval.types.ts b/tools/src/tester/types/eval.types.ts index 0686a6896..c778a1447 100644 --- a/tools/src/tester/types/eval.types.ts +++ b/tools/src/tester/types/eval.types.ts @@ -43,6 +43,7 @@ export interface ChapterEvaluation { overall: Evaluation, operation?: Operation, path?: string, + pending?: string, request?: { parameters?: Record request?: Evaluation @@ -79,6 +80,7 @@ export type EvaluationWithOutput = { export enum Result { PASSED = 'PASSED', + IGNORED = 'IGNORED', FAILED = 'FAILED', SKIPPED = 'SKIPPED', ERROR = 'ERROR', diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index 78fc5ab3e..23b6d6a0f 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -90,6 +90,7 @@ export type Chapter = ChapterRequest & { * A brief description of the chapter. */ synopsis: string; + pending?: string; response?: ExpectedResponse; warnings?: Warnings; }; diff --git a/tools/tests/tester/fixtures/evals/ignored/ignored.yaml b/tools/tests/tester/fixtures/evals/ignored/ignored.yaml new file mode 100644 index 000000000..6e1fe9838 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/ignored/ignored.yaml @@ -0,0 +1,15 @@ +display_path: ignored/ignored.yaml +full_path: tools/tests/tester/fixtures/stories/ignored/ignored.yaml + +result: PASSED +description: This story should ignored. +prologues: [] +chapters: + - title: This PUT /{index} chapter should pending. + overall: + result: IGNORED + message: This chapter is ignored because it is not relevant to the current test. +epilogues: + - title: DELETE /books + overall: + result: PASSED \ No newline at end of file diff --git a/tools/tests/tester/fixtures/stories/ignored/ignored.yaml b/tools/tests/tester/fixtures/stories/ignored/ignored.yaml new file mode 100644 index 000000000..ce6ad9219 --- /dev/null +++ b/tools/tests/tester/fixtures/stories/ignored/ignored.yaml @@ -0,0 +1,14 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: This story should ignored. +epilogues: + - path: /books + method: DELETE + status: [200, 404] +chapters: + - synopsis: This PUT /{index} chapter should pending. + path: /{index} + pending: This chapter is ignored because it is not relevant to the current test. + method: PUT + parameters: + index: books diff --git a/tools/tests/tester/integ/StoryEvaluator.test.ts b/tools/tests/tester/integ/StoryEvaluator.test.ts index cae1de746..7413420cf 100644 --- a/tools/tests/tester/integ/StoryEvaluator.test.ts +++ b/tools/tests/tester/integ/StoryEvaluator.test.ts @@ -69,6 +69,12 @@ test('skipped/semver', async () => { expect(actual).toEqual(expected) }) +test('ignored', async () => { + const actual = await load_actual_evaluation(story_evaluator, 'ignored/ignored') + const expected = load_expected_evaluation('ignored/ignored') + expect(actual).toEqual(expected) +}) + test('with an unexpected error deserializing data', async () => { opensearch_http_client.request = jest.fn().mockRejectedValue(new Error('This was unexpected.')) const actual = await load_actual_evaluation(story_evaluator, 'passed/passed') diff --git a/tools/tests/tester/integ/TestRunner.test.ts b/tools/tests/tester/integ/TestRunner.test.ts index 8e2bfb353..0d5a3abe2 100644 --- a/tools/tests/tester/integ/TestRunner.test.ts +++ b/tools/tests/tester/integ/TestRunner.test.ts @@ -34,6 +34,7 @@ test('stories folder', async () => { 'error/prologue_error', 'failed/invalid_data', 'failed/not_found', + 'ignored/ignored', 'passed/multiple_methods', 'passed/passed', 'passed/value_type',