From 7fd1eb427a2774c810f89d41b5e4c7f54623b831 Mon Sep 17 00:00:00 2001
From: dblock <dblock@amazon.com>
Date: Wed, 14 Aug 2024 12:14:41 -0400
Subject: [PATCH] Specify warnings: at story level.

Signed-off-by: dblock <dblock@amazon.com>
---
 TESTING_GUIDE.md                                  |  2 +-
 json_schemas/test_story.schema.yaml               |  2 ++
 tests/default/ingest/pipeline/neural_search.yaml  |  2 ++
 tools/src/tester/StoryEvaluator.ts                | 15 ++++++++-------
 tools/src/tester/types/story.types.ts             |  1 +
 .../fixtures/evals/error/prologue_error.yaml      |  5 -----
 .../fixtures/stories/error/prologue_error.yaml    |  2 ++
 7 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md
index d35c96aa5..e7e6c231e 100644
--- a/TESTING_GUIDE.md
+++ b/TESTING_GUIDE.md
@@ -299,7 +299,7 @@ WARNING Multiple paths detected, please group similar tests together and move pa
 
 #### Suppressing Warnings
 
-The test runner may generate warnings that can be suppressed with `warnings:`. For example, to suppress the multiple paths detected warning.
+The test runner may generate warnings that can be suppressed with `warnings:` at story or chapter level. For example, to suppress the multiple paths detected warning.
 
 ```yaml
 - synopsis: Create an index.
diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml
index 928ba4bad..62f4f747a 100644
--- a/json_schemas/test_story.schema.yaml
+++ b/json_schemas/test_story.schema.yaml
@@ -23,6 +23,8 @@ properties:
     $ref: '#/definitions/Version'
   distributions:
     $ref: '#/definitions/Distributions'
+  warnings:
+    $ref: '#/definitions/Warnings'
 required: [chapters,description]
 additionalProperties: false
 
diff --git a/tests/default/ingest/pipeline/neural_search.yaml b/tests/default/ingest/pipeline/neural_search.yaml
index 2d8b41ec1..1f0c6de0a 100644
--- a/tests/default/ingest/pipeline/neural_search.yaml
+++ b/tests/default/ingest/pipeline/neural_search.yaml
@@ -1,6 +1,8 @@
 $schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test the creation a neural search ingest pipeline.
+warnings:
+  multiple-paths-detected: false
 prologues:
   - path: /_cluster/settings
     method: PUT
diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts
index 3f396a3c6..3b2c3bf4c 100644
--- a/tools/src/tester/StoryEvaluator.ts
+++ b/tools/src/tester/StoryEvaluator.ts
@@ -63,21 +63,22 @@ export default class StoryEvaluator {
       prologues,
       epilogues,
       result: overall_result(prologues.concat(chapters).concat(epilogues).concat(prologues).map(e => e.overall)),
-      warnings: this.#chapter_warnings(story.chapters)
+      warnings: this.#chapter_warnings(story)
     }
   }
 
-  #chapter_warnings(chapters: Chapter[]): string[] | undefined {
+  #chapter_warnings(story: Story): string[] | undefined {
     const result = _.compact([
-      this.#warning_if_mismatched_chapter_paths(chapters)
+      this.#warning_if_mismatched_chapter_paths(story)
     ])
     return result.length > 0 ? result : undefined
   }
 
-  #warning_if_mismatched_chapter_paths(chapters: Chapter[]): string | undefined {
-    const paths = _.compact(_.map(chapters, (chapter) => {
-      const multiple_paths_detected = chapter.warnings?.['multiple-paths-detected'] ?? true
-      if (multiple_paths_detected) return chapter.path
+  #warning_if_mismatched_chapter_paths(story: Story): string | undefined {
+    if (story.warnings?.['multiple-paths-detected'] === false) return
+    const paths = _.compact(_.map(story.chapters, (chapter) => {
+      if (chapter.warnings?.['multiple-paths-detected'] === false) return
+      return chapter.path
     }))
     const normalized_paths = _.map(paths, (path) => path.replaceAll(/\/\{[^}]+}/g, '').replaceAll('//', '/'))
     const paths_counts: Record<string, number> = Object.assign((_.values(_.groupBy(normalized_paths)).map(p => { return { [p[0]] : p.length } })))
diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts
index 83d307b44..2ee8503e0 100644
--- a/tools/src/tester/types/story.types.ts
+++ b/tools/src/tester/types/story.types.ts
@@ -103,6 +103,7 @@ export interface Story {
   chapters: Chapter[];
   version?: Version;
   distributions?: Distributions;
+  warnings?: Warnings;
 }
 /**
  * This interface was referenced by `Story`'s JSON-Schema
diff --git a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml b/tools/tests/tester/fixtures/evals/error/prologue_error.yaml
index d91beb82f..c27415bf3 100644
--- a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml
+++ b/tools/tests/tester/fixtures/evals/error/prologue_error.yaml
@@ -3,11 +3,6 @@ full_path: tools/tests/tester/fixtures/stories/error/prologue_error.yaml
 
 result: ERROR
 description: This story should failed due to missing info in the spec.
-warnings:
-  - |
-    Multiple paths detected, please group similar tests together and move paths not being tested to prologues or epilogues.
-      /_cat/health
-      /_cat/indices
 prologues:
   - title: PUT /books
     overall:
diff --git a/tools/tests/tester/fixtures/stories/error/prologue_error.yaml b/tools/tests/tester/fixtures/stories/error/prologue_error.yaml
index 8b11deb92..b3fd6b8bc 100644
--- a/tools/tests/tester/fixtures/stories/error/prologue_error.yaml
+++ b/tools/tests/tester/fixtures/stories/error/prologue_error.yaml
@@ -1,6 +1,8 @@
 $schema: ../../../../../../json_schemas/test_story.schema.yaml
 
 description: This story should failed due to missing info in the spec.
+warnings:
+  multiple-paths-detected: false
 prologues:
   - path: /books
     method: PUT