Specify warnings: at story level.

Signed-off-by: dblock <[email protected]>

TESTING_GUIDE.md

@@ -299,7 +299,7 @@ WARNING Multiple paths detected, please group similar tests together and move pa
 
 #### Suppressing Warnings
 
-The test runner may generate warnings that can be suppressed with `warnings:`. For example, to suppress the multiple paths detected warning.
+The test runner may generate warnings that can be suppressed with `warnings:` at story or chapter level. For example, to suppress the multiple paths detected warning.
 
 ```yaml
 - synopsis: Create an index.

json_schemas/test_story.schema.yaml

@@ -23,6 +23,8 @@ properties:
     $ref: '#/definitions/Version'
   distributions:
     $ref: '#/definitions/Distributions'
+  warnings:
+    $ref: '#/definitions/Warnings'
 required: [chapters,description]
 additionalProperties: false
 

tests/default/ingest/pipeline/neural_search.yaml

@@ -1,6 +1,8 @@
 $schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test the creation a neural search ingest pipeline.
+warnings:
+  multiple-paths-detected: false
 prologues:
   - path: /_cluster/settings
     method: PUT

tools/src/tester/StoryEvaluator.ts

@@ -63,21 +63,22 @@ export default class StoryEvaluator {
       prologues,
       epilogues,
       result: overall_result(prologues.concat(chapters).concat(epilogues).concat(prologues).map(e => e.overall)),
-      warnings: this.#chapter_warnings(story.chapters)
+      warnings: this.#chapter_warnings(story)
     }
   }
 
-  #chapter_warnings(chapters: Chapter[]): string[] | undefined {
+  #chapter_warnings(story: Story): string[] | undefined {
     const result = _.compact([
-      this.#warning_if_mismatched_chapter_paths(chapters)
+      this.#warning_if_mismatched_chapter_paths(story)
     ])
     return result.length > 0 ? result : undefined
   }
 
-  #warning_if_mismatched_chapter_paths(chapters: Chapter[]): string | undefined {
-    const paths = _.compact(_.map(chapters, (chapter) => {
-      const multiple_paths_detected = chapter.warnings?.['multiple-paths-detected'] ?? true
-      if (multiple_paths_detected) return chapter.path
+  #warning_if_mismatched_chapter_paths(story: Story): string | undefined {
+    if (story.warnings?.['multiple-paths-detected'] === false) return
+    const paths = _.compact(_.map(story.chapters, (chapter) => {
+      if (chapter.warnings?.['multiple-paths-detected'] === false) return
+      return chapter.path
     }))
     const normalized_paths = _.map(paths, (path) => path.replaceAll(/\/\{[^}]+}/g, '').replaceAll('//', '/'))
     const paths_counts: Record<string, number> = Object.assign((_.values(_.groupBy(normalized_paths)).map(p => { return { [p[0]] : p.length } })))

tools/src/tester/types/story.types.ts

@@ -103,6 +103,7 @@ export interface Story {
   chapters: Chapter[];
   version?: Version;
   distributions?: Distributions;
+  warnings?: Warnings;
 }
 /**
  * This interface was referenced by `Story`'s JSON-Schema

tools/tests/tester/fixtures/evals/error/prologue_error.yaml

@@ -3,11 +3,6 @@ full_path: tools/tests/tester/fixtures/stories/error/prologue_error.yaml
 
 result: ERROR
 description: This story should failed due to missing info in the spec.
-warnings:
-  - |
-    Multiple paths detected, please group similar tests together and move paths not being tested to prologues or epilogues.
-      /_cat/health
-      /_cat/indices
 prologues:
   - title: PUT /books
     overall:

tools/tests/tester/fixtures/stories/error/prologue_error.yaml

@@ -1,6 +1,8 @@
 $schema: ../../../../../../json_schemas/test_story.schema.yaml
 
 description: This story should failed due to missing info in the spec.
+warnings:
+  multiple-paths-detected: false
 prologues:
   - path: /books
     method: PUT