Merge pull request #498 from chris48s/yaml-multidoc

allow parsing multi-doc yaml files

docs/docs/usage-examples.md

@@ -28,10 +28,12 @@ By default, v8r queries [Schema Store](https://www.schemastore.org/) to detect a
 ```bash
 # if v8r can't auto-detect a schema for your file..
 $ v8r feature.geojson
+ℹ Processing ./feature.geojson
 ✖ Could not find a schema to validate feature.geojson
 
 # ..you can specify one using the --schema flag
 $ v8r feature.geojson --schema https://json.schemastore.org/geojson
+ℹ Processing ./feature.geojson
 ℹ Validating feature.geojson against schema from https://json.schemastore.org/geojson ...
 ✔ feature.geojson is valid
 ```
@@ -50,11 +52,29 @@ Using the `--schema` flag will validate all files matched by the glob pattern ag
                  "fileMatch": ["*.geojson"] } ] }
 ```
 
-```bash
+```
 $ v8r feature.geojson -c my-catalog.json
+ℹ Processing ./feature.geojson
 ℹ Found schema in my-catalog.json ...
 ℹ Validating feature.geojson against schema from https://json.schemastore.org/geojson ...
 ✔ feature.geojson is valid
 ```
 
 This can be used to specify different custom schemas for multiple file patterns.
+
+## Files Containing Multiple Documents
+
+A single YAML file can contain [multiple documents](https://www.yaml.info/learn/document.html). v8r is able to parse and validate these files. In this situation:
+
+- All documents within the file are assumed to conform to the same schema. It is not possible to validate documents within the same file against different schemas
+- Documents within the file are referred to as `multi-doc.yml[0]`, `multi-doc.yml[1]`, etc
+
+```
+$ v8r catalog-info.yaml
+ℹ Processing ./catalog-info.yaml
+ℹ Found schema in https://www.schemastore.org/api/json/catalog.json ...
+ℹ Validating ./catalog-info.yaml against schema from https://json.schemastore.org/catalog-info.json ...
+✔ ./catalog-info.yaml[0] is valid
+
+✔ ./catalog-info.yaml[1] is valid
+```

src/cli.js

@@ -10,6 +10,7 @@ import { getCatalogs, getMatchForFilename } from "./catalogs.js";
 import { getFiles } from "./glob.js";
 import { getFromUrlOrFile } from "./io.js";
 import logger from "./logger.js";
+import { getDocumentLocation } from "./output-formatters.js";
 import { parseFile } from "./parser.js";
 
 const EXIT = {
@@ -33,61 +34,122 @@ function getFlatCache() {
   return flatCache.load("v8r", CACHE_DIR);
 }
 
-async function validateFile(filename, config, plugins, cache) {
-  logger.info(`Processing ${filename}`);
+async function validateDocument(
+  fileLocation,
+  documentIndex,
+  document,
+  schemaLocation,
+  schema,
+  strictMode,
+  cache,
+  resolver,
+) {
   let result = {
-    fileLocation: filename,
-    schemaLocation: null,
+    fileLocation,
+    documentIndex,
+    schemaLocation,
     valid: null,
     errors: [],
     code: null,
   };
+  try {
+    const { valid, errors } = await validate(
+      document,
+      schema,
+      strictMode,
+      cache,
+      resolver,
+    );
+    result.valid = valid;
+    result.errors = errors;
+
+    const documentLocation = getDocumentLocation(result);
+    if (valid) {
+      logger.success(`${documentLocation} is valid\n`);
+    } else {
+      logger.error(`${documentLocation} is invalid\n`);
+    }
+
+    result.code = valid ? EXIT.VALID : EXIT.INVALID;
+    return result;
+  } catch (e) {
+    logger.error(`${e.message}\n`);
+    result.code = EXIT.ERROR;
+    return result;
+  }
+}
+
+async function validateFile(filename, config, plugins, cache) {
+  logger.info(`Processing ${filename}`);
+
+  let schema, schemaLocation, documents, strictMode, resolver;
+
   try {
     const catalogs = getCatalogs(config);
     const catalogMatch = config.schema
       ? {}
       : await getMatchForFilename(catalogs, filename, cache);
-    const schemaLocation = config.schema || catalogMatch.location;
-    result.schemaLocation = schemaLocation;
-    const schema = await getFromUrlOrFile(schemaLocation, cache);
+    schemaLocation = config.schema || catalogMatch.location;
+    schema = await getFromUrlOrFile(schemaLocation, cache);
     logger.info(
       `Validating ${filename} against schema from ${schemaLocation} ...`,
     );
 
-    const data = parseFile(
+    documents = parseFile(
       plugins,
       await fs.promises.readFile(filename, "utf8"),
       filename,
       catalogMatch.parser,
     );
 
-    const strictMode = config.verbose >= 2 ? "log" : false;
-    const resolver = isUrl(schemaLocation)
+    strictMode = config.verbose >= 2 ? "log" : false;
+    resolver = isUrl(schemaLocation)
       ? (location) => getFromUrlOrFile(location, cache)
       : (location) =>
           getFromUrlOrFile(location, cache, path.dirname(schemaLocation));
-    const { valid, errors } = await validate(
-      data,
+  } catch (e) {
+    logger.error(`${e.message}\n`);
+    return [
+      {
+        fileLocation: filename,
+        documentIndex: null,
+        schemaLocation: schemaLocation || null,
+        valid: null,
+        errors: [],
+        code: EXIT.ERROR,
+      },
+    ];
+  }
+
+  let results = [];
+  for (let i = 0; i < documents.length; i++) {
+    const documentIndex = documents.length === 1 ? null : i;
+    const result = await validateDocument(
+      filename,
+      documentIndex,
+      documents[i],
+      schemaLocation,
       schema,
       strictMode,
       cache,
       resolver,
     );
-    result.valid = valid;
-    result.errors = errors;
-    if (valid) {
-      logger.success(`${filename} is valid\n`);
-    } else {
-      logger.error(`${filename} is invalid\n`);
-    }
 
-    result.code = valid ? EXIT.VALID : EXIT.INVALID;
-    return result;
-  } catch (e) {
-    logger.error(`${e.message}\n`);
-    result.code = EXIT.ERROR;
-    return result;
+    results.push(result);
+
+    for (const plugin of plugins) {
+      const message = plugin.getSingleResultLogMessage(
+        result,
+        filename,
+        config.format,
+      );
+      if (message != null) {
+        logger.log(message);
+        break;
+      }
+    }
   }
+  return results;
 }
 
 function resultsToStatusCode(results, ignoreErrors) {
@@ -122,21 +184,8 @@ function Validator() {
 
     let results = [];
     for (const filename of filenames) {
-      const result = await validateFile(filename, config, plugins, cache);
-      results.push(result);
-
-      for (const plugin of plugins) {
-        const message = plugin.getSingleResultLogMessage(
-          result,
-          filename,
-          config.format,
-        );
-        if (message != null) {
-          logger.log(message);
-          break;
-        }
-      }
-
+      const fileResults = await validateFile(filename, config, plugins, cache);
+      results = results.concat(fileResults);
       cache.resetCounters();
     }
 

src/cli.spec.js

@@ -414,6 +414,24 @@ describe("CLI", function () {
       });
     });
 
+    it("should validate yaml files containing multiple documents", function () {
+      return cli({
+        patterns: ["./testfiles/files/multi-doc.yaml"],
+        schema: "./testfiles/schemas/schema.json",
+      }).then((result) => {
+        assert.equal(result, 99);
+        assert(
+          logContainsSuccess("./testfiles/files/multi-doc.yaml[0] is valid"),
+        );
+        assert(
+          logContainsSuccess("./testfiles/files/multi-doc.yaml[1] is valid"),
+        );
+        assert(
+          logContainsError("./testfiles/files/multi-doc.yaml[2] is invalid"),
+        );
+      });
+    });
+
     it("should validate json5 files", function () {
       return cli({
         patterns: ["./testfiles/files/valid.json5"],
@@ -998,7 +1016,7 @@ describe("CLI", function () {
       tearDown();
     });
 
-    it("should log errors in text format when format is text", async function () {
+    it("should log errors in text format when format is text (single doc)", async function () {
       return cli({
         patterns: [
           "{./testfiles/files/valid.json,./testfiles/files/invalid.json,./testfiles/files/not-supported.txt}",
@@ -1014,7 +1032,21 @@ describe("CLI", function () {
       });
     });
 
-    it("should output json report when format is json", async function () {
+    it("should log errors in text format when format is text (multi doc)", async function () {
+      return cli({
+        patterns: ["./testfiles/files/multi-doc.yaml"],
+        schema: "./testfiles/schemas/schema.json",
+        format: "text",
+      }).then(() => {
+        assert(
+          logger.stdout.includes(
+            "./testfiles/files/multi-doc.yaml[2]#/num must be number\n",
+          ),
+        );
+      });
+    });
+
+    it("should output json report when format is json (single doc)", async function () {
       return cli({
         patterns: [
           "{./testfiles/files/valid.json,./testfiles/files/invalid.json,./testfiles/files/not-supported.txt}",
@@ -1038,23 +1070,74 @@ describe("CLI", function () {
                 },
               ],
               fileLocation: "./testfiles/files/invalid.json",
+              documentIndex: null,
               schemaLocation: "./testfiles/schemas/schema.json",
               valid: false,
             },
             {
               code: 1,
               errors: [],
               fileLocation: "./testfiles/files/not-supported.txt",
+              documentIndex: null,
               schemaLocation: "./testfiles/schemas/schema.json",
               valid: null,
             },
             {
               code: 0,
               errors: [],
               fileLocation: "./testfiles/files/valid.json",
+              documentIndex: null,
+              schemaLocation: "./testfiles/schemas/schema.json",
+              valid: true,
+            },
+          ],
+        };
+        assert.deepStrictEqual(JSON.parse(logger.stdout[0]), expected);
+      });
+    });
+
+    it("should output json report when format is json (multi doc)", async function () {
+      return cli({
+        patterns: ["./testfiles/files/multi-doc.yaml"],
+        schema: "./testfiles/schemas/schema.json",
+        format: "json",
+      }).then(() => {
+        const expected = {
+          results: [
+            {
+              code: 0,
+              errors: [],
+              fileLocation: "./testfiles/files/multi-doc.yaml",
+              documentIndex: 0,
+              schemaLocation: "./testfiles/schemas/schema.json",
+              valid: true,
+            },
+            {
+              code: 0,
+              errors: [],
+              fileLocation: "./testfiles/files/multi-doc.yaml",
+              documentIndex: 1,
               schemaLocation: "./testfiles/schemas/schema.json",
               valid: true,
             },
+            {
+              code: 99,
+              errors: [
+                {
+                  instancePath: "/num",
+                  keyword: "type",
+                  message: "must be number",
+                  params: {
+                    type: "number",
+                  },
+                  schemaPath: "#/properties/num/type",
+                },
+              ],
+              fileLocation: "./testfiles/files/multi-doc.yaml",
+              documentIndex: 2,
+              schemaLocation: "./testfiles/schemas/schema.json",
+              valid: false,
+            },
           ],
         };
         assert.deepStrictEqual(JSON.parse(logger.stdout[0]), expected);

src/output-formatters.js

@@ -1,13 +1,20 @@
 import Ajv from "ajv";
 
-function formatErrors(filename, errors) {
+function getDocumentLocation(result) {
+  if (result.documentIndex == null) {
+    return result.fileLocation;
+  }
+  return `${result.fileLocation}[${result.documentIndex}]`;
+}
+
+function formatErrors(location, errors) {
   const ajv = new Ajv();
   return (
     ajv.errorsText(errors, {
       separator: "\n",
-      dataVar: filename + "#",
+      dataVar: location + "#",
     }) + "\n"
   );
 }
 
-export { formatErrors };
+export { formatErrors, getDocumentLocation };