feat: add support for specifying max errors

CMSgov · Sep 20, 2023 · 48db48a · 48db48a
1 parent 23b9ff3
commit 48db48a
Show file tree

Hide file tree

Showing 4 changed files with 80 additions and 12 deletions.
diff --git a/src/csv.ts b/src/csv.ts
@@ -51,6 +51,11 @@ export const TALL_COLUMNS = [
   "additional_generic_notes",
 ]
 
+interface ValidateCsvOptions {
+  maxErrors?: number
+  onValueCallback?: (value: { [key: string]: string }) => void
+}
+
 /**
  *
  * @param input Browser File or ReadableStream for streaming file content
@@ -59,7 +64,7 @@ export const TALL_COLUMNS = [
  */
 export async function validateCsv(
   input: File | NodeJS.ReadableStream,
-  onValueCallback?: (value: { [key: string]: string }) => void
+  options: ValidateCsvOptions = {}
 ): Promise<ValidationResult> {
   let index = 0
   const errors: CsvValidationError[] = []
@@ -69,7 +74,8 @@ export async function validateCsv(
 
   const handleParseStep = (
     step: Papa.ParseStepResult<string[]>,
-    resolve: (result: ValidationResult | PromiseLike<ValidationResult>) => void
+    resolve: (result: ValidationResult | PromiseLike<ValidationResult>) => void,
+    parser: Papa.Parser
   ) => {
     const row: string[] = step.data
     // Ignore empty lines
@@ -92,17 +98,33 @@ export async function validateCsv(
             message: "Errors were seen in headers so rows were not evaluated",
           }),
         })
+        parser.abort()
       } else {
         tall = isTall(dataColumns)
       }
     } else {
       const cleanRow = cleanRowFields(objectFromKeysValues(dataColumns, row))
       errors.push(...validateRow(cleanRow, index, dataColumns, !tall))
-      if (onValueCallback) {
-        onValueCallback(cleanRow)
+
+      if (options.onValueCallback) {
+        options.onValueCallback(cleanRow)
       }
     }
 
+    if (
+      options.maxErrors &&
+      options.maxErrors > 0 &&
+      errors.length > options.maxErrors
+    ) {
+      resolve({
+        valid: false,
+        errors: errors
+          .map(csvErrorToValidationError)
+          .slice(0, options.maxErrors),
+      })
+      parser.abort()
+    }
+
     ++index
   }
 
@@ -131,9 +153,9 @@ export async function validateCsv(
     Papa.parse(input, {
       header: false,
       // chunkSize: 64 * 1024,
-      step: (row: Papa.ParseStepResult<string[]>) => {
+      step: (row: Papa.ParseStepResult<string[]>, parser: Papa.Parser) => {
         try {
-          handleParseStep(row, resolve)
+          handleParseStep(row, resolve, parser)
         } catch (e) {
           reject(e)
         }

diff --git a/src/json.ts b/src/json.ts
@@ -184,6 +184,13 @@ export const JSON_SCHEMA = {
   required: [...METADATA_REQUIRED, "standard_charge_information"],
 }
 
+interface ValidateJsonOptions {
+  maxErrors?: number
+  onValueCallback?: (
+    val: JsonTypes.JsonPrimitive | JsonTypes.JsonStruct
+  ) => void
+}
+
 /**
  *
  * @param jsonInput Browser File or ReadableStream to stream content from
@@ -192,9 +199,7 @@ export const JSON_SCHEMA = {
  */
 export async function validateJson(
   jsonInput: File | NodeJS.ReadableStream,
-  onValueCallback?: (
-    val: JsonTypes.JsonPrimitive | JsonTypes.JsonStruct
-  ) => void
+  options: ValidateJsonOptions = {}
 ): Promise<ValidationResult> {
   const validator = new Ajv({ allErrors: true })
   addFormats(validator)
@@ -228,8 +233,19 @@ export async function validateJson(
               }))
           )
         }
-        if (onValueCallback) {
-          onValueCallback(value)
+        if (options.onValueCallback) {
+          options.onValueCallback(value)
+        }
+        if (
+          options.maxErrors &&
+          options.maxErrors > 0 &&
+          errors.length > options.maxErrors
+        ) {
+          resolve({
+            valid: false,
+            errors: errors.slice(0, options.maxErrors),
+          })
+          parser.end()
         }
       }
     }
@@ -249,7 +265,13 @@ export async function validateJson(
           )
         )
       }
-      resolve({ valid, errors })
+      resolve({
+        valid,
+        errors:
+          options.maxErrors && options.maxErrors > 0
+            ? errors.slice(0, options.maxErrors)
+            : errors,
+      })
     }
 
     parser.onError = (e) => reject(e)

diff --git a/test/csv.e2e.spec.ts b/test/csv.e2e.spec.ts
@@ -26,6 +26,22 @@ test("sample-1", async (t) => {
   ])
 })
 
+test("sample-1 maxErrors", async (t) => {
+  t.deepEqual(
+    (await validateCsv(loadFixtureStream("sample-1.csv"), { maxErrors: 1 }))
+      .errors,
+    [
+      {
+        field: "code | 1 | type",
+        message:
+          '"code | 1 | type" value "C" is not one of the allowed values: "CPT", "HCPCS", "ICD", "MS-DRG", "R-DRG", "S-DRG", "APS-DRG", "AP-DRG", "APR-DRG", "APC", "NDC", "HIPPS", "LOCAL", "EAPG", "CDT", "RC", "CDM"',
+        path: "C4",
+        warning: true,
+      },
+    ]
+  )
+})
+
 test("sample-1 sync", (t) => {
   t.deepEqual(validateCsvSync(loadFixture("sample-1.csv")).errors, [
     {

diff --git a/test/json.spec.ts b/test/json.spec.ts
@@ -14,6 +14,14 @@ test("validateJson empty", async (t) => {
   t.deepEqual(result.errors.length, 4)
 })
 
+test("validateJson maxErrors", async (t) => {
+  const result = await validateJson(loadFixtureStream("sample-empty.json"), {
+    maxErrors: 1,
+  })
+  t.is(result.valid, false)
+  t.deepEqual(result.errors.length, 1)
+})
+
 test("validateJson valid", async (t) => {
   const result = await validateJson(loadFixtureStream("sample-valid.json"))
   t.is(result.valid, true)