automate content generation and validation via CLI (#614)

* Introduce cucumber testing of yaml unit tests
* introduce content generation and validation via CLI
* use junit
* eslint format

.github/workflows/create-release.yml

@@ -5,7 +5,7 @@ on:
       - "*"
 jobs:
   build-release:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     steps:
       # Check-out the repository under $GITHUB_WORKSPACE

.github/workflows/run-tests.yml

@@ -7,18 +7,18 @@ on:
   push:
     branches:
       - master
+      - develop
+      - 'feature/**'  # This will match any branch starting with "feature"
+
   pull_request:
 
 # the job requires some dependencies to be installed (including submodules), runs the tests, and then reports results
 jobs:
   # one job that runs tests
   run-tests:
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
-      matrix:
-        os: [macos-latest, ubuntu-latest, windows-latest]
-        arch: [arm64, x86_64]
 
     # Checkout repository and its submodules
     steps:
@@ -32,39 +32,19 @@ jobs:
         run: echo ::set-output name=NODE_VERSION::$(cat .nvmrc)
 
       - name: Install required node.js version
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b
         with:
           node-version: ${{ steps.nvmrc.outputs.NODE_VERSION }}
-
-      - name: Install required Python version
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-
-      # Initialize the workspace with submodules and dependencies.
-      - name: Initialize workspace
-        shell: bash
-        run: make init
-
-      # Compile Schematron to XSL.
-      - name: Compile Schematron
-        shell: bash
-        run: make build-validations
-
-      - name: Run complete test suite
-        shell: bash
-        if: runner.os == 'Linux'
+      - name: Install OSCAL CLI
         run: |
-          make test
-
-      - name: Run limited test suite
+          make init-validations
+      - name: Run Cucumber tests
         shell: bash
-        if: runner.os == 'Windows' || runner.os == 'macOS'
         run: |
-          make test-validations test-web
-
-      - name: Build all
-        shell: bash
-        if: runner.os == 'Windows' || runner.os == 'macOS'
-        run: |
-          make build
+          make build-validations
+      - name : Publish all Junit XML tests results in Github Summary
+        uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86
+        if: always()
+        with:
+            paths: |
+              **/reports/junit-*.xml

.gitignore

@@ -35,3 +35,4 @@ src/validations/test/rules/rev5/ssp-result.html
 src/validations/test/rules/rev4/ssp-result.html
 src/validations/test/rules/rev5/poam-result.html
 src/validations/test/rules/rev5/sar-result.html
+/reports

.nvmrc

@@ -1 +1 @@
-v18.15.0
+v20.16.0

Makefile

@@ -13,7 +13,7 @@ include src/web/module.mk
 
 all: clean build test  ## Complete clean build with tests
 
-init: init-repo init-validations init-web  ## Initialize project dependencies
+init: init-repo init-validations init-content init-web  ## Initialize project dependencies
 
 init-repo:
 	git submodule update --init --recursive

cucumber.json

@@ -1,11 +1,11 @@
 {
-    "default": {
-      "requireModule": ["ts-node/register"],
-      "import": ["features/**/*.ts"],
-      "format": [
-        ["html", "reports/constraints.html"]
-      ],
-      "retry": 2,
-      "retryTagFilter": "@flaky"
-    }
-  }
+  "default": {
+    "requireModule": ["ts-node/register"],
+    "import": ["features/**/*.ts"],
+    "format": [
+      ["junit", "reports/junit-constraints.xml"]
+    ],
+    "retry": 2,
+    "retryTagFilter": "@flaky"
+  }
+}

features/steps/fedramp_extensions_steps.ts

@@ -1,12 +1,12 @@
 import { Given, Then, When, setDefaultTimeout } from "@cucumber/cucumber";
 import { expect } from "chai";
-import { readFileSync, readdirSync, writeFileSync, unlinkSync } from "fs";
+import { readFileSync, readdirSync, unlinkSync, writeFileSync } from "fs";
 import { load } from "js-yaml";
-import { executeOscalCliCommand, validateFileSarif } from "oscal";
-import { validateWithSarif } from "oscal/dist/commands";
+import { executeOscalCliCommand, validateFile, validateWithSarif } from "oscal";
 import { dirname, join } from "path";
+import { Exception, Log } from "sarif";
 import { fileURLToPath } from "url";
-const DEFAULT_TIMEOUT = 17000;
+const DEFAULT_TIMEOUT = 60000;
 
 setDefaultTimeout(DEFAULT_TIMEOUT);
 
@@ -77,7 +77,7 @@ When("I process the constraint unit test {string}", async function (testFile) {
 
 Then("the constraint unit test should pass", async function () {
   const result = await processTestCase(currentTestCase);
-  expect(result).to.equal("pass");
+  expect(result.status).to.equal("pass", result.errorMessage);
 });
 
 async function processTestCase({ "test-case": testCase }: any) {
@@ -121,18 +121,29 @@ async function processTestCase({ "test-case": testCase }: any) {
 
   //Validate processed content
   // Check expectations
-  const sarifResponse = await validateWithSarif([
-    processedContentPath,
-    "--sarif-include-pass",
-    ...metaschemaDocuments.flatMap((x) => [
-      "-c",
-      "./src/validations/constraints/" + x,
-    ]),
-  ]);
-  if (processedContentPath != contentPath) {
-    unlinkSync(processedContentPath);
+  try {
+    const sarifResponse = await validateWithSarif([
+      processedContentPath,
+      "--sarif-include-pass",
+      ...metaschemaDocuments.flatMap((x) => [
+        "-c",
+        "./src/validations/constraints/" + x,
+      ]),
+    ]);
+    if(typeof sarifResponse.runs[0].tool.driver.rules==='undefined'){
+      const [result,error]=await executeOscalCliCommand("validate",[processedContentPath,...metaschemaDocuments.flatMap((x) => [
+        "-c",
+        "./src/validations/constraints/" + x,
+      ])]);
+      return {status:'fail',errorMessage:error}
+    }  
+    if (processedContentPath != contentPath) {
+      unlinkSync(processedContentPath);
+    }
+    return checkConstraints(sarifResponse, testCase.expectations);
+  } catch(e) {
+    return { status: "fail", errorMessage: e.toString() };
   }
-  return checkConstraints(sarifResponse, testCase.expectations);
 }
 
 async function checkConstraints(
@@ -143,42 +154,62 @@ async function checkConstraints(
   const [run] = runs;
   const { results, tool } = run;
   if (!results) {
-    return "no results in sarif output";
+    console.error("No Results")
+    return { status: "fail", errorMessage: "No results in SARIF output" };
   }
   const { driver } = tool;
   if (runs.length != 1) {
-    throw "no runs found in sarif";
+    console.error("No Runs")
+    return { status: "fail", errorMessage: "No runs found in SARIF" };
+  }
+  const rules  = runs[0].tool.driver.rules;
+  if (typeof rules==='undefined'||rules.length == 0) {
+    return { status: "fail", errorMessage: "No rules found in SARIF" };
   }
-  const { rules } = runs[0].tool.driver;
   let constraintResults = [];
+  let errors = [];
+  // List all SARIF outputs with "fail" result
+  const failedResults = results.filter(result => result.kind === "fail");
+  if (failedResults.length > 0) {
+    errors.push("Failed SARIF outputs:");
+    failedResults.forEach(result => {
+      const rule = rules.find(r => r.id === result.ruleId);
+      const ruleName = rule ? rule.name : result.ruleId;
+      errors.push(`- ${ruleName}: ${result.message.text}`);
+    });
+  }
+
   for (const expectation of constraints) {
     const constraint_id = expectation["constraint-id"];
     const expectedResult = expectation.result;
+    console.log("Checking status of constraint: "+constraint_id+" expecting:"+expectedResult);
     const constraintMatch = rules.find((x) => x.name === constraint_id);
     const { id } = constraintMatch || { id: undefined };
     if (!id) {
-      writeFileSync("./"+constraint_id+".sarif.json", JSON.stringify(sarifOutput));
-      console.error("Sarif results written to file: ./"+constraint_id+".sarif.json");
-      throw constraint_id + " rule not defined in sarif results";
+      console.log("Recieved: "+id);
+      writeFileSync("./" + constraint_id + ".sarif.json", JSON.stringify(sarifOutput));
+      console.log("SARIF results written to file: ./" + constraint_id + ".sarif.json");
+      errors.push(`${constraint_id} rule not defined in SARIF results`);
+      continue;
     }
     const constraintResult = results.find((x) => x.ruleId === id);
+    console.log("Recieved: "+constraintResult.kind);
 
     const constraintMatchesExpectation = constraintResult.kind == expectedResult;
-    constraintResults.push(constraintResult ? "pass" : "fail");
+    constraintResults.push(constraintMatchesExpectation ? "pass" : "fail");
     if (!constraintMatchesExpectation) {
-      console.error(
-        constraint_id +
-          " Did not match expected " +
-          result +
-          " recieved " +
-          constraintResult.kind,
+      errors.push(
+        `${constraint_id}: Expected ${expectedResult}, received ${constraintResult.kind}`
       );
     }
   }
-  if (constraintResults.includes("fail")) {
-    return "fail";
+
+  if (errors.length > 0) {
+    return {
+      status: "fail",
+      errorMessage: "Test failed with the following errors:\n" + errors.join("\n")
+    };
   }
-  return "pass";
+  return { status: "pass", errorMessage: "" };
 }
 
-// We don't need the Before hook anymore, so it's removed

package.json

@@ -13,7 +13,7 @@
     "@apidevtools/json-schema-ref-parser": "^11.6.4",
     "chai": "^5.1.1",
     "js-yaml": "^4.1.0",
-    "oscal": "^1.3.2",
+    "oscal": "^1.3.4",
     "ts-node": "^10.9.2",
     "xml2js": "^0.6.2"
   },