Yarn Audit Fixes and Extensions (#1081)

Co-authored-by: Tim Jacomb <[email protected]>
Co-authored-by: Tim Jacomb <[email protected]>

resources/uk/gov/hmcts/pipeline/yarn/prettyPrintAudit.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+filename="$1"
+
+while IFS= read -r line; do
+    echo "$line" | jq -r '
+        .module_name as $moduleName |
+        .id as $id |
+        .title as $issue |
+        .url as $url |
+        .severity as $severity |
+        .vulnerable_versions as $vulnVers |
+        .patched_versions as $patchVers |
+        .recommendation as $rec |
+        .findings[] |
+        "├─ " + $moduleName + ": " + .version,
+        "│  ├─ ID: " + ($id|tostring),
+        "│  ├─ Issue: " + $issue,
+        "│  ├─ URL: " + $url,
+        "│  ├─ Severity: " + $severity,
+        "│  ├─ Vulnerable Versions: " + $vulnVers,
+        "│  ├─ Patched Versions: " + $patchVers,
+        "│  ├─ Via: " + (.paths | join(", ")),
+        "│  └─ Recommendation: " + $rec,
+        ""'
+done < "$filename"
+

resources/uk/gov/hmcts/pipeline/yarn/yarn-audit-with-suppressions.sh

@@ -1,51 +1,146 @@
 #!/usr/bin/env bash
-# This is a wrapper script for yarn audit that allows suppressing
-# vulnerabilities without a fix
 
-set +e
-today=$(date +%Y-%m-%d)
-enddate="2023-06-15"
+################################################################################
+# Yarn Audit Wrapper
+#
+# This script performs a security audit on Yarn dependencies, with the ability
+# to suppress vulnerabilities that are known and have no fix. The audit results
+# are output in JSON format. Any new vulnerabilities are reported to the user.
+#
+# Required Dependencies:
+# - jq: A lightweight and flexible command-line JSON processor
+# - yarn: Fast, reliable, and secure dependency management
+# - prettyPrintAudit.sh: Script to pretty print the audit results
+#
+# Usage:
+# Mostly used in the pipeline but feel free to use the script locally, should still work there:
+# Execute the script in the directory containing your project and yarn-audit-known-issues file:
+# ./yarn-audit-with-suppressions.sh
+#
+# Exit Codes:
+# 0 - Success, no vulnerabilities found or only known vulnerabilities found
+# 1 - Unhandled vulnerabilities were found
+################################################################################
 
-if [[ "$today" > "$enddate" ]]; then
-yarn npm audit --recursive --environment production
-result=$?
-yarn npm audit --recursive --environment production --json > yarn-audit-result
-else
- yarn npm audit --environment production
- result=$?
- yarn npm audit --environment production --json > yarn-audit-result
-fi
 
+# Exit script on error
 set -e
 
-if [ "$result" != 0 ]; then
-  if [ -f yarn-audit-known-issues ]; then
-    set +e
-    cat yarn-audit-result | jq -cr '.advisories| to_entries[] | {"type": "auditAdvisory", "data": { "advisory": .value }}' > yarn-audit-issues
-    set -e
-
-    if diff -q yarn-audit-known-issues yarn-audit-issues > /dev/null 2>&1; then
-      rm -f yarn-audit-issues
-      echo
-      echo Ignoring known vulnerabilities
-      exit 0
+# Check for dependencies
+command -v yarn >/dev/null 2>&1 || { echo >&2 "yarn is required but it's not installed. Aborting."; exit 1; }
+command -v jq >/dev/null 2>&1 || { echo >&2 "jq is required but it's not installed. Aborting."; exit 1; }
+
+# Temporary files cleanup function
+cleanup() {
+rm -f new_vulnerabilities unneeded_suppressions sorted-yarn-audit-issues sorted-yarn-audit-known-issues active_suppressions unused_suppressions
+
+}
+
+# Function to print guidance message in case of found vulnerabilities
+print_guidance() {
+cat <<'EOF'
+
+  Security vulnerabilities were found that were not ignored.
+
+  Check to see if these vulnerabilities apply to production
+
+  and/or if they have fixes available. If they do not have
+
+  fixes and they do not apply to production, you may ignore them
+
+  To ignore these vulnerabilities, run:
+  `yarn npm audit --recursive --environment production --json > yarn-audit-known-issues`
+
+  and commit the yarn-audit-known-issues file
+
+EOF
+}
+
+# Function to check for unneeded suppressions
+check_for_unneeded_suppressions() {
+  while IFS= read -r line; do
+    if ! grep -Fxq "$line" sorted-yarn-audit-issues; then
+      echo "$line" >> unneeded_suppressions
     fi
+  done < sorted-yarn-audit-known-issues
+
+  if [[ -s unneeded_suppressions ]]; then
+    echo "WARNING: Unneeded suppressions found. You can safely delete these from the yarn-audit-known-issues file:"
+    source prettyPrintAudit.sh unneeded_suppressions
   fi
-  cat <<'EOF'
-    Security vulnerabilities were found that were not ignored
+}
 
-    Check to see if these vulnerabilities apply to production
-    and/or if they have fixes available. If they do not have
-    fixes and they do not apply to production, you may ignore them
+# Perform yarn audit and process the results
+yarn npm audit --recursive --environment production --json > yarn-audit-result
+jq -cr '.advisories | to_entries[].value' < yarn-audit-result | sort > sorted-yarn-audit-issues
 
-    To ignore these vulnerabilities, run:
+# Check if there were any vulnerabilities
+if [[ ! -s sorted-yarn-audit-issues ]];  then
+  echo "No vulnerabilities found in project dependencies."
 
-    `yarn npm audit --environment production --recursive --json | jq -cr '.advisories| to_entries[] | {"type": "auditAdvisory", "data": { "advisory": .value }}' > yarn-audit-known-issues`
+  # Check for unneeded suppressions when no vulnerabilities are present
+  if [ -f yarn-audit-known-issues ]; then
+    # Convert JSON array into sorted list of suppressed issues
+    jq -cr '.advisories | to_entries[].value' yarn-audit-known-issues \
+    | sort > sorted-yarn-audit-known-issues
 
-    and commit the yarn-audit-known-issues file
-EOF
+    # When no vulnerabilities are found, all suppressions are unneeded
+    check_for_unneeded_suppressions
+  fi
+
+  cleanup
+  exit 0
+fi
+
+# Check if there are known vulnerabilities
+if [ ! -f yarn-audit-known-issues ]; then
+  source prettyPrintAudit.sh sorted-yarn-audit-issues
+  print_guidance
+  cleanup
+  exit 1
+else
+  # Test for old format of yarn-audit-known-issues
+  if ! jq 'has("actions", "advisories", "metadata")' yarn-audit-known-issues | grep -q true; then
+    echo "You have an invalid `yarn-audit-known-issues` file. \nThe command to suppress known vulnerabilities has changed. Please now use the following: \n`yarn npm audit --recursive --environment production --json > yarn-audit-known-issues`"
+    exit 1
+  fi
+
+  # Handle edge case for when audit returns in different orders for the two files
+  # Convert JSON array into sorted list of issues.
+  jq -cr '.advisories | to_entries[].value' yarn-audit-known-issues \
+  | sort > sorted-yarn-audit-known-issues
+
+  # Retain old data ingestion style for cosmosDB
+  jq -cr '.advisories| to_entries[] | {"type": "auditAdvisory", "data": { "advisory": .value }}' yarn-audit-known-issues > yarn-audit-known-issues-result
+
+  # Check each issue in sorted-yarn-audit-result is also present in sorted-yarn-audit-known-issues
+  while IFS= read -r line; do
+    if ! grep -Fxq "$line" sorted-yarn-audit-known-issues; then
+      echo "$line" >> new_vulnerabilities
+    fi
+  done < sorted-yarn-audit-issues
+
+  # Check for unneeded suppressions
+  check_for_unneeded_suppressions
 
-  rm -f yarn-audit-issues
+  # Check if there were any new vulnerabilities
+  if [[ -s new_vulnerabilities ]]; then
+    echo "Unsuppressed vulnerabilities found:"
+    source prettyPrintAudit.sh new_vulnerabilities
+    print_guidance
+    cleanup
+    exit 1
+  else
+    echo "Active suppressed vulnerabilities:"
+    while IFS= read -r line; do
+        if grep -Fxq "$line" sorted-yarn-audit-issues; then
+            echo "$line" >> active_suppressions
+        fi
+    done < sorted-yarn-audit-known-issues
 
-  exit "$result"
+    source prettyPrintAudit.sh active_suppressions
+    cleanup
+    exit 0
+  fi
 fi
+

src/uk/gov/hmcts/contino/YarnBuilder.groovy

@@ -13,7 +13,7 @@ class YarnBuilder extends AbstractBuilder {
   private static final String NVMRC = '.nvmrc'
   private static final Float DESIRED_MIN_VERSION = 18.16
   private static final LocalDate NODEJS_EXPIRATION = LocalDate.of(2023, 8, 31)
-  private static final String CVE_KNOWN_ISSUES_FILE_PATH = 'yarn-audit-known-issues'
+  private static final String CVE_KNOWN_ISSUES_FILE_PATH = 'yarn-audit-known-issues-result'
 
   def securitytest
 
@@ -130,6 +130,7 @@ class YarnBuilder extends AbstractBuilder {
 
       corepackEnable()
       steps.writeFile(file: 'yarn-audit-with-suppressions.sh', text: steps.libraryResource('uk/gov/hmcts/pipeline/yarn/yarn-audit-with-suppressions.sh'))
+      steps.writeFile(file: 'prettyPrintAudit.sh', text: steps.libraryResource('uk/gov/hmcts/pipeline/yarn/prettyPrintAudit.sh'))
 
       steps.sh """
          export PATH=\$HOME/.local/bin:\$PATH
@@ -326,10 +327,7 @@ EOF
     if (!steps.fileExists(INSTALL_CHECK_FILE)) {
       steps.sh("touch ${INSTALL_CHECK_FILE}")
       corepackEnable()
-      boolean zeroInstallEnabled = steps.fileExists(".yarn/cache")
-      if (!zeroInstallEnabled) {
-        runYarn("install")
-      }
+      runYarn("install")
     }
     runYarn(task, prepend)
   }