From 3d896ebb602d6d3c19aa3e7e3700c9ef3d0819fb Mon Sep 17 00:00:00 2001 From: Danny Furnivall Date: Wed, 21 Jun 2023 15:01:15 +0100 Subject: [PATCH] Yarn Audit Fixes and Extensions (#1081) Co-authored-by: Tim Jacomb Co-authored-by: Tim Jacomb <21194782+timja@users.noreply.github.com> --- .../hmcts/pipeline/yarn/prettyPrintAudit.sh | 26 +++ .../yarn/yarn-audit-with-suppressions.sh | 167 ++++++++++++++---- src/uk/gov/hmcts/contino/YarnBuilder.groovy | 8 +- 3 files changed, 160 insertions(+), 41 deletions(-) create mode 100755 resources/uk/gov/hmcts/pipeline/yarn/prettyPrintAudit.sh diff --git a/resources/uk/gov/hmcts/pipeline/yarn/prettyPrintAudit.sh b/resources/uk/gov/hmcts/pipeline/yarn/prettyPrintAudit.sh new file mode 100755 index 0000000000..878941eb61 --- /dev/null +++ b/resources/uk/gov/hmcts/pipeline/yarn/prettyPrintAudit.sh @@ -0,0 +1,26 @@ +#!/bin/bash +filename="$1" + +while IFS= read -r line; do + echo "$line" | jq -r ' + .module_name as $moduleName | + .id as $id | + .title as $issue | + .url as $url | + .severity as $severity | + .vulnerable_versions as $vulnVers | + .patched_versions as $patchVers | + .recommendation as $rec | + .findings[] | + "├─ " + $moduleName + ": " + .version, + "│ ├─ ID: " + ($id|tostring), + "│ ├─ Issue: " + $issue, + "│ ├─ URL: " + $url, + "│ ├─ Severity: " + $severity, + "│ ├─ Vulnerable Versions: " + $vulnVers, + "│ ├─ Patched Versions: " + $patchVers, + "│ ├─ Via: " + (.paths | join(", ")), + "│ └─ Recommendation: " + $rec, + ""' +done < "$filename" + diff --git a/resources/uk/gov/hmcts/pipeline/yarn/yarn-audit-with-suppressions.sh b/resources/uk/gov/hmcts/pipeline/yarn/yarn-audit-with-suppressions.sh index 3b9beac85c..fb7bb78edc 100755 --- a/resources/uk/gov/hmcts/pipeline/yarn/yarn-audit-with-suppressions.sh +++ b/resources/uk/gov/hmcts/pipeline/yarn/yarn-audit-with-suppressions.sh @@ -1,51 +1,146 @@ #!/usr/bin/env bash -# This is a wrapper script for yarn audit that allows suppressing -# vulnerabilities without a fix -set +e -today=$(date +%Y-%m-%d) -enddate="2023-06-15" +################################################################################ +# Yarn Audit Wrapper +# +# This script performs a security audit on Yarn dependencies, with the ability +# to suppress vulnerabilities that are known and have no fix. The audit results +# are output in JSON format. Any new vulnerabilities are reported to the user. +# +# Required Dependencies: +# - jq: A lightweight and flexible command-line JSON processor +# - yarn: Fast, reliable, and secure dependency management +# - prettyPrintAudit.sh: Script to pretty print the audit results +# +# Usage: +# Mostly used in the pipeline but feel free to use the script locally, should still work there: +# Execute the script in the directory containing your project and yarn-audit-known-issues file: +# ./yarn-audit-with-suppressions.sh +# +# Exit Codes: +# 0 - Success, no vulnerabilities found or only known vulnerabilities found +# 1 - Unhandled vulnerabilities were found +################################################################################ -if [[ "$today" > "$enddate" ]]; then -yarn npm audit --recursive --environment production -result=$? -yarn npm audit --recursive --environment production --json > yarn-audit-result -else - yarn npm audit --environment production - result=$? - yarn npm audit --environment production --json > yarn-audit-result -fi +# Exit script on error set -e -if [ "$result" != 0 ]; then - if [ -f yarn-audit-known-issues ]; then - set +e - cat yarn-audit-result | jq -cr '.advisories| to_entries[] | {"type": "auditAdvisory", "data": { "advisory": .value }}' > yarn-audit-issues - set -e - - if diff -q yarn-audit-known-issues yarn-audit-issues > /dev/null 2>&1; then - rm -f yarn-audit-issues - echo - echo Ignoring known vulnerabilities - exit 0 +# Check for dependencies +command -v yarn >/dev/null 2>&1 || { echo >&2 "yarn is required but it's not installed. Aborting."; exit 1; } +command -v jq >/dev/null 2>&1 || { echo >&2 "jq is required but it's not installed. Aborting."; exit 1; } + +# Temporary files cleanup function +cleanup() { +rm -f new_vulnerabilities unneeded_suppressions sorted-yarn-audit-issues sorted-yarn-audit-known-issues active_suppressions unused_suppressions + +} + +# Function to print guidance message in case of found vulnerabilities +print_guidance() { +cat <<'EOF' + + Security vulnerabilities were found that were not ignored. + + Check to see if these vulnerabilities apply to production + + and/or if they have fixes available. If they do not have + + fixes and they do not apply to production, you may ignore them + + To ignore these vulnerabilities, run: + `yarn npm audit --recursive --environment production --json > yarn-audit-known-issues` + + and commit the yarn-audit-known-issues file + +EOF +} + +# Function to check for unneeded suppressions +check_for_unneeded_suppressions() { + while IFS= read -r line; do + if ! grep -Fxq "$line" sorted-yarn-audit-issues; then + echo "$line" >> unneeded_suppressions fi + done < sorted-yarn-audit-known-issues + + if [[ -s unneeded_suppressions ]]; then + echo "WARNING: Unneeded suppressions found. You can safely delete these from the yarn-audit-known-issues file:" + source prettyPrintAudit.sh unneeded_suppressions fi - cat <<'EOF' - Security vulnerabilities were found that were not ignored +} - Check to see if these vulnerabilities apply to production - and/or if they have fixes available. If they do not have - fixes and they do not apply to production, you may ignore them +# Perform yarn audit and process the results +yarn npm audit --recursive --environment production --json > yarn-audit-result +jq -cr '.advisories | to_entries[].value' < yarn-audit-result | sort > sorted-yarn-audit-issues - To ignore these vulnerabilities, run: +# Check if there were any vulnerabilities +if [[ ! -s sorted-yarn-audit-issues ]]; then + echo "No vulnerabilities found in project dependencies." - `yarn npm audit --environment production --recursive --json | jq -cr '.advisories| to_entries[] | {"type": "auditAdvisory", "data": { "advisory": .value }}' > yarn-audit-known-issues` + # Check for unneeded suppressions when no vulnerabilities are present + if [ -f yarn-audit-known-issues ]; then + # Convert JSON array into sorted list of suppressed issues + jq -cr '.advisories | to_entries[].value' yarn-audit-known-issues \ + | sort > sorted-yarn-audit-known-issues - and commit the yarn-audit-known-issues file -EOF + # When no vulnerabilities are found, all suppressions are unneeded + check_for_unneeded_suppressions + fi + + cleanup + exit 0 +fi + +# Check if there are known vulnerabilities +if [ ! -f yarn-audit-known-issues ]; then + source prettyPrintAudit.sh sorted-yarn-audit-issues + print_guidance + cleanup + exit 1 +else + # Test for old format of yarn-audit-known-issues + if ! jq 'has("actions", "advisories", "metadata")' yarn-audit-known-issues | grep -q true; then + echo "You have an invalid `yarn-audit-known-issues` file. \nThe command to suppress known vulnerabilities has changed. Please now use the following: \n`yarn npm audit --recursive --environment production --json > yarn-audit-known-issues`" + exit 1 + fi + + # Handle edge case for when audit returns in different orders for the two files + # Convert JSON array into sorted list of issues. + jq -cr '.advisories | to_entries[].value' yarn-audit-known-issues \ + | sort > sorted-yarn-audit-known-issues + + # Retain old data ingestion style for cosmosDB + jq -cr '.advisories| to_entries[] | {"type": "auditAdvisory", "data": { "advisory": .value }}' yarn-audit-known-issues > yarn-audit-known-issues-result + + # Check each issue in sorted-yarn-audit-result is also present in sorted-yarn-audit-known-issues + while IFS= read -r line; do + if ! grep -Fxq "$line" sorted-yarn-audit-known-issues; then + echo "$line" >> new_vulnerabilities + fi + done < sorted-yarn-audit-issues + + # Check for unneeded suppressions + check_for_unneeded_suppressions - rm -f yarn-audit-issues + # Check if there were any new vulnerabilities + if [[ -s new_vulnerabilities ]]; then + echo "Unsuppressed vulnerabilities found:" + source prettyPrintAudit.sh new_vulnerabilities + print_guidance + cleanup + exit 1 + else + echo "Active suppressed vulnerabilities:" + while IFS= read -r line; do + if grep -Fxq "$line" sorted-yarn-audit-issues; then + echo "$line" >> active_suppressions + fi + done < sorted-yarn-audit-known-issues - exit "$result" + source prettyPrintAudit.sh active_suppressions + cleanup + exit 0 + fi fi + diff --git a/src/uk/gov/hmcts/contino/YarnBuilder.groovy b/src/uk/gov/hmcts/contino/YarnBuilder.groovy index bd433585b8..7debfa6265 100644 --- a/src/uk/gov/hmcts/contino/YarnBuilder.groovy +++ b/src/uk/gov/hmcts/contino/YarnBuilder.groovy @@ -13,7 +13,7 @@ class YarnBuilder extends AbstractBuilder { private static final String NVMRC = '.nvmrc' private static final Float DESIRED_MIN_VERSION = 18.16 private static final LocalDate NODEJS_EXPIRATION = LocalDate.of(2023, 8, 31) - private static final String CVE_KNOWN_ISSUES_FILE_PATH = 'yarn-audit-known-issues' + private static final String CVE_KNOWN_ISSUES_FILE_PATH = 'yarn-audit-known-issues-result' def securitytest @@ -130,6 +130,7 @@ class YarnBuilder extends AbstractBuilder { corepackEnable() steps.writeFile(file: 'yarn-audit-with-suppressions.sh', text: steps.libraryResource('uk/gov/hmcts/pipeline/yarn/yarn-audit-with-suppressions.sh')) + steps.writeFile(file: 'prettyPrintAudit.sh', text: steps.libraryResource('uk/gov/hmcts/pipeline/yarn/prettyPrintAudit.sh')) steps.sh """ export PATH=\$HOME/.local/bin:\$PATH @@ -326,10 +327,7 @@ EOF if (!steps.fileExists(INSTALL_CHECK_FILE)) { steps.sh("touch ${INSTALL_CHECK_FILE}") corepackEnable() - boolean zeroInstallEnabled = steps.fileExists(".yarn/cache") - if (!zeroInstallEnabled) { - runYarn("install") - } + runYarn("install") } runYarn(task, prepend) }