diff --git a/resources/uk/gov/hmcts/pipeline/yarn/yarnv4audit.py b/resources/uk/gov/hmcts/pipeline/yarn/yarnv4audit.py new file mode 100644 index 0000000000..925e332dc3 --- /dev/null +++ b/resources/uk/gov/hmcts/pipeline/yarn/yarnv4audit.py @@ -0,0 +1,163 @@ +import json +from dataclasses import dataclass, asdict +import subprocess +import os + +@dataclass +class Vulnerability: + name: str + id: str + severity: str + vulnerable_versions: str + issue: str + url: str + tree_versions: list[str] + dependents: list[str] + + def to_json(self): + return asdict(self) + + def format_vulnerability(self) -> str: + return ( + f"├─ {self.name}\n" + f"│ ├─ ID: {self.id if self.id is not None else 'N/A'}\n" + f"│ ├─ URL: {self.url if self.url is not None else 'N/A'}\n" + f"│ ├─ Issue: {self.issue if self.issue is not None else 'N/A'}\n" + f"│ ├─ Severity: {self.severity if self.severity is not None else 'N/A'}\n" + f"│ ├─ Vulnerable Versions: {self.vulnerable_versions if self.vulnerable_versions is not None else 'N/A'}\n" + f"│ ├─ Tree Versions: {', '.join(self.tree_versions) if self.tree_versions else 'N/A'}\n" + f"│ └─ Dependents: {', '.join(self.dependents) if self.dependents else 'N/A'}" + ) + + def __eq__(self, other): + if isinstance(other, Vulnerability): + return self.id == other.id + # Compare based on id only - allows us to handle situations + # where the other text of the vulnerability changes over time + return False + +def run_audit_command(): + command = ['yarn', 'npm', 'audit', '--recursive', '--environment', 'production', '--json'] + result = subprocess.run(command, capture_output=True, text=True) + # if any errors, print them and exit + if result.stderr != '': + print("Error running command `yarn npm audit --recursive --environment production --json` - Please raise a request in #platops-help. Error: ", repr(result.stderr)) + exit(1) + else: + return result.stdout, result.returncode + +def validate_audit_response(response, exit_code): + if exit_code == 0: + print("No vulnerabilities found, nice work!") + exit(0) + list_of_issues = split_and_strip_json_blocks(response) + print("Found", len(list_of_issues), "vulnerabilities") + if check_json_keys(list_of_issues): + vulnerabilities = build_list_of_issues(list_of_issues) + return vulnerabilities + else: + print("Error parsing JSON returned from audit endpoint - please raise a request in #platops-help") + exit(1) + +def split_and_strip_json_blocks(json_block): + stripped_response = json_block.strip() + list_of_issues = stripped_response.split('\n') + return list_of_issues + +def check_yarn_audit_known_issues(): + if os.path.exists('yarn-audit-known-issues'): + print("Found yarn-audit-known-issues file - checking for suppressed vulnerabilities") + with open('yarn-audit-known-issues') as f: + known_issues_file_content = f.read() + known_issues_stripped = split_and_strip_json_blocks(known_issues_file_content) + if check_json_keys(known_issues_stripped): + known_issues_list = build_list_of_issues(known_issues_stripped) + return known_issues_list + else : + print("Error parsing JSON in your yarn-audit-known-issues file - delete the file and use the following command:") + print("`yarn npm audit --recursive --environment production --json > yarn-audit-known-issues`") + exit(1) + else: + return [] + +def check_json_keys(json_blocks): + try: + for block in json_blocks: + data = json.loads(block) + # Check if both 'value' and 'children' keys are in the JSON block (yarn v4 schema) + if "value" not in data or "children" not in data: + return False + return True + except json.JSONDecodeError: + return False + +def combine_suppressions_and_vulnerabilities(suppressions, vulnerabilities): + # comparison logic is based on the ID of the vulnerability - if the ID is the same, we assume it's the same + # vulnerability + unsuppressed_vulnerabilities = [item for item in vulnerabilities if item not in suppressions] + unneeded_suppressions = [item for item in suppressions if item not in vulnerabilities] + suppressed_active_vulnerabilities = [item for item in vulnerabilities if item in suppressions] + return unsuppressed_vulnerabilities, unneeded_suppressions, suppressed_active_vulnerabilities + + +def build_list_of_issues(json_blocks): + """ + Creates a list of Vulnerability objects from a list of JSON blocks. + Each JSON block is parsed once and then used to construct a Vulnerability object. + + :param json_blocks: A list of JSON strings. + :return: A list of Vulnerability objects. + """ + vulnerabilities = [] + for block in json_blocks: + parsed_data = json.loads(block) + vuln = Vulnerability( + name=parsed_data.get('value'), + id=parsed_data.get('children', {}).get('ID'), + issue=parsed_data.get('children', {}).get('Issue'), + url=parsed_data.get('children', {}).get('URL'), + severity=parsed_data.get('children', {}).get('Severity'), + vulnerable_versions=parsed_data.get('children', {}).get('Vulnerable Versions'), + tree_versions=parsed_data.get('children', {}).get('Tree Versions', []), + dependents=parsed_data.get('children', {}).get('Dependents', []) + ) + vulnerabilities.append(vuln) + return vulnerabilities + +def print_vulnerabilities(vulnerabilities): + print("Found", len(vulnerabilities), "active vulnerabilities, please fix these before pushing again.") + for vulnerability in vulnerabilities: + print(vulnerability.format_vulnerability()) + +def print_suppressions(suppressions): + print("Found", len(suppressions), "unnecessary suppression(s), please check these are still needed." + "If not, please remove them from your yarn-audit-known-issues file") + for suppression in suppressions: + print(suppression.format_vulnerability()) + +def decide_what_to_print(unsuppressed_vulnerabilities, unneeded_suppressions): + if len(unsuppressed_vulnerabilities) == 0 and len(unneeded_suppressions) == 0: + print("All vulnerabilities are suppressed and there are no unneeded_suppressions - no action required, nice work!") + return + if len(unsuppressed_vulnerabilities) > 0: + print_vulnerabilities(unsuppressed_vulnerabilities) + return + if len(unneeded_suppressions) > 0: + print_suppressions(unneeded_suppressions) + +def build_parent_json_for_cosmosDB(vulnerabilities, suppressions): + if os.environ.get('CI') != 'true': + print("Not running in CI, skipping parent JSON block") + parent_block = {"suppressed_vulnerabilities": [s.to_json() for s in suppressions], + "unsuppressed_vulnerabilities": [v.to_json() for v in vulnerabilities]} + with open("audit-v4-cosmosdb-output", "w") as file: + file.write(json.dumps(parent_block, indent=2)) + +audit_output, return_code = run_audit_command() +vulnerabilities = validate_audit_response(audit_output, return_code) +suppressions = check_yarn_audit_known_issues() +unsuppressed_vulnerabilities, unneeded_suppressions, suppressed_active_vulnerabilities = combine_suppressions_and_vulnerabilities(suppressions, vulnerabilities) +decide_what_to_print(unsuppressed_vulnerabilities, unneeded_suppressions) +build_parent_json_for_cosmosDB(unsuppressed_vulnerabilities, suppressed_active_vulnerabilities) +if len(unsuppressed_vulnerabilities) > 0: + exit(1) diff --git a/src/uk/gov/hmcts/contino/YarnBuilder.groovy b/src/uk/gov/hmcts/contino/YarnBuilder.groovy index c200edb2c5..644e596af7 100644 --- a/src/uk/gov/hmcts/contino/YarnBuilder.groovy +++ b/src/uk/gov/hmcts/contino/YarnBuilder.groovy @@ -123,9 +123,55 @@ class YarnBuilder extends AbstractBuilder { } } - def securityCheck() { + def yarnVersionCheck() { + def major = 0 + def minor = 0 + def patch = 0 + try { steps.sh """ + jq -r '.packageManager' package.json | sed 's/yarn@//' > yarn_version + """ + + def versionString = steps.readFile('yarn_version').trim() + steps.println(versionString) + + def parts = versionString.split("\\.") + major = parts.size() > 0 ? parts[0].toInteger() : major + minor = parts.size() > 1 ? parts[1].toInteger() : minor + patch = parts.size() > 2 ? parts[2].toInteger() : patch + + if (major < 3) { + steps.println("Version is less than 3.0.0. This needs updating as we only support 3.0.x upwards.") + return "<3" + } else if (major == 3) { + steps.println("v3 detected - continuing") + return "v3" + } else if (major == 4) { + if (minor == 0 && patch == 0) { + steps.println("v4.0.0 detected. You will need to upgrade yarn to at least v4.0.1, as 4.0.0 has an unsupported audit format.") + return "v4.0.0" + } else { + steps.println("Version is v4.0.1 or higher.") + return "v4.0.1+" + } + } else { + steps.println("Version is greater than 4.0.0. Using the updated configuration for yarn npm audit.") + return "v4+" + } + } catch (Exception e) { + steps.echo e.getMessage() + return "error" + } + } + + + def securityCheck() { + + def version = yarnVersionCheck() + if (version == 'v3') { + try { + steps.sh """ set +ex export NVM_DIR='/home/jenkinsssh/.nvm' # TODO get home from variable . /opt/nvm/nvm.sh || true @@ -133,29 +179,58 @@ class YarnBuilder extends AbstractBuilder { set -ex """ - corepackEnable() - steps.writeFile(file: 'yarn-audit-with-suppressions.sh', text: steps.libraryResource('uk/gov/hmcts/pipeline/yarn/yarn-audit-with-suppressions.sh')) - steps.writeFile(file: 'prettyPrintAudit.sh', text: steps.libraryResource('uk/gov/hmcts/pipeline/yarn/prettyPrintAudit.sh')) + corepackEnable() + steps.writeFile(file: 'yarn-audit-with-suppressions.sh', text: steps.libraryResource('uk/gov/hmcts/pipeline/yarn/yarn-audit-with-suppressions.sh')) + steps.writeFile(file: 'prettyPrintAudit.sh', text: steps.libraryResource('uk/gov/hmcts/pipeline/yarn/prettyPrintAudit.sh')) - steps.sh """ + steps.sh """ export PATH=\$HOME/.local/bin:\$PATH chmod +x yarn-audit-with-suppressions.sh ./yarn-audit-with-suppressions.sh """ - } finally { - steps.sh """ + } finally { + steps.sh """ cat yarn-audit-result | jq -c '. | {type: "auditSummary", data: .metadata}' > yarn-audit-issues-result-summary cat yarn-audit-result | jq -cr '.advisories| to_entries[] | {"type": "auditAdvisory", "data": { "advisory": .value }}' >> yarn-audit-issues-advisories cat yarn-audit-issues-result-summary yarn-audit-issues-advisories > yarn-audit-issues-result """ - String issues = steps.readFile('yarn-audit-issues-result') - String knownIssues = null - if (steps.fileExists(CVE_KNOWN_ISSUES_FILE_PATH)) { - knownIssues = steps.readFile(CVE_KNOWN_ISSUES_FILE_PATH) + String issues = steps.readFile('yarn-audit-issues-result') + String knownIssues = null + if (steps.fileExists(CVE_KNOWN_ISSUES_FILE_PATH)) { + knownIssues = steps.readFile(CVE_KNOWN_ISSUES_FILE_PATH) + } + def cveReport = prepareCVEReport(issues, knownIssues) + new CVEPublisher(steps) + .publishCVEReport('node', cveReport) } - def cveReport = prepareCVEReport(issues, knownIssues) - new CVEPublisher(steps) - .publishCVEReport('node', cveReport) + } + else if (version == 'v4.0.0') { + steps.println("Unsupported version, please upgrade to 4.0.1 or higher") +// todo : handle + } + else if (version == '<3') { + steps.println("Version too low") +// todo : handle + + } + else if (version == 'v4.0.1+') { + securityCheckYarnV4() + } + } + + def securityCheckYarnV4() { + try { + steps.writeFile(file: 'yarnv4audit.py', text: steps.libraryResource('uk/gov/hmcts/pipeline/yarn/yarnv4audit.py')) + steps.sh """ + export PATH=\$HOME/.local/bin:\$PATH + chmod +x yarnv4audit.py + python3.10 --version + python3.10 yarnv4audit.py + """ + } + finally { + LinkedHashMap CVEReport = prepareCVEReportYarn4('audit-v4-cosmosdb-output') + this.steps.echo CVEReport.toString() } } @@ -176,6 +251,14 @@ class YarnBuilder extends AbstractBuilder { } } + def prepareCVEReportYarn4(String inputFilePath) { + String jsonString = new File(inputFilePath) + JsonSlurper slurper = new JsonSlurper() + LinkedHashMap parsedData = slurper.parseText(jsonString) + return parsedData + } + + def prepareCVEReport(String issues, String knownIssues) { def jsonSlurper = new JsonSlurper()