diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index 8e7842e95..dc44d93e8 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -38,7 +38,12 @@ on: type: boolean required: false default: true - push-dirty: + sbom: + description: Generate SBOM? + type: boolean + required: false + default: true + push-critical: description: Push scanned images that have critical vulnerabilities? type: boolean required: false @@ -158,7 +163,7 @@ jobs: - name: Install Trivy run: | - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0 + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.62.1 - name: Install yq run: | @@ -252,14 +257,14 @@ jobs: run: if [ $(wc -l < ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images) -le 1 ]; then exit 1; fi - name: Scan built container images - run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro.name }}-${{ matrix.distro.release }} ${{ steps.write-kolla-tag.outputs.kolla-tag }} + run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro.name }}-${{ matrix.distro.release }} ${{ steps.write-kolla-tag.outputs.kolla-tag }} ${{ inputs.sbom && '--sbom'}} - name: Move image scan logs to output artifact run: mv image-scan-output image-build-logs/image-scan-output - - name: Fail if no images have passed scanning + - name: Fail if any images have critical vulnerabilities run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi - if: ${{ !inputs.push-dirty }} + if: ${{ !inputs.push-critical }} - name: Copy clean images to push-attempt-images list run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt @@ -269,13 +274,13 @@ jobs: # This should be reverted when it's decided to filter high level CVEs as well. - name: Append dirty images to push list run: | - cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt + cat image-build-logs/image-scan-output/high-images.txt >> image-build-logs/push-attempt-images.txt if: ${{ inputs.push }} - name: Append images with critical vulnerabilities to push list run: | cat image-build-logs/image-scan-output/critical-images.txt >> image-build-logs/push-attempt-images.txt - if: ${{ inputs.push && inputs.push-dirty }} + if: ${{ inputs.push && inputs.push-critical }} - name: Push images run: | @@ -324,12 +329,12 @@ jobs: # This can be used again instead of "Fail when critical vulnerabilities are found" when it's # decided to fail the job on detecting high CVEs as well. # - name: Fail when images failed scanning - # run: if [ $(wc -l < image-build-logs/image-scan-output/dirty-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/dirty-images.txt && exit 1; fi - # if: ${{ !inputs.push-dirty && !cancelled() }} + # run: if [ $(wc -l < image-build-logs/image-scan-output/high-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/high-images.txt && exit 1; fi + # if: ${{ !inputs.push-critical && !cancelled() }} - name: Fail when critical vulnerabilities are found run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi - if: ${{ !inputs.push-dirty && !cancelled() }} + if: ${{ !inputs.push-critical && !cancelled() }} # NOTE(mgoddard): Trigger another CI workflow in the # stackhpc-release-train repository. diff --git a/tools/scan-images.sh b/tools/scan-images.sh index d71aec4b2..385e152db 100755 --- a/tools/scan-images.sh +++ b/tools/scan-images.sh @@ -1,44 +1,55 @@ #!/usr/bin/env bash set -eo pipefail -# Check correct usage -if [[ ! $2 ]]; then - echo "Usage: scan-images.sh " - exit 2 -fi - -set -u +# Global variables +scan_common_args=" \ + --exit-code 1 \ + --scanners vuln \ + --format json \ + --severity HIGH,CRITICAL \ + --ignore-unfixed \ + --db-repository ghcr.io/aquasecurity/trivy-db:2 \ + --db-repository public.ecr.aws/aquasecurity/trivy-db \ + --java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \ + --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db " -# Check that trivy is installed -if ! trivy --version; then - echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.49.1' -fi - -# Clear any previous outputs -rm -rf image-scan-output +# Print usage instructions and error with wrong inputs +usage() { + echo "Usage: scan-images.sh [--sbom]" + exit 2 +} -# Make a fresh output directory -mkdir -p image-scan-output +# Check dependencies are installed, print installation instructions otherwise +check_deps_installed() { + if ! trivy --version > /dev/null; then + echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.62.1' + exit 1 + fi + if ! yq --version > /dev/null; then + echo 'Please install yq: sudo dnf/apt install yq' + exit 1 + fi +} -# Get built container images -docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2" > $1-scanned-container-images.txt +# Prepare output files +file_prep() { + rm -rf image-scan-output + mkdir -p image-scan-output + touch image-scan-output/clean-images.txt image-scan-output/high-images.txt image-scan-output/critical-images.txt +} -# Make a file of imagename:tag -images=$(grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d:) +# Gather image lists +get_images() { + docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2" > $1-scanned-container-images.txt + grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d: +} -# Ensure output files exist -touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt image-scan-output/critical-images.txt +# Generate ignored vulnerabilities file +generate_trivy_ignore() { + local imagename=$1 + local global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) + local image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) -# If Trivy detects no vulnerabilities, add the image name to clean-images.txt. -# If there are vulnerabilities detected, add it to dirty-images.txt and -# generate a csv summary -# If the image contains at least one critical vulnerabilities, add it to -# critical-images.txt -for image in $images; do - filename=$(basename $image | sed 's/:/\./g') - imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g') - global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml) - image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml) touch .trivyignore for vulnerability in $global_vulnerabilities; do echo $vulnerability >> .trivyignore @@ -46,57 +57,112 @@ for image in $images; do for vulnerability in $image_vulnerabilities; do echo $vulnerability >> .trivyignore done - if $(trivy image \ - --quiet \ - --exit-code 1 \ - --scanners vuln \ - --format json \ - --severity HIGH,CRITICAL \ - --output image-scan-output/${filename}.json \ - --ignore-unfixed \ - --db-repository ghcr.io/aquasecurity/trivy-db:2 \ - --db-repository public.ecr.aws/aquasecurity/trivy-db \ - --java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \ - --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db \ - $image); then - # Clean up the output file for any images with no vulnerabilities - rm -f image-scan-output/${filename}.json - - # Add the image to the clean list +} + +# Put results into CSV +generate_summary_csv() { + local imagename=$1 + local filename=$2 + + echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${imagename}/${filename}-summary.csv + + jq -r '.Results[] + | select(.Vulnerabilities) + | .Vulnerabilities + | map(select(.PkgName | test("kernel") | not )) + | group_by(.VulnerabilityID) + | map( + [ + (map(.PkgName) | unique | join(";")), + (map(.PkgPath | select( . != null )) | join(";")), + .[0].PkgID, + .[0].VulnerabilityID, + .[0].FixedVersion, + .[0].PrimaryURL, + .[0].Severity + ] + ) + | .[] + | @csv' image-scan-output/${imagename}/${filename}-scan.json >> image-scan-output/${imagename}/${filename}-summary.csv +} + +# Categorise images based on severity +categorise_image() { + local imagename=$1 + local filename=$2 + local image=$3 + + if [ $(grep "CRITICAL" image-scan-output/${imagename}/${filename}-summary.csv -c) -gt 0 ]; then + echo "${image}" >> image-scan-output/critical-images.txt + else + echo "${image}" >> image-scan-output/high-images.txt + fi +} + +# Generate SBOM, return correct scan command for SBOM +generate_sbom() { + local imagename=$1 + local filename=$2 + local image=$3 + trivy image \ + --format spdx-json \ + --output image-scan-output/${imagename}/${filename}-sbom.json \ + $image > /dev/null 2>&1 + echo "trivy sbom $scan_common_args \ + --output image-scan-output/${imagename}/${filename}-scan.json \ + image-scan-output/${imagename}/${filename}-sbom.json" +} + +# Scan images, generate SBOMs if requested +scan_image() { + local image=$1 + local filename=$(basename $image | sed 's/:/\./g') + local imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g') + + mkdir -p image-scan-output/$imagename + generate_trivy_ignore $imagename + + # If SBOM is required, generate it first and scan the results, otherwise we + # scan the image directly. + if $generate_sbom; then + echo "Generating SBOM for $imagename" + scan_command=$(generate_sbom $imagename $filename $image) + else + scan_command="trivy image $scan_common_args \ + --output image-scan-output/${imagename}/${filename}-scan.json $image" + fi + + # Run scan against image or SBOM, format output. If no results, delete files. + echo "Scanning $imagename for vulnerabilities" + if $scan_command > /dev/null 2>&1; then + rm -f image-scan-output/${imagename}/${filename}-scan.json echo "${image}" >> image-scan-output/clean-images.txt else + generate_summary_csv $imagename $filename + categorise_image $imagename $filename $image + fi +} - # Write a header for the summary CSV - echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}.summary.csv - - # Write the summary CSV data - jq -r '.Results[] - | select(.Vulnerabilities) - | .Vulnerabilities - # Ignore packages with "kernel" in the PkgName - | map(select(.PkgName | test("kernel") | not )) - | group_by(.VulnerabilityID) - | map( - [ - (map(.PkgName) | unique | join(";")), - (map(.PkgPath | select( . != null )) | join(";")), - .[0].PkgID, - .[0].VulnerabilityID, - .[0].FixedVersion, - .[0].PrimaryURL, - .[0].Severity - ] - ) - | .[] - | @csv' image-scan-output/${filename}.json >> image-scan-output/${filename}.summary.csv - - if [ $(grep "CRITICAL" image-scan-output/${filename}.summary.csv -c) -gt 0 ]; then - # If the image contains critical vulnerabilities, add the image to critical list - echo "${image}" >> image-scan-output/critical-images.txt - else - # Otherwise, add the image to the dirty list - echo "${image}" >> image-scan-output/dirty-images.txt - fi +# Main function +main() { + if [[ ! $2 ]]; then + usage fi - rm .trivyignore -done + + generate_sbom=false + if [[ "$3" == "--sbom" ]]; then + generate_sbom=true + fi + + set -u + + check_deps_installed + file_prep + + images=$(get_images $1 $2) + for image in $images; do + scan_image $image + done +} + +main "$@"