Skip to content

Commit feedec5

Browse files
Alex-WelshAlex-Welsh
committed
CI: Refactor container image scanning script
1 parent d84882b commit feedec5

File tree

2 files changed

+141
-92
lines changed

2 files changed

+141
-92
lines changed

‎.github/workflows/stackhpc-container-image-build.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ jobs:
150150
151151
- name: Install Trivy
152152
run: |
153-
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0
153+
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.62.1
154154
155155
- name: Install yq
156156
run: |

‎tools/scan-images.sh

Lines changed: 140 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -1,111 +1,160 @@
11
#!/usr/bin/env bash
22
set -eo pipefail
33

4-
# Check correct usage
5-
if [[ ! $2 ]]; then
6-
echo "Usage: scan-images.sh <os-distribution> <image-tag>"
7-
exit 2
8-
fi
9-
10-
set -u
4+
# Global variables
5+
scan_common_args=" \
6+
--exit-code 1 \
7+
--scanners vuln \
8+
--format json \
9+
--severity HIGH,CRITICAL \
10+
--ignore-unfixed \
11+
--db-repository ghcr.io/aquasecurity/trivy-db:2 \
12+
--db-repository public.ecr.aws/aquasecurity/trivy-db \
13+
--java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \
14+
--java-db-repository public.ecr.aws/aquasecurity/trivy-java-db "
1115

12-
# Check that trivy is installed
13-
if ! trivy --version; then
14-
echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.49.1'
15-
fi
16-
17-
# Clear any previous outputs
18-
rm -rf image-scan-output
16+
# Print usage instructions and error with wrong inputs
17+
usage() {
18+
echo "Usage: scan-images.sh <os-distribution> <image-tag> [--sbom]"
19+
exit 2
20+
}
1921

20-
# Make fresh output directories
21-
mkdir -p image-scan-output image-sboms
22+
# Check dependencies are installed, print installation instructions otherwise
23+
check_deps_installed() {
24+
if ! trivy --version > /dev/null; then
25+
echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.62.1'
26+
exit 1
27+
fi
28+
if ! yq --version > /dev/null; then
29+
echo 'Please install yq: sudo dnf/apt install yq'
30+
exit 1
31+
fi
32+
}
2233

23-
# Get built container images
24-
docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2" > $1-scanned-container-images.txt
34+
# Prepare output files
35+
file_prep() {
36+
rm -rf image-scan-output
37+
mkdir -p image-scan-output
38+
touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt image-scan-output/critical-images.txt
39+
}
2540

26-
# Make a file of imagename:tag
27-
images=$(grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d:)
41+
# Gather image lists
42+
get_images() {
43+
docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2" > $1-scanned-container-images.txt
44+
grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d:
45+
}
2846

29-
# Ensure output files exist
30-
touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt image-scan-output/critical-images.txt
47+
# Generate ignored vulnerabilities file
48+
generate_trivy_ignore() {
49+
local imagename=$1
50+
local global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
51+
local image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
3152

32-
# If Trivy detects no vulnerabilities, add the image name to clean-images.txt.
33-
# If there are vulnerabilities detected, add it to dirty-images.txt and
34-
# generate a csv summary
35-
# If the image contains at least one critical vulnerabilities, add it to
36-
# critical-images.txt
37-
for image in $images; do
38-
filename=$(basename $image | sed 's/:/\./g')
39-
imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g')
40-
global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml)
41-
image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml)
4253
touch .trivyignore
43-
mkdir -p image-scan-output/$filename
4454
for vulnerability in $global_vulnerabilities; do
4555
echo $vulnerability >> .trivyignore
4656
done
4757
for vulnerability in $image_vulnerabilities; do
4858
echo $vulnerability >> .trivyignore
4959
done
50-
if $(trivy image \
51-
--quiet \
52-
--exit-code 1 \
53-
--scanners vuln \
54-
--format json \
55-
--severity HIGH,CRITICAL \
56-
--output image-scan-output/${filename}/${filename}.json \
57-
--ignore-unfixed \
58-
--db-repository ghcr.io/aquasecurity/trivy-db:2 \
59-
--db-repository public.ecr.aws/aquasecurity/trivy-db \
60-
--java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \
61-
--java-db-repository public.ecr.aws/aquasecurity/trivy-java-db \
62-
$image); then
63-
# Clean up the output file for any images with no vulnerabilities
64-
rm -f image-scan-output/${filename}/${filename}.json
65-
66-
# Add the image to the clean list
60+
}
61+
62+
# Put results into CSV
63+
generate_summary_csv() {
64+
local imagename=$1
65+
local filename=$2
66+
67+
echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${imagename}/${filename}-summary.csv
68+
69+
jq -r '.Results[]
70+
| select(.Vulnerabilities)
71+
| .Vulnerabilities
72+
| map(select(.PkgName | test("kernel") | not ))
73+
| group_by(.VulnerabilityID)
74+
| map(
75+
[
76+
(map(.PkgName) | unique | join(";")),
77+
(map(.PkgPath | select( . != null )) | join(";")),
78+
.[0].PkgID,
79+
.[0].VulnerabilityID,
80+
.[0].FixedVersion,
81+
.[0].PrimaryURL,
82+
.[0].Severity
83+
]
84+
)
85+
| .[]
86+
| @csv' image-scan-output/${imagename}/${filename}-scan.json >> image-scan-output/${imagename}/${filename}-summary.csv
87+
}
88+
89+
# Categorise images based on severity
90+
categorise_image() {
91+
local imagename=$1
92+
local filename=$2
93+
local image=$3
94+
95+
if [ $(grep "CRITICAL" image-scan-output/${imagename}/${filename}-summary.csv -c) -gt 0 ]; then
96+
echo "${image}" >> image-scan-output/critical-images.txt
97+
else
98+
echo "${image}" >> image-scan-output/high-images.txt
99+
fi
100+
}
101+
102+
# Scan images, generate SBOMs if requested
103+
scan_image() {
104+
local image=$1
105+
local filename=$(basename $image | sed 's/:/\./g')
106+
local imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g')
107+
108+
mkdir -p image-scan-output/$imagename
109+
generate_trivy_ignore $imagename
110+
111+
echo "Scanning $imagename"
112+
113+
# If SBOM is required, generate that first, then generate scan results from it
114+
if $generate_sbom; then
115+
trivy image \
116+
--format spdx-json \
117+
--output image-scan-output/${imagename}/${filename}-sbom.json \
118+
$image
119+
scan_command="trivy sbom $scan_common_args \
120+
--output image-scan-output/${imagename}/${filename}-scan.json \
121+
image-scan-output/${imagename}/${filename}-sbom.json"
122+
else
123+
scan_command="trivy image $scan_common_args \
124+
--output image-scan-output/${imagename}/${filename}-scan.json $image"
125+
fi
126+
echo "scan command"
127+
echo "$scan_command"
128+
# Run scan, against image or SBOM. If no results, delete files.
129+
if $scan_command; then
130+
rm -f image-scan-output/${imagename}/${filename}-scan.json
67131
echo "${image}" >> image-scan-output/clean-images.txt
68132
else
133+
generate_summary_csv $imagename $filename
134+
categorise_image $imagename $filename $image
135+
fi
136+
}
69137

70-
# Write a header for the summary CSV
71-
echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}/${filename}.summary.csv
72-
73-
# Write the summary CSV data
74-
jq -r '.Results[]
75-
| select(.Vulnerabilities)
76-
| .Vulnerabilities
77-
# Ignore packages with "kernel" in the PkgName
78-
| map(select(.PkgName | test("kernel") | not ))
79-
| group_by(.VulnerabilityID)
80-
| map(
81-
[
82-
(map(.PkgName) | unique | join(";")),
83-
(map(.PkgPath | select( . != null )) | join(";")),
84-
.[0].PkgID,
85-
.[0].VulnerabilityID,
86-
.[0].FixedVersion,
87-
.[0].PrimaryURL,
88-
.[0].Severity
89-
]
90-
)
91-
| .[]
92-
| @csv' image-scan-output/${filename}/${filename}.json >> image-scan-output/${filename}/${filename}.summary.csv
93-
94-
if [ $(grep "CRITICAL" image-scan-output/${filename}/${filename}.summary.csv -c) -gt 0 ]; then
95-
# If the image contains critical vulnerabilities, add the image to critical list
96-
echo "${image}" >> image-scan-output/critical-images.txt
97-
else
98-
# Otherwise, add the image to the dirty list
99-
echo "${image}" >> image-scan-output/dirty-images.txt
100-
fi
138+
# Main function
139+
main() {
140+
if [[ ! $2 ]]; then
141+
usage
101142
fi
102-
trivy image \
103-
--quiet \
104-
--format spdx \
105-
--output image-scan-output/${filename}/${filename}-sbom.spdx \
106-
--db-repository ghcr.io/aquasecurity/trivy-db:2 \
107-
--db-repository public.ecr.aws/aquasecurity/trivy-db \
108-
--java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \
109-
--java-db-repository public.ecr.aws/aquasecurity/trivy-java-db \
110-
$image
111-
done
143+
144+
generate_sbom=false
145+
if [[ "$3" == "--sbom" ]]; then
146+
generate_sbom=true
147+
fi
148+
149+
set -u
150+
151+
check_deps_installed
152+
file_prep
153+
154+
images=$(get_images $1 $2)
155+
for image in $images; do
156+
scan_image $image
157+
done
158+
}
159+
160+
main "$@"

0 commit comments

Comments
 (0)