Add validation script + tests for quay.io SNAFU

Ref: https://github.com/containers/podman/discussions slash 19796

Also modify testlib.sh slightly so it doesn't always print the full path
of the command being executed - it's hard on the eyes.

Signed-off-by: Chris Evich <[email protected]>

common/test/testlib.sh

@@ -101,19 +101,19 @@ test_cmd() {
     fi
 
     if [[ -n "$e_exit" ]] && [[ $e_exit -ne $a_exit ]]; then
-        _test_report "Expected exit-code $e_exit but received $a_exit while executing $1" "1" "$a_out_f"
+        _test_report "Expected exit-code $e_exit but received $a_exit while executing $(basename $1)" "1" "$a_out_f"
     elif [[ -z "$e_out_re" ]] && [[ -n "$(<$a_out_f)" ]]; then
         _test_report "Expecting no output from $*" "1" "$a_out_f"
     elif [[ -n "$e_out_re" ]]; then
         if ((TEST_DEBUG)); then
             echo "Received $(wc -l $a_out_f | awk '{print $1}') output lines of $(wc -c $a_out_f | awk '{print $1}') bytes total"
         fi
         if grep -Eq "$e_out_re" "${a_out_f}.oneline"; then
-            _test_report "Command $1 exited as expected with expected output" "0" "$a_out_f"
+            _test_report "Command $(basename $1) exited as expected with expected output" "0" "$a_out_f"
         else
             _test_report "Expecting regex '$e_out_re' match to (whitespace-squashed) output" "1" "$a_out_f"
         fi
     else # Pass
-        _test_report "Command $1 exited as expected ($a_exit)" "0" "$a_out_f"
+        _test_report "Command $(basename $1) exited as expected ($a_exit)" "0" "$a_out_f"
     fi
 }

validate_image_cirrus/requirements.txt

@@ -0,0 +1,4 @@
+PyYAML
+gql[requests]
+requests
+urllib3

validate_image_cirrus/test/run_all_tests.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Convenience script for executing all tests
+
+set -e
+
+cd $(dirname $0)
+for testscript in test???-*.sh; do
+    echo -e "\nExecuting $testscript..." > /dev/stderr
+    ./$testscript
+done

validate_image_cirrus/test/testbin-validate_image_cirrus.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+# Integration tests for validate_image_cirrus.py
+
+BIN_DIR=$(realpath "$(dirname ${BASH_SOURCE[0]})/..")
+BIN="$BIN_DIR/validate_image_cirrus.py"
+REQS=$(realpath "$BIN_DIR/requirements.txt")
+source $(dirname ${BASH_SOURCE[0]})/testlib.sh || exit 1
+
+# A valid manifest-list FQIN to test against
+VALID_ML_FQIN="quay.io/podman/stable:v3.4.4"
+# A valid regular/single-arch image to test against
+# must not clash with VALID_ML_FQIN
+VALID_FQIN="quay.io/podman/stable:v2.1.1"
+
+# Avoid needing to re-install all deps every test-run, re-use
+# python venv if requirements file is unchanged.  Note: This
+# will leak tempdirs when file does change between multiple runs.
+REQ_SHA=$(cat $REQS | sha256sum | awk '{print $1}')
+VENV_DIR="/var/tmp/validate_image_cirrus_venv_$REQ_SHA"
+if [[ ! -d "$VENV_DIR" ]] || [[ ! -r "$VENV_DIR/good_to_go" ]]; then
+  rm -f "$VENV_DIR/good_to_go"
+  virtualenv "$VENV_DIR"
+  source $VENV_DIR/bin/activate
+  pip3 install --upgrade pip
+  pip3 install --upgrade -r "$REQS"
+  # This can take a long time to transfer, cache a copy here
+  SKOPEO="skopeo sync -a --scoped --preserve-digests -s docker -d dir"
+  $SKOPEO "$VALID_ML_FQIN" "$VENV_DIR"
+  $SKOPEO "$VALID_FQIN" "$VENV_DIR"
+  touch "$VENV_DIR/good_to_go"
+  echo ""
+else
+    source $VENV_DIR/bin/activate
+fi
+
+# /tmp may be a ramdisk w/ limited space available
+TMP=$(mktemp -p '/var/tmp' -d "validate_image_cirrus_tmp_XXXXX")
+trap "rm -rf $TMP" EXIT
+
+test_cmd "The script is runable and --help works" \
+  0 "Show internal debugging/processing details" \
+  $BIN --help
+
+test_cmd "The script returns error when no argument given" \
+  2 "usage:.+error:.+the following arguments are required" \
+  $BIN
+
+# The simple test image is missing otherwise required labels
+# workaround this for general testing purposes.
+RLARG="-l name,license,vendor,version"
+
+# Confirm all test images validate
+test_cmd "Script exits cleanly on all pulled known-clean test sources." \
+  0 "Sanity: PASS" \
+  $BIN -v $RLARG "$VENV_DIR/$VALID_FQIN" "$VENV_DIR/$VALID_ML_FQIN"
+
+# Confirm both regular image and manifest-list
+for fqin in $VALID_FQIN $VALID_ML_FQIN; do
+  echo -e "\n##### Testing '$fqin' #####\n"
+
+  # Confirm image validates before messing with contents (on a copy)
+  test_cmd "Script exits cleanly on pulled known-clean test source for '$fqin'" \
+    0 "Sanity: PASS" \
+    $BIN -v $RLARG "$VENV_DIR/$fqin"
+
+  test_cmd "Script flags unexpected platforms" \
+    10 "Expected platforms: FAIL" \
+    $BIN -v $RLARG -p foo/bar,bar/foo "$VENV_DIR/$fqin"
+
+  NO_TAG_FQIN_DIR="$TMP/${fqin%:*}"
+  NO_TAG_PARENT_DIR="$TMP/$(dirname $fqin)/"
+  mkdir -p "$NO_TAG_PARENT_DIR"
+  cp -a "$VENV_DIR/$fqin" "$NO_TAG_PARENT_DIR"
+  mv "$TMP/$fqin" "$NO_TAG_FQIN_DIR"
+  test_cmd "A fqin dir w/ missing tag is rejected" \
+    9 "Validation results.+FAIL.+missing ':'" \
+    $BIN -v $NO_TAG_FQIN_DIR
+
+  NO_REG_PFX="$TMP/foo/bar:latest"
+  mkdir -p "$NO_REG_PFX"
+  mv "$NO_TAG_FQIN_DIR"/* "$NO_REG_PFX/"
+  test_cmd "A fqin dir missing the reg-server is rejected" \
+    9 "Validation results.+Missing.+from path-derived FQIN" \
+    $BIN -v "$NO_REG_PFX"
+
+  BAD_MANIFEST_FQIN_DIR="$TMP/${fqin}"
+  cp -a "$VENV_DIR/$fqin" "$(dirname $BAD_MANIFEST_FQIN_DIR)"
+  echo "$RANDOM$RANDOM}}[" >> "$BAD_MANIFEST_FQIN_DIR/manifest.json"
+  test_cmd "A fqin dir w/ corrupt manifest.json is rejected" \
+    9 "Validation results.+Failed to parse" \
+    $BIN -v "$BAD_MANIFEST_FQIN_DIR"
+
+  NO_MANIFEST_FQIN_DIR="$TMP/${fqin}"
+  cp -a "$VENV_DIR/$fqin" "$(dirname $NO_MANIFEST_FQIN_DIR)"
+  rm "$NO_MANIFEST_FQIN_DIR/manifest.json"
+  test_cmd "A fqin dir missing a manifest.json is rejected" \
+    9 "Validation results.+No manifest\\.json" \
+    $BIN -v "$NO_MANIFEST_FQIN_DIR"
+done
+
+##### Tests which only pertain to manifest-list
+
+test_cmd "Default arguments work with manifest-list test image" \
+  0 "Sanity: PASS.+Expected labels: PASS.+Expected platforms: PASS" \
+  $BIN "$VENV_DIR/$VALID_ML_FQIN"
+
+test_cmd "Script flags missing manifest-list labels" \
+  10 "Missing labels: \['barfoo', 'foobar', 'snafu'\].+Expected labels: FAIL" \
+  $BIN -v -l name,foobar,license,barfoo,vendor,version,snafu "$VENV_DIR/$VALID_ML_FQIN"
+
+test_cmd "Script flags unexpected manifest-list platforms" \
+  10 "DEBUG: Missing platforms: \['bar/foo', 'foo/bar', 'sna/fu'\].+Expected platforms: FAIL" \
+  $BIN -v -p foo/bar,bar/foo,linux/s390x,sna/fu "$VENV_DIR/$VALID_ML_FQIN"
+
+# Must be last call
+exit_with_status

validate_image_cirrus/test/testlib.sh

@@ -0,0 +1 @@
+../../common/test/testlib.sh