diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ffce21..d416f74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,23 @@ jobs: - run: rustup component add clippy - run: cargo clippy -- -D warnings + gnu-testsuite: + name: GNU test suite + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - run: cargo build --release + # do not fail, the report is merely informative (at least until all tests pass reliably) + - run: ./tests/run-upstream-testsuite.sh release || true + env: + TERM: xterm + - uses: actions/upload-artifact@v4 + with: + name: test-results.json + path: tests/test-results.json + - run: ./tests/print-test-results.sh tests/test-results.json + coverage: name: Code Coverage runs-on: ${{ matrix.job.os }} diff --git a/tests/print-test-results.sh b/tests/print-test-results.sh new file mode 100755 index 0000000..23136d6 --- /dev/null +++ b/tests/print-test-results.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Print the test results written to a JSON file by run-upstream-testsuite.sh +# in a markdown format. The printout includes the name of the test, the result, +# the URL to the test script and the contents of stdout and stderr. +# It can be used verbatim as the description when filing an issue for a test +# with an unexpected result. + +json="test-results.json" +[[ -n $1 ]] && json="$1" + +codeblock () { echo -e "\`\`\`\n$1\n\`\`\`"; } + +jq -c '.tests[]' "$json" | while read -r test +do + name=$(echo "$test" | jq -r '.test') + echo "# test: $name" + result=$(echo "$test" | jq -r '.result') + echo "result: $result" + url=$(echo "$test" | jq -r '.url') + echo "url: $url" + if [[ "$result" != "SKIP" ]] + then + stdout=$(echo "$test" | jq -r '.stdout' | base64 -d) + if [[ -n "$stdout" ]] + then + echo "## stdout" + codeblock "$stdout" + fi + stderr=$(echo "$test" | jq -r '.stderr' | base64 -d) + if [[ -n "$stderr" ]] + then + echo "## stderr" + codeblock "$stderr" + fi + fi + echo "" +done diff --git a/tests/run-upstream-testsuite.sh b/tests/run-upstream-testsuite.sh new file mode 100755 index 0000000..d85dfd6 --- /dev/null +++ b/tests/run-upstream-testsuite.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +# Run the GNU upstream test suite for diffutils against a local build of the +# Rust implementation, print out a summary of the test results, and writes a +# JSON file ('test-results.json') containing detailed information about the +# test run. + +# The JSON file contains metadata about the test run, and for each test the +# result as well as the contents of stdout, stderr, and of all the files +# written by the test script, if any (excluding subdirectories). + +# The script takes a shortcut to fetch only the test suite from the upstream +# repository and carefully avoids running the autotools machinery which is +# time-consuming and resource-intensive, and doesn't offer the option to not +# build the upstream binaries. As a consequence, the environment in which the +# tests are run might not match exactly that used when the upstream tests are +# run through the autotools. + +# By default it expects a release build of the diffutils binary, but a +# different build profile can be specified as an argument +# (e.g. 'dev' or 'test'). +# Unless overriden by the $TESTS environment variable, all tests in the test +# suite will be run. Tests targeting a command that is not yet implemented +# (e.g. cmp, diff3 or sdiff) are skipped. + +scriptpath=$(dirname "$(readlink -f "$0")") +rev=$(git rev-parse HEAD) + +# Allow passing a specific profile as parameter (default to "release") +profile="release" +[[ -n $1 ]] && profile="$1" + +# Verify that the diffutils binary was built for the requested profile +binary="$scriptpath/../target/$profile/diffutils" +if [[ ! -x "$binary" ]] +then + echo "Missing build for profile $profile" + exit 1 +fi + +# Work in a temporary directory +tempdir=$(mktemp -d) +cd "$tempdir" + +# Check out the upstream test suite +gitserver="https://git.savannah.gnu.org" +testsuite="$gitserver/git/diffutils.git" +echo "Fetching upstream test suite from $testsuite" +git clone -n --depth=1 --filter=tree:0 "$testsuite" &> /dev/null +cd diffutils +git sparse-checkout set --no-cone tests &> /dev/null +git checkout &> /dev/null +upstreamrev=$(git rev-parse HEAD) + +# Ensure that calling `diff` invokes the built `diffutils` binary instead of +# the upstream `diff` binary that is most likely installed on the system +mkdir src +cd src +ln -s "$binary" diff +cd ../tests + +if [[ -n "$TESTS" ]] +then + tests="$TESTS" +else + # Get a list of all upstream tests (default if $TESTS isn't set) + echo -e '\n\nprinttests:\n\t@echo "${TESTS}"' >> Makefile.am + tests=$(make -f Makefile.am printtests) +fi +total=$(echo "$tests" | wc -w) +echo "Running $total tests" +export LC_ALL=C +export KEEP=yes +exitcode=0 +timestamp=$(date -Iseconds) +urlroot="$gitserver/cgit/diffutils.git/tree/tests/" +passed=0 +failed=0 +skipped=0 +normal="$(tput sgr0)" +for test in $tests +do + result="FAIL" + url="$urlroot$test?id=$upstreamrev" + # Run only the tests that invoke `diff`, + # because other binaries aren't implemented yet + if ! grep -E -s -q "(cmp|diff3|sdiff)" "$test" + then + sh "$test" 1> stdout.txt 2> stderr.txt && result="PASS" || exitcode=1 + json+="{\"test\":\"$test\",\"result\":\"$result\"," + json+="\"url\":\"$url\"," + json+="\"stdout\":\"$(base64 -w0 < stdout.txt)\"," + json+="\"stderr\":\"$(base64 -w0 < stderr.txt)\"," + json+="\"files\":{" + cd gt-$test.* + # Note: this doesn't include the contents of subdirectories, + # but there isn't much value added in doing so + for file in * + do + [[ -f "$file" ]] && json+="\"$file\":\"$(base64 -w0 < "$file")\"," + done + json="${json%,}}}," + cd - > /dev/null + [[ "$result" = "PASS" ]] && (( passed++ )) + [[ "$result" = "FAIL" ]] && (( failed++ )) + else + result="SKIP" + (( skipped++ )) + json+="{\"test\":\"$test\",\"url\":\"$url\",\"result\":\"$result\"}," + fi + color=2 # green + [[ "$result" = "FAIL" ]] && color=1 # red + [[ "$result" = "SKIP" ]] && color=3 # yellow + printf " %-40s $(tput setaf $color)$result$(tput sgr0)\n" "$test" +done +echo "" +echo -n "Summary: TOTAL: $total / " +echo -n "$(tput setaf 2)PASS$normal: $passed / " +echo -n "$(tput setaf 1)FAIL$normal: $failed / " +echo "$(tput setaf 3)SKIP$normal: $skipped" +echo "" + +json="\"tests\":[${json%,}]" +metadata="\"timestamp\":\"$timestamp\"," +metadata+="\"revision\":\"$rev\"," +metadata+="\"upstream-revision\":\"$upstreamrev\"," +if [[ -n "$GITHUB_ACTIONS" ]] +then + metadata+="\"branch\":\"$GITHUB_REF\"," +fi +json="{$metadata $json}" + +# Clean up +cd "$scriptpath" +rm -rf "$tempdir" + +resultsfile="test-results.json" +echo "$json" | jq > "$resultsfile" +echo "Results written to $scriptpath/$resultsfile" + +exit $exitcode