Diff options
| author | Bryce Kwon <bryce@brycekwon.com> | |
|---|---|---|
| committer | Bryce Kwon <bryce@brycekwon.com> | |
| commit | ||
| parent | ||
| tree | ||
| download | ||
Run every test script and sum up the results
make stopped at the first failing script and left every later one
unrun. Each script's closing line also counts its skipped tests as
passes, so a run that skipped half the suite read like a full one, and
the counts the library writes for each script were never read.
Diffstat (limited to 'tests/aggregate-results.sh')
| -rw-r--r-- | tests/aggregate-results.sh | 60 |
1 file changed, 60 insertions, 0 deletions
diff --git a/tests/aggregate-results.sh b/tests/aggregate-results.sh new file mode 100644 index 0000000..27ac1df --- /dev/null +++ b/tests/aggregate-results.sh @@ -0,0 +1,60 @@ +# Sums the counts the test scripts named as arguments left in test-results/ +# into one line, with skipped tests counted apart from passes, and names the +# scripts that failed or were skipped whole. A script that stopped before it +# could write its counts, or that left with a status other than 0, failed +# whatever its counts say. tests/Makefile runs this once every script has run. + +scripts=0 +tests=0 +passed=0 +skipped=0 +failed=0 +failed_scripts= +skipped_scripts= +missing= + +for script +do + name=${script%.sh} + scripts=$((scripts + 1)) + if test "$(cat "test-results/$name.exit" 2>/dev/null)" != 0 || + ! test -f "test-results/$name.counts" + then + failed_scripts="$failed_scripts $name" + fi + test -f "test-results/$name.counts" || continue + + total=0 success=0 fixed=0 broken=0 failure=0 prereqs= + while read -r key value + do + case "$key" in + total) total=$value ;; + success) success=$value ;; + fixed) fixed=$value ;; + broken) broken=$value ;; + failed) failure=$value ;; + missing_prereq) prereqs=$value ;; + esac + done <"test-results/$name.counts" + + test "$total" != 0 || skipped_scripts="$skipped_scripts $name" + tests=$((tests + total)) + passed=$((passed + success)) + # A known breakage that is still broken counts as neither, and one that + # has started to pass fails the script, as the library has it. + failed=$((failed + failure + fixed)) + skipped=$((skipped + total - success - failure - fixed - broken)) + for prereq in $(echo "$prereqs" | tr ',' ' ') + do + case " $missing " in + *" $prereq "*) ;; + *) missing="$missing $prereq" ;; + esac + done +done + +test -z "$skipped_scripts" || echo "skipped whole:$skipped_scripts" +test -z "$missing" || echo "skipped for want of:$missing" +test -z "$failed_scripts" || echo "failed:$failed_scripts" +echo "$scripts scripts, $tests tests: $passed passed, $skipped skipped, $failed failed" +test -z "$failed_scripts" |