tests/aggregate-results.sh


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

# Sums the counts the test scripts named as arguments left in test-results/
# into one line, with skipped tests counted apart from passes, and names the
# scripts that failed or were skipped whole. A script that stopped before it
# could write its counts, or that left with a status other than 0, failed
# whatever its counts say. tests/Makefile runs this once every script has run.

scripts=0
tests=0
passed=0
skipped=0
failed=0
failed_scripts=
skipped_scripts=
missing=

for script
do
	name=${script%.sh}
	scripts=$((scripts + 1))
	if test "$(cat "test-results/$name.exit" 2>/dev/null)" != 0 ||
		! test -f "test-results/$name.counts"
	then
		failed_scripts="$failed_scripts $name"
	fi
	# setup.sh fails a script that leaves a sanitizer report, and this
	# makes sure of it.
	for report in "test-results/$name.sanitizer".*
	do
		test -f "$report" || continue
		echo "sanitizer report: $report"
		case " $failed_scripts " in
		*" $name "*) ;;
		*) failed_scripts="$failed_scripts $name" ;;
		esac
	done
	test -f "test-results/$name.counts" || continue

	total=0 success=0 fixed=0 broken=0 failure=0 prereqs=
	while read -r key value
	do
		case "$key" in
		total) total=$value ;;
		success) success=$value ;;
		fixed) fixed=$value ;;
		broken) broken=$value ;;
		failed) failure=$value ;;
		missing_prereq) prereqs=$value ;;
		esac
	done <"test-results/$name.counts"

	test "$total" != 0 || skipped_scripts="$skipped_scripts $name"
	tests=$((tests + total))
	passed=$((passed + success))
	# A known breakage that is still broken counts as neither, and one that
	# has started to pass fails the script, as the library has it.
	failed=$((failed + failure + fixed))
	skipped=$((skipped + total - success - failure - fixed - broken))
	for prereq in $(echo "$prereqs" | tr ',' ' ')
	do
		case " $missing " in
		*" $prereq "*) ;;
		*) missing="$missing $prereq" ;;
		esac
	done
done

test -z "$skipped_scripts" || echo "skipped whole:$skipped_scripts"
test -z "$missing" || echo "skipped for want of:$missing"
test -z "$failed_scripts" || echo "failed:$failed_scripts"
echo "$scripts scripts, $tests tests: $passed passed, $skipped skipped, $failed failed"
test -z "$failed_scripts"