Diff options
| author | Bryce Kwon <bryce@brycekwon.com> | |
|---|---|---|
| committer | Bryce Kwon <bryce@brycekwon.com> | |
| commit | ||
| parent | ||
| tree | ||
| download | ||
Run every test script and sum up the results
make stopped at the first failing script and left every later one
unrun. Each script's closing line also counts its skipped tests as
passes, so a run that skipped half the suite read like a full one, and
the counts the library writes for each script were never read.
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/Makefile | 24 | |
| -rw-r--r-- | tests/aggregate-results.sh | 60 |
2 files changed, 79 insertions, 5 deletions
diff --git a/tests/Makefile b/tests/Makefile index e165b01..362ed45 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -17,16 +17,30 @@ SHELL_PATH ?= $(SHELL) # Escaped for the single quotes that the recipe below puts around the path. SHELL_PATH_SQ = $(subst ','\'',$(SHELL_PATH)) -TESTS = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh) +TESTS = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)) +# A failing script would stop make and leave every later one unrun, so the +# scripts run in a make of their own that keeps going, and the summary fails +# the run once all of them have had their turn. .PHONY: all -all: $(TESTS) +all: + @$(MAKE) -k $(TESTS) || : + @'$(SHELL_PATH_SQ)' aggregate-results.sh $(TESTS) + +# Counts left by a script since renamed or removed would otherwise reach the +# summary. The directory is made at once, as a script that stops early writes +# nothing there itself. +.PHONY: pre-clean +pre-clean: + @$(RM) -r test-results && mkdir test-results # Each test is named after a file that already exists, so without this make -# would decide there is nothing to do and run none of them. +# would decide there is nothing to do and run none of them. The exit status +# is kept for the summary, since a script can fail without a failed test. .PHONY: $(TESTS) -$(TESTS): - @'$(SHELL_PATH_SQ)' $@ $(CGIT_TEST_OPTS) +$(TESTS): pre-clean + @'$(SHELL_PATH_SQ)' $@ $(CGIT_TEST_OPTS); \ + status=$$?; echo $$status >test-results/$(@:.sh=.exit); exit $$status .PHONY: clean clean: diff --git a/tests/aggregate-results.sh b/tests/aggregate-results.sh new file mode 100644 index 0000000..27ac1df --- /dev/null +++ b/tests/aggregate-results.sh @@ -0,0 +1,60 @@ +# Sums the counts the test scripts named as arguments left in test-results/ +# into one line, with skipped tests counted apart from passes, and names the +# scripts that failed or were skipped whole. A script that stopped before it +# could write its counts, or that left with a status other than 0, failed +# whatever its counts say. tests/Makefile runs this once every script has run. + +scripts=0 +tests=0 +passed=0 +skipped=0 +failed=0 +failed_scripts= +skipped_scripts= +missing= + +for script +do + name=${script%.sh} + scripts=$((scripts + 1)) + if test "$(cat "test-results/$name.exit" 2>/dev/null)" != 0 || + ! test -f "test-results/$name.counts" + then + failed_scripts="$failed_scripts $name" + fi + test -f "test-results/$name.counts" || continue + + total=0 success=0 fixed=0 broken=0 failure=0 prereqs= + while read -r key value + do + case "$key" in + total) total=$value ;; + success) success=$value ;; + fixed) fixed=$value ;; + broken) broken=$value ;; + failed) failure=$value ;; + missing_prereq) prereqs=$value ;; + esac + done <"test-results/$name.counts" + + test "$total" != 0 || skipped_scripts="$skipped_scripts $name" + tests=$((tests + total)) + passed=$((passed + success)) + # A known breakage that is still broken counts as neither, and one that + # has started to pass fails the script, as the library has it. + failed=$((failed + failure + fixed)) + skipped=$((skipped + total - success - failure - fixed - broken)) + for prereq in $(echo "$prereqs" | tr ',' ' ') + do + case " $missing " in + *" $prereq "*) ;; + *) missing="$missing $prereq" ;; + esac + done +done + +test -z "$skipped_scripts" || echo "skipped whole:$skipped_scripts" +test -z "$missing" || echo "skipped for want of:$missing" +test -z "$failed_scripts" || echo "failed:$failed_scripts" +echo "$scripts scripts, $tests tests: $passed passed, $skipped skipped, $failed failed" +test -z "$failed_scripts" |