Diff options
| -rw-r--r-- | tests/Makefile | 24 | |
| -rw-r--r-- | tests/aggregate-results.sh | 60 |
2 files changed, 79 insertions, 5 deletions
diff --git a/tests/Makefile b/tests/Makefile index e165b01..362ed45 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -17,16 +17,30 @@ SHELL_PATH ?= $(SHELL) # Escaped for the single quotes that the recipe below puts around the path. SHELL_PATH_SQ = $(subst ','\'',$(SHELL_PATH)) -TESTS = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh) +TESTS = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)) +# A failing script would stop make and leave every later one unrun, so the +# scripts run in a make of their own that keeps going, and the summary fails +# the run once all of them have had their turn. .PHONY: all -all: $(TESTS) +all: + @$(MAKE) -k $(TESTS) || : + @'$(SHELL_PATH_SQ)' aggregate-results.sh $(TESTS) + +# Counts left by a script since renamed or removed would otherwise reach the +# summary. The directory is made at once, as a script that stops early writes +# nothing there itself. +.PHONY: pre-clean +pre-clean: + @$(RM) -r test-results && mkdir test-results # Each test is named after a file that already exists, so without this make -# would decide there is nothing to do and run none of them. +# would decide there is nothing to do and run none of them. The exit status +# is kept for the summary, since a script can fail without a failed test. .PHONY: $(TESTS) -$(TESTS): - @'$(SHELL_PATH_SQ)' $@ $(CGIT_TEST_OPTS) +$(TESTS): pre-clean + @'$(SHELL_PATH_SQ)' $@ $(CGIT_TEST_OPTS); \ + status=$$?; echo $$status >test-results/$(@:.sh=.exit); exit $$status .PHONY: clean clean: diff --git a/tests/aggregate-results.sh b/tests/aggregate-results.sh new file mode 100644 index 0000000..27ac1df --- /dev/null +++ b/tests/aggregate-results.sh @@ -0,0 +1,60 @@ +# Sums the counts the test scripts named as arguments left in test-results/ +# into one line, with skipped tests counted apart from passes, and names the +# scripts that failed or were skipped whole. A script that stopped before it +# could write its counts, or that left with a status other than 0, failed +# whatever its counts say. tests/Makefile runs this once every script has run. + +scripts=0 +tests=0 +passed=0 +skipped=0 +failed=0 +failed_scripts= +skipped_scripts= +missing= + +for script +do + name=${script%.sh} + scripts=$((scripts + 1)) + if test "$(cat "test-results/$name.exit" 2>/dev/null)" != 0 || + ! test -f "test-results/$name.counts" + then + failed_scripts="$failed_scripts $name" + fi + test -f "test-results/$name.counts" || continue + + total=0 success=0 fixed=0 broken=0 failure=0 prereqs= + while read -r key value + do + case "$key" in + total) total=$value ;; + success) success=$value ;; + fixed) fixed=$value ;; + broken) broken=$value ;; + failed) failure=$value ;; + missing_prereq) prereqs=$value ;; + esac + done <"test-results/$name.counts" + + test "$total" != 0 || skipped_scripts="$skipped_scripts $name" + tests=$((tests + total)) + passed=$((passed + success)) + # A known breakage that is still broken counts as neither, and one that + # has started to pass fails the script, as the library has it. + failed=$((failed + failure + fixed)) + skipped=$((skipped + total - success - failure - fixed - broken)) + for prereq in $(echo "$prereqs" | tr ',' ' ') + do + case " $missing " in + *" $prereq "*) ;; + *) missing="$missing $prereq" ;; + esac + done +done + +test -z "$skipped_scripts" || echo "skipped whole:$skipped_scripts" +test -z "$missing" || echo "skipped for want of:$missing" +test -z "$failed_scripts" || echo "failed:$failed_scripts" +echo "$scripts scripts, $tests tests: $passed passed, $skipped skipped, $failed failed" +test -z "$failed_scripts" |