Run every test script and sum up the results

make stopped at the first failing script and left every later one
unrun. Each script's closing line also counts its skipped tests as
passes, so a run that skipped half the suite read like a full one, and
the counts the library writes for each script were never read.

1 file changed, 19 insertions, 5 deletions

diff --git a/tests/Makefile b/tests/Makefileindex e165b01..362ed45 100644--- a/tests/Makefile+++ b/tests/Makefile@@ -17,16 +17,30 @@ SHELL_PATH ?= $(SHELL) # Escaped for the single quotes that the recipe below puts around the path. SHELL_PATH_SQ = $(subst ','\'',$(SHELL_PATH)) -TESTS = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)+TESTS = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)) +# A failing script would stop make and leave every later one unrun, so the+# scripts run in a make of their own that keeps going, and the summary fails+# the run once all of them have had their turn. .PHONY: all-all: $(TESTS)+all:+	@$(MAKE) -k $(TESTS) || :+	@'$(SHELL_PATH_SQ)' aggregate-results.sh $(TESTS)++# Counts left by a script since renamed or removed would otherwise reach the+# summary. The directory is made at once, as a script that stops early writes+# nothing there itself.+.PHONY: pre-clean+pre-clean:+	@$(RM) -r test-results && mkdir test-results  # Each test is named after a file that already exists, so without this make-# would decide there is nothing to do and run none of them.+# would decide there is nothing to do and run none of them. The exit status+# is kept for the summary, since a script can fail without a failed test. .PHONY: $(TESTS)-$(TESTS):-	@'$(SHELL_PATH_SQ)' $@ $(CGIT_TEST_OPTS)+$(TESTS): pre-clean+	@'$(SHELL_PATH_SQ)' $@ $(CGIT_TEST_OPTS); \+	status=$$?; echo $$status >test-results/$(@:.sh=.exit); exit $$status  .PHONY: clean clean:
diff --git a/tests/Makefile b/tests/Makefile
index e165b01..362ed45 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -17,16 +17,30 @@ SHELL_PATH ?= $(SHELL)
 # Escaped for the single quotes that the recipe below puts around the path.
 SHELL_PATH_SQ = $(subst ','\'',$(SHELL_PATH))
 
-TESTS = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)
+TESTS = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh))
 
+# A failing script would stop make and leave every later one unrun, so the
+# scripts run in a make of their own that keeps going, and the summary fails
+# the run once all of them have had their turn.
 .PHONY: all
-all: $(TESTS)
+all:
+	@$(MAKE) -k $(TESTS) || :
+	@'$(SHELL_PATH_SQ)' aggregate-results.sh $(TESTS)
+
+# Counts left by a script since renamed or removed would otherwise reach the
+# summary. The directory is made at once, as a script that stops early writes
+# nothing there itself.
+.PHONY: pre-clean
+pre-clean:
+	@$(RM) -r test-results && mkdir test-results
 
 # Each test is named after a file that already exists, so without this make
-# would decide there is nothing to do and run none of them.
+# would decide there is nothing to do and run none of them. The exit status
+# is kept for the summary, since a script can fail without a failed test.
 .PHONY: $(TESTS)
-$(TESTS):
-	@'$(SHELL_PATH_SQ)' $@ $(CGIT_TEST_OPTS)
+$(TESTS): pre-clean
+	@'$(SHELL_PATH_SQ)' $@ $(CGIT_TEST_OPTS); \
+	status=$$?; echo $$status >test-results/$(@:.sh=.exit); exit $$status
 
 .PHONY: clean
 clean: