This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
Re: [PATCH 0/N] test-suite improvement - PASS/FAIL: initial patch
- From: Tomas Dohnalek <tdohnale at redhat dot com>
- To: "Joseph S. Myers" <joseph at codesourcery dot com>
- Cc: libc-alpha at sourceware dot org, Andreas Jaeger <aj at suse dot com>
- Date: Sun, 16 Sep 2012 11:46:57 -0400 (EDT)
- Subject: Re: [PATCH 0/N] test-suite improvement - PASS/FAIL: initial patch
Hi,
I have reworked the patch, so now it creates .test-result files to each test,
where is stores result of single test-case in text form and than collects it.
What do you think?
Q: Is there any reason why (in Makerules) rule `common-mostlyclean' is not not
dependent on rule `do-tests-clean' and that is why there is code duplicity?
----- Original Message -----
> On Mon, 10 Sep 2012, Andreas Jaeger wrote:
>
> > will it handle running the testsuite with parallel make? In that
> > case the
> > output might not be deterministic, so you need to sort it before
> > comparing...
>
> I'm not sure having two parallel runs append to the same file from
> the
> shell is safe, though it may be in practice.
>
> I'd suggest that you generate a .test-status or similar file for each
> test
> with its result - and then collect all the data at the end with a
> separate
> script that produces the summary from all those files in the build
> tree.
> That also looks like it will work better if someone does
>
> make check
> (a test fails because of an old kernel)
> make check
> (to continue the test run)
>
> rather than producing a summary of only the results from the second
> part
> of the run.
>
>
> --
> Joseph S. Myers
> joseph@codesourcery.com
>
---
* Makerules: New rule `tests-collect-results', covered abi-checks
by PASS/FAIL line.
* Rules: New variables `evaluate-test' and `test-name',
covered all ordinary tests by PASS/FAIL line.
* scripts/evaluate-test.sh: New file.
diff --git a/Makerules b/Makerules
index 1281b94..faa9fc9 100644
--- a/Makerules
+++ b/Makerules
@@ -1123,11 +1123,15 @@ ALL_BUILD_CFLAGS = $(BUILD_CFLAGS) $(BUILD_CPPFLAGS) -D_GNU_SOURCE \
# Support the GNU standard name for this target.
.PHONY: check
-check: tests
+check: tests tests-collect-results
# Special target to run tests which cannot be run unconditionally.
# Maintainers should use this target.
.PHONY: xcheck
-xcheck: xtests
+xcheck: xtests tests-collect-results
+
+tests-summary = $(common-objpfx)tests.sum
+tests-collect-results:
+ -cat $(common-objpfx)*/*.test-result > $(tests-summary)
all-nonlib = $(strip $(tests) $(xtests) $(test-srcs) $(test-extras) $(others))
ifneq (,$(all-nonlib))
@@ -1166,7 +1170,8 @@ check-abi-%: $(common-objpfx)config.make %.abilist $(objpfx)%.symlist
check-abi-%: $(common-objpfx)config.make %.abilist $(common-objpfx)%.symlist
$(check-abi)
define check-abi
- diff -p -U 0 $(filter %.abilist,$^) $(filter %.symlist,$^)
+ diff -p -U 0 $(filter %.abilist,$^) $(filter %.symlist,$^); \
+ $(evaluate-test)
endef
update-abi-%: $(objpfx)%.symlist %.abilist
@@ -1275,7 +1280,10 @@ do-tests-clean:
-rm -f $(addprefix $(objpfx),$(addsuffix .out,$(tests) $(xtests) \
$(test-srcs)) \
$(addsuffix -bp.out,$(tests) $(xtests) \
- $(test-srcs)))
+ $(test-srcs)) \
+ $(addsuffix .test-result,$(tests) $(xtests) \
+ $(test-srcs) $(extra-libs)))
+ -rm -f $(tests-summary)
# Remove the object files.
common-mostlyclean:
@@ -1289,10 +1297,13 @@ common-mostlyclean:
$(addsuffix .out,$(tests) $(xtests) \
$(test-srcs)) \
$(addsuffix -bp.out,$(tests) $(xtests) \
- $(test-srcs)))
+ $(test-srcs)) \
+ $(addsuffix .test-result,$(tests) $(xtests) \
+ $(test-srcs) $(extra-libs)))
-rm -f $(addprefix $(objpfx),$(extra-objs) $(extra-test-objs) \
$(install-lib) $(install-lib.so) \
$(install-lib.so:%.so=%_pic.a))
+ -rm -f $(tests-summary)
-rm -f core
-rm -f $(objpfx)rtld-*.os
$(rmobjs)
diff --git a/Rules b/Rules
index 17d938e..4037694 100644
--- a/Rules
+++ b/Rules
@@ -127,6 +127,9 @@ binaries-shared-tests = $(filter-out $(binaries-pie) $(binaries-static), \
$(binaries-all-tests))
binaries-shared-notests = $(filter-out $(binaries-pie) $(binaries-static), \
$(binaries-all-notests))
+test-name = $(subdir)/$(*F)
+evaluate-test = $(..)scripts/evaluate-test.sh $$? $(test-name) > \
+ $(objpfx)$(*F).test-result
ifneq "$(strip $(binaries-shared-notests))" ""
$(addprefix $(objpfx),$(binaries-shared-notests)): %: %.o \
@@ -178,11 +181,11 @@ ifneq "$(strip $(tests) $(xtests) $(test-srcs))" ""
make-test-out = GCONV_PATH=$(common-objpfx)iconvdata LC_ALL=C \
$($*-ENV) $(built-program-cmd) $($*-ARGS)
$(objpfx)%-bp.out: %.input $(objpfx)%-bp
- $(make-test-out) > $@ < $(word 1,$^)
+ $(make-test-out) > $@ < $(word 1,$^); $(evaluate-test)
$(objpfx)%.out: %.input $(objpfx)%
- $(make-test-out) > $@ < $(word 1,$^)
+ $(make-test-out) > $@ < $(word 1,$^); $(evaluate-test)
$(objpfx)%.out: /dev/null $(objpfx)% # Make it 2nd arg for canned sequence.
- $(make-test-out) > $@
+ $(make-test-out) > $@; $(evaluate-test)
endif # tests
diff --git a/scripts/evaluate-test.sh b/scripts/evaluate-test.sh
new file mode 100755
index 0000000..e5f8ad0
--- /dev/null
+++ b/scripts/evaluate-test.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+# Copyright (C) 2012 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+#
+# This script is used to evaluate return code of a single testcase
+# and produce appropriate output.
+# usage: evaluate-test.sh test-rc test-name
+
+test_rc=$1
+test_name=$2
+
+if [ $test_rc -eq 0 ]; then
+ result="PASS"
+else
+ result="FAIL"
+fi
+
+echo "$result: $test_name"
+exit $test_rc