[2.20] [3/6] Support expected failures in .test-result files

["Joseph S. Myers" <joseph at codesourcery dot com>]

This patch makes it possible for .test-result files for individual
tests to contain XPASS and XFAIL rather than PASS and FAIL in cases
where failure is expected.  This replaces the marking of two
individual tests with "-" to cause them to be expected at makefile
level to fail; evaluate-test.sh will ensure it exits with status 0 for
an expected failure.

With such an expected failure mechanism, it would be possible for
particular architectures to mark tests as expected to fail in their
sysdeps makefiles, so providing more structured information about such
expectations (which presently goes on per-release wiki pages and
distributions' own tracking systems for expected failures) and
reducing the need for duplicating such tracking for each release and
distribution.  Whether we wish to use it this way is a separate policy
question - if we do, we might want rules such as requiring such
system-specific XFAILs to have a comment referring to a GCC or glibc
bug, with the GCC or glibc bug having a mention of the XFAIL, and
include a review of XPASSes in the things architecture maintainers
should do for each release, to see if any such expectations of failure
can be removed.

Tested x86_64.

2014-01-10  Joseph Myers  <joseph@codesourcery.com>

	* scripts/evaluate-test.sh: Take new argument indicating whether
	failure is expected.
	* Makeconfig (evaluate-test): Pass argument to evaluate-test.sh
	indicating whether failure is expected.
	* conform/Makefile (test-xfail-run-conformtest): New variable.
	($(objpfx)run-conformtest.out): Don't expect to fail at makefile
	level.
	* posix/Makefile (test-xfail-annexc): New variable.
	($(objpfx)annexc.out): Don't expect to fail at makefile level.

diff --git a/Makeconfig b/Makeconfig
index a3aa65a..ea40b38 100644
--- a/Makeconfig
+++ b/Makeconfig
@@ -1087,8 +1087,11 @@ else
 test-name = $(@F:.out=)
 endif
 
-# Command to output a test status line (such as PASS: test-name).
+# Command to output a test status line (such as PASS: test-name).  If
+# test-xfail-$(@F:.out=) has a nonempty value, the status will be
+# XPASS or XFAIL rather than PASS or FAIL.
 evaluate-test = $(..)scripts/evaluate-test.sh $(test-name) $$? \
+		  $(if $(test-xfail-$(@F:.out=)),true,false) \
 		  > $(objpfx)$(@F:.out=).test-result
 
 endif # Makeconfig not yet included
diff --git a/conform/Makefile b/conform/Makefile
index 0761a1c..8e2b9d9 100644
--- a/conform/Makefile
+++ b/conform/Makefile
@@ -28,10 +28,11 @@ tests: $(objpfx)run-conformtest.out
 endif
 endif
 
+test-xfail-run-conformtest = yes
 $(objpfx)run-conformtest.out: run-conformtest.sh conformtest.pl \
 			      $(wildcard data/*.h-data) \
 			      $(wildcard data/*/*.h-data)
-	-$(BASH) -e $< $(objpfx) $(PERL) '$(CC)' \
+	$(BASH) -e $< $(objpfx) $(PERL) '$(CC)' \
 	  '-I../include $(+sysdep-includes) $(sysincludes) -I..'; \
 	$(evaluate-test)
 
diff --git a/posix/Makefile b/posix/Makefile
index ac462c9..1eda4fb 100644
--- a/posix/Makefile
+++ b/posix/Makefile
@@ -228,8 +228,9 @@ tests: $(objpfx)bug-regex2-mem $(objpfx)bug-regex14-mem \
 xtests: $(objpfx)bug-ga2-mem
 endif
 
+test-xfail-annexc = yes
 $(objpfx)annexc.out: $(objpfx)annexc
-	-$(dir $<)$(notdir $<) '$(CC)' \
+	$(dir $<)$(notdir $<) '$(CC)' \
 	  '$(patsubst %,-I../%,$(sorted-subdirs)) -I../include $(+sysdep-includes) $(sysincludes) -I..' > $@; \
 	$(evaluate-test)
 
diff --git a/scripts/evaluate-test.sh b/scripts/evaluate-test.sh
index e118a45..a7f326c 100755
--- a/scripts/evaluate-test.sh
+++ b/scripts/evaluate-test.sh
@@ -17,10 +17,11 @@
 # License along with the GNU C Library; if not, see
 # <http://www.gnu.org/licenses/>.
 
-# usage: evaluate-test.sh test_name rc
+# usage: evaluate-test.sh test_name rc xfail
 
 test_name=$1
 rc=$2
+xfail=$3
 
 if [ $rc -eq 0 ]; then
   result="PASS"
@@ -28,5 +29,10 @@ else
   result="FAIL"
 fi
 
+if $xfail; then
+  result="X$result"
+  rc=0
+fi
+
 echo "$result: $test_name"
 exit $rc

-- 
Joseph S. Myers
joseph@codesourcery.com