compare_dg_tests.pl: Handle tests that can FAIL but never PASS
The GDB testsuite has a "-nopass" option to the "gdb_test" TCL procedure
which emits a FAIL if the test fails but doesn't emit anything if the test
passes.
This is causing trouble for the CI bisection logic, because if a flaky test
produces a FAIL in a single retry of the testsuite then we don't recognise
its absence in other runs as a kind of PASS.
Suggested-by: Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org>
Change-Id: I8539393424d1f7dbb8eba2b42b5ea94fd321d14e
diff --git a/compare_dg_tests.pl b/compare_dg_tests.pl
index f660000..27d464b 100755
--- a/compare_dg_tests.pl
+++ b/compare_dg_tests.pl
@@ -1090,10 +1090,20 @@
if ($diag ne "PASS") {
my $xfail_seen = 0;
+ my $test_in_all_results = 1;
# Did this test pass or xfail in any of the other results?
foreach my $result (@results) {
- if ($result->{testcases}->{$tc}->{PASS}) {
+ if (not exists $result->{testcases}->{$tc}) {
+ # Check whether the exp file is present in this result.
+ foreach my $key (keys %{$result->{testcases}}) {
+ if (index($key, "$current_tool:$current_exp=") == 0) {
+ # The exp file is present but this test isn't.
+ $test_in_all_results = 0;
+ last;
+ }
+ }
+ } elsif ($result->{testcases}->{$tc}->{PASS}) {
print OUTPUT "PASS: $2\n";
$printed = 1;
@@ -1113,6 +1123,16 @@
# Adjust the summary statistics.
$res->{$diag}--;
$res->{XFAIL}++;
+ } elsif (not $test_in_all_results) {
+ # If this test isn't present in all the sum files then consider its
+ # absence from at least one of them to mean that it didn't fail, and
+ # therefore leave it out of the merged sum file. This addresses a
+ # situation that can happen in GDB tests where if a test succeeds it
+ # doesn't produce a PASS but if it fails it produces a FAIL.
+ $printed = 1;
+
+ # Adjust the summary statistics.
+ $res->{$diag}--;
}
}
} elsif (m/^(# of expected passes|# of unexpected failures|# of expected failures|# of known failures|# of unsupported tests|# of untested testcases|# of unresolved testcases)(\s+)(\d+)$/) {
diff --git a/testsuite/sum-merge-1.txt b/testsuite/sum-merge-1.txt
index 1848be3..7bc957b 100644
--- a/testsuite/sum-merge-1.txt
+++ b/testsuite/sum-merge-1.txt
@@ -199,6 +199,7 @@
FAIL: test-fail-unresolved-unsupported
FAIL: test-fail-unresolved-untested
FAIL: test-fail-unresolved-unresolved
+FAIL: test-fail-absent-absent
Running tests-xfail.exp ...
XFAIL: test-xfail-pass-pass
@@ -530,11 +531,14 @@
UNRESOLVED: test-unresolved-unresolved-untested
UNRESOLVED: test-unresolved-unresolved-unresolved
+Running test-that-only-appears-in-sum-1.exp ...
+FAIL: Test that this will appear in the merged sum file
+
=== gcc-compare-results Summary ===
# of <unexpected status> 1
# of expected passes 64
-# of unexpected failures 64
+# of unexpected failures 66
# of expected failures 64
# of known failures 64
# of untested testcases 64
diff --git a/testsuite/sum-merge-2.txt b/testsuite/sum-merge-2.txt
index 7814f2c..f17e65f 100644
--- a/testsuite/sum-merge-2.txt
+++ b/testsuite/sum-merge-2.txt
@@ -199,6 +199,7 @@
UNRESOLVED: test-fail-unresolved-unsupported
UNRESOLVED: test-fail-unresolved-untested
UNRESOLVED: test-fail-unresolved-unresolved
+FAIL: test-absent-fail-absent
Running tests-xfail.exp ...
PASS: test-xfail-pass-pass
@@ -533,7 +534,7 @@
=== gcc-compare-results Summary ===
# of expected passes 64
-# of unexpected failures 64
+# of unexpected failures 65
# of expected failures 64
# of known failures 64
# of untested testcases 64
diff --git a/testsuite/sum-merge-3.txt b/testsuite/sum-merge-3.txt
index 1addd3c..d5d76ff 100644
--- a/testsuite/sum-merge-3.txt
+++ b/testsuite/sum-merge-3.txt
@@ -199,6 +199,7 @@
UNSUPPORTED: test-fail-unresolved-unsupported
UNTESTED: test-fail-unresolved-untested
UNRESOLVED: test-fail-unresolved-unresolved
+FAIL: test-absent-absent-fail
Running tests-xfail.exp ...
PASS: test-xfail-pass-pass
@@ -533,7 +534,7 @@
=== gcc-compare-results Summary ===
# of expected passes 64
-# of unexpected failures 64
+# of unexpected failures 65
# of expected failures 64
# of known failures 64
# of untested testcases 64
diff --git a/testsuite/sum-merge-expected.txt b/testsuite/sum-merge-expected.txt
index 07e9a94..f127364 100644
--- a/testsuite/sum-merge-expected.txt
+++ b/testsuite/sum-merge-expected.txt
@@ -534,11 +534,14 @@
UNRESOLVED: test-unresolved-unresolved-untested
UNRESOLVED: test-unresolved-unresolved-unresolved
+Running test-that-only-appears-in-sum-1.exp ...
+FAIL: Test that this will appear in the merged sum file
+
=== gcc-compare-results Summary ===
# of <unexpected status> 1
# of expected passes 169
-# of unexpected failures 36
+# of unexpected failures 37
# of expected failures 127
# of known failures 36
# of untested testcases 36