compare_dg_tests.pl: Add --flaky-tests= option
This is similar to --unstable-tests, but reads a file produced by
validate_failures.py.
Such tests are ignored in the comparison results, but displayed when
using the long format (-l).
We currently ignore the expire=YYYYMMDD attribute.
Change-Id: I5bcdd2df9cc19a880f6196c3ca8fd62dce2961f8
diff --git a/compare_dg_tests.pl b/compare_dg_tests.pl
index dba648e..7b5f8b4 100755
--- a/compare_dg_tests.pl
+++ b/compare_dg_tests.pl
@@ -12,6 +12,7 @@
sub read_sum($);
sub read_unstable($);
+sub read_flaky($$);
sub dump_result($);
sub compare_results($$);
sub usage();
@@ -112,7 +113,7 @@
my $ERROR_APPEARS = "ERROR appears [ =>ERROR]";
my $UNHANDLED_CASES = "Unhandled cases [ ..??.. ]";
-my $UNSTABLE_CASES = "Unstable cases, ignored [~RANDOM ]";
+my $UNSTABLE_CASES = "Flaky cases, ignored [~RANDOM ]";
my $HWDEP_CASES = "HW dependent cases [HW-DEPENDENT]";
# Handle status transitions:
@@ -232,6 +233,7 @@
my $nounstable=0;
my $unstablefile=0;
my @unstable_markers=();
+my $flakyfile=0;
my $no_hwdep=0; # Ignore hw-dependent flag (ie. consider all tests as *not* hw-dependent
my $hwdep_file=0; # File containing the list of hw-dependent tests
@@ -254,6 +256,7 @@
"no-hwdep" => \$no_hwdep,
"hwdep-tests=s" => \$hwdep_file,
"hwdep-marker=s" => \@hwdep_markers,
+ "flaky-tests=s" => \$flakyfile,
"merge" => \$merge_mode);
# Merge is a separate operating mode of this script where instead of comparing sum files,
@@ -300,6 +303,7 @@
my @unstablelist = ();
@unstablelist = read_unstable($unstablefile) if ($unstablefile ne 0);
+read_flaky($flakyfile, \@unstablelist) if ($flakyfile ne 0);
# Import list of hw-dependent files, if provided
my @hwdep_list = ();
@@ -452,6 +456,37 @@
return @unstable_tests;
}
+# Parse list of flaky tests and append to $unstable_tests. This is
+# similar to read_unstable, but reads a file produced by
+# validate_failures.py
+# FIXME: Handle expire attribute
+sub read_flaky($$)
+{
+ my ($flaky_file, $unstable_tests) = @_;
+
+ open FLAKYFILE, $flaky_file or die $!;
+ while (<FLAKYFILE>)
+ {
+ # Only parse lines starting with "flaky[.*] |"
+ if (/^[ ]*flaky.* \| /)
+ {
+ chomp;
+
+ my $test = $_;
+
+ # Check if line is of type: 'flaky,expire=YYYYMMDD | testname'
+ if (/^[ ]*flaky,expire=.* \| (PASS|XPASS|FAIL|XFAIL|KFAIL|UNSUPPORTED|UNTESTED|UNRESOLVED|ERROR): /)
+ {
+ $test =~ s/.*flaky,expire=.* | (PASS|XPASS|FAIL|XFAIL|KFAIL|UNSUPPORTED|UNTESTED|UNRESOLVED|ERROR): //;
+ } else {
+ $test =~ s/.*flaky \| (PASS|XPASS|FAIL|XFAIL|KFAIL|UNSUPPORTED|UNTESTED|UNRESOLVED|ERROR): //;
+ }
+ push @$unstable_tests, $test;
+ }
+ }
+ close FLAKYFILE;
+}
+
######################################################
# DIFFING
sub handle_pass_fail($$$$)
diff --git a/compare_tests b/compare_tests
index 193c693..27acaf6 100755
--- a/compare_tests
+++ b/compare_tests
@@ -83,6 +83,10 @@
compr_suffix=""
fi
;;
+ "-flaky-tests")
+ flaky_tests=$2
+ shift 2
+ ;;
"-?")
usage
;;
@@ -153,9 +157,13 @@
fi
done
${CONFIG_SHELL-/usr/bin/perl} ${my_path}/compare_dg_tests.pl \
- ${pass_thresh:+-pass-thresh=${pass_thresh}} -l \
- --unstable-test=${my_path}/unstable-tests.txt ${unstable_target} \
- --hwdep-tests=${my_path}/host-hw-dependent-tests.txt $sum1 $sum2
+ -l \
+ ${pass_thresh:+-pass-thresh=${pass_thresh}} \
+ ${flaky_tests:+-flaky-tests=${flaky_tests}} \
+ --unstable-test=${my_path}/unstable-tests.txt \
+ ${unstable_target} \
+ --hwdep-tests=${my_path}/host-hw-dependent-tests.txt \
+ $sum1 $sum2
ret=$?
case $ret in
2)
diff --git a/testsuite/Makefile b/testsuite/Makefile
index 171073a..4f3c3dc 100644
--- a/testsuite/Makefile
+++ b/testsuite/Makefile
@@ -1,9 +1,9 @@
check:
LC_ALL=C ../compare_dg_tests.pl --hwdep-tests=hw-dependent-tests.txt sum-1.txt sum-2.txt > cmp-1-2.txt ; diff cmp-1-2.txt expected-1-2.txt
LC_ALL=C ../compare_dg_tests.pl --hwdep-tests=hw-dependent-tests.txt -l sum-1.txt sum-2.txt > cmp-1-2-long.txt ; diff cmp-1-2-long.txt expected-1-2-long.txt
- LC_ALL=C ../compare_dg_tests.pl --merge -o merged-sum.txt sum-merge-1.txt sum-merge-2.txt sum-merge-3.txt ; diff merged-sum.txt sum-merge-expected.txt
+ LC_ALL=C ../compare_dg_tests.pl --hwdep-tests=hw-dependent-tests.txt --flaky-tests=flaky-tests.txt --pass-thresh=0.25 -l sum-1-flaky.txt sum-2-flaky.txt > cmp-1-2-flaky.txt ; diff cmp-1-2-flaky.txt expected-1-2-flaky.txt
# Test that merging a merged file doesn't duplicate the header note.
LC_ALL=C ../compare_dg_tests.pl --merge -o merged-sum.txt sum-merge-expected.txt sum-merge-1.txt ; diff merged-sum.txt sum-merge-expected.txt
clean:
- rm -f cmp-1-2.txt cmp-1-2-long.txt merged-sum.txt
+ rm -f cmp-1-2.txt cmp-1-2-long.txt merged-sum.txt cmp-1-2-flaky.txt
diff --git a/testsuite/expected-1-2-flaky.txt b/testsuite/expected-1-2-flaky.txt
new file mode 100644
index 0000000..791ea08
--- /dev/null
+++ b/testsuite/expected-1-2-flaky.txt
@@ -0,0 +1,112 @@
+Comparing:
+REFERENCE:sum-1-flaky.txt
+CURRENT: sum-2-flaky.txt
+
+ +---------+---------+
+o RUN STATUS: | REF | RES |
+ +------------------------------------------+---------+---------+
+ | Passes [PASS] | 8 | 7 |
+ | Unexpected fails [FAIL] | 8 | 7 |
+ | Errors [ERROR] | 0 | 0 |
+ | Unexpected passes [XPASS] | 8 | 7 |
+ | Expected fails [XFAIL] | 8 | 7 |
+ | Known fails [KFAIL] | 8 | 7 |
+ | Unresolved [UNRESOLVED] | 8 | 7 |
+ | Unsupported [UNSUPPORTED] | 8 | 7 |
+ | Untested [UNTESTED] | 8 | 7 |
+ +------------------------------------------+---------+---------+
+
+ REF PASS ratio: 0.250000
+ RES PASS ratio: 0.250000
+
+o REGRESSIONS:
+ +------------------------------------------+---------+
+ +------------------------------------------+---------+
+ | TOTAL_REGRESSIONS | 0 |
+ +------------------------------------------+---------+
+
+
+o IMPROVEMENTS TO BE CHECKED:
+ +------------------------------------------+---------+
+ | Flaky cases, ignored [~RANDOM ] | 64 |
+ +------------------------------------------+---------+
+ | TOTAL_IMPROVEMENTS_TO_BE_CHECKED | 64 |
+ +------------------------------------------+---------+
+
+ - Flaky cases, ignored [~RANDOM ]:
+
+ Executed from: gcc-compare-results:tests-1.exp
+ gcc-compare-results:tests-1.exp=test2-pass-xpass
+ gcc-compare-results:tests-1.exp=test3-pass-fail
+ gcc-compare-results:tests-1.exp=test4-pass-xfail
+ gcc-compare-results:tests-1.exp=test5-pass-disappears
+ gcc-compare-results:tests-1.exp=test6-pass-unsupported
+ gcc-compare-results:tests-1.exp=test7-pass-untested
+ gcc-compare-results:tests-1.exp=test8-pass-unresolved
+ gcc-compare-results:tests-1.exp=test9-pass-kfail
+ Executed from: gcc-compare-results:tests-10.exp
+ gcc-compare-results:tests-10.exp=test1-kfail-pass
+ gcc-compare-results:tests-10.exp=test2-kfail-xpass
+ gcc-compare-results:tests-10.exp=test3-kfail-fail
+ gcc-compare-results:tests-10.exp=test4-kfail-xfail
+ gcc-compare-results:tests-10.exp=test5-kfail-disappears
+ gcc-compare-results:tests-10.exp=test6-kfail-unsupported
+ gcc-compare-results:tests-10.exp=test7-kfail-untested
+ gcc-compare-results:tests-10.exp=test8-kfail-unresolved
+ Executed from: gcc-compare-results:tests-2.exp
+ gcc-compare-results:tests-2.exp=test1-xpass-pass
+ gcc-compare-results:tests-2.exp=test3-xpass-fail
+ gcc-compare-results:tests-2.exp=test4-xpass-xfail
+ gcc-compare-results:tests-2.exp=test5-xpass-disappears
+ gcc-compare-results:tests-2.exp=test6-xpass-unsupported
+ gcc-compare-results:tests-2.exp=test7-xpass-untested
+ gcc-compare-results:tests-2.exp=test8-xpass-unresolved
+ gcc-compare-results:tests-2.exp=test9-xpass-kfail
+ Executed from: gcc-compare-results:tests-3.exp
+ gcc-compare-results:tests-3.exp=test1-fail-pass
+ gcc-compare-results:tests-3.exp=test2-fail-xpass
+ gcc-compare-results:tests-3.exp=test4-fail-xfail
+ gcc-compare-results:tests-3.exp=test5-fail-disappears
+ gcc-compare-results:tests-3.exp=test6-fail-unsupported
+ gcc-compare-results:tests-3.exp=test7-fail-untested
+ gcc-compare-results:tests-3.exp=test8-fail-unresolved
+ gcc-compare-results:tests-3.exp=test9-fail-kfail
+ Executed from: gcc-compare-results:tests-4.exp
+ gcc-compare-results:tests-4.exp=test1-xfail-pass
+ gcc-compare-results:tests-4.exp=test2-xfail-xpass
+ gcc-compare-results:tests-4.exp=test3-xfail-fail
+ gcc-compare-results:tests-4.exp=test5-xfail-disappears
+ gcc-compare-results:tests-4.exp=test6-xfail-unsupported
+ gcc-compare-results:tests-4.exp=test7-xfail-untested
+ gcc-compare-results:tests-4.exp=test8-xfail-unresolved
+ gcc-compare-results:tests-4.exp=test9-xfail-kfail
+ Executed from: gcc-compare-results:tests-5.exp
+ gcc-compare-results:tests-5.exp=test1-unsupported-pass
+ gcc-compare-results:tests-5.exp=test2-unsupported-xpass
+ gcc-compare-results:tests-5.exp=test3-unsupported-fail
+ gcc-compare-results:tests-5.exp=test4-unsupported-xfail
+ gcc-compare-results:tests-5.exp=test5-unsupported-disappears
+ gcc-compare-results:tests-5.exp=test6-unsupported-untested
+ gcc-compare-results:tests-5.exp=test7-unsupported-unresolved
+ gcc-compare-results:tests-5.exp=test9-unsupported-kfail
+ Executed from: gcc-compare-results:tests-6.exp
+ gcc-compare-results:tests-6.exp=test1-untested-pass
+ gcc-compare-results:tests-6.exp=test2-untested-xpass
+ gcc-compare-results:tests-6.exp=test3-untested-fail
+ gcc-compare-results:tests-6.exp=test4-untested-xfail
+ gcc-compare-results:tests-6.exp=test5-untested-disappears
+ gcc-compare-results:tests-6.exp=test6-untested-unsupported
+ gcc-compare-results:tests-6.exp=test7-untested-unresolved
+ gcc-compare-results:tests-6.exp=test9-untested-kfail
+ Executed from: gcc-compare-results:tests-7.exp
+ gcc-compare-results:tests-7.exp=test1-unresolved-pass
+ gcc-compare-results:tests-7.exp=test2-unresolved-xpass
+ gcc-compare-results:tests-7.exp=test3-unresolved-fail
+ gcc-compare-results:tests-7.exp=test4-unresolved-xfail
+ gcc-compare-results:tests-7.exp=test5-unresolved-disappears
+ gcc-compare-results:tests-7.exp=test6-unresolved-unsupported
+ gcc-compare-results:tests-7.exp=test7-unresolved-untested
+ gcc-compare-results:tests-7.exp=test9-unresolved-kfail
+
+
+
diff --git a/testsuite/flaky-tests.txt b/testsuite/flaky-tests.txt
new file mode 100644
index 0000000..a132838
--- /dev/null
+++ b/testsuite/flaky-tests.txt
@@ -0,0 +1,84 @@
+# From https://ci.linaro.org/job/some/random/job/:
+
+ === gcc-compare-results tests ===
+
+Running target mytarget
+Running /foo/bar/testsuite/tests-1.exp ...
+flaky | PASS: test2-pass-xpass
+flaky | PASS: test3-pass-fail
+flaky | PASS: test4-pass-xfail
+flaky | PASS: test5-pass-disappears
+flaky | PASS: test6-pass-unsupported
+flaky | PASS: test7-pass-untested
+flaky | PASS: test8-pass-unresolved
+flaky | PASS: test9-pass-kfail
+
+Running /foo/bar/testsuite/tests-2.exp ...
+flaky | XPASS: test1-xpass-pass
+flaky | XPASS: test3-xpass-fail
+flaky | XPASS: test4-xpass-xfail
+flaky | XPASS: test5-xpass-disappears
+flaky | XPASS: test6-xpass-unsupported
+flaky | XPASS: test7-xpass-untested
+flaky | XPASS: test8-xpass-unresolved
+flaky | XPASS: test9-xpass-kfail
+
+Running /foo/bar/testsuite/tests-3.exp ...
+flaky | FAIL: test1-fail-pass
+flaky | FAIL: test2-fail-xpass
+flaky | FAIL: test4-fail-xfail
+flaky | FAIL: test5-fail-disappears
+flaky | FAIL: test6-fail-unsupported
+flaky | FAIL: test7-fail-untested
+flaky | FAIL: test8-fail-unresolved
+flaky | FAIL: test9-fail-kfail
+
+Running /foo/bar/testsuite/tests-4.exp ...
+flaky | XFAIL: test1-xfail-pass
+flaky | XFAIL: test2-xfail-xpass
+flaky | XFAIL: test3-xfail-fail
+flaky | XFAIL: test5-xfail-disappears
+flaky | XFAIL: test6-xfail-unsupported
+flaky | XFAIL: test7-xfail-untested
+flaky | XFAIL: test8-xfail-unresolved
+flaky | XFAIL: test9-xfail-kfail
+
+Running /foo/bar/testsuite/tests-5.exp ...
+flaky | UNSUPPORTED: test1-unsupported-pass
+flaky | UNSUPPORTED: test2-unsupported-xpass
+flaky | UNSUPPORTED: test3-unsupported-fail
+flaky | UNSUPPORTED: test4-unsupported-xfail
+flaky | UNSUPPORTED: test5-unsupported-disappears
+flaky | UNSUPPORTED: test6-unsupported-untested
+flaky | UNSUPPORTED: test7-unsupported-unresolved
+flaky | UNSUPPORTED: test9-unsupported-kfail
+
+Running /foo/bar/testsuite/tests-6.exp ...
+flaky | UNTESTED: test1-untested-pass
+flaky | UNTESTED: test2-untested-xpass
+flaky | UNTESTED: test3-untested-fail
+flaky | UNTESTED: test4-untested-xfail
+flaky | UNTESTED: test5-untested-disappears
+flaky | UNTESTED: test6-untested-unsupported
+flaky | UNTESTED: test7-untested-unresolved
+flaky | UNTESTED: test9-untested-kfail
+
+Running /foo/bar/testsuite/tests-7.exp ...
+flaky | UNRESOLVED: test1-unresolved-pass
+flaky | UNRESOLVED: test2-unresolved-xpass
+flaky | UNRESOLVED: test3-unresolved-fail
+flaky | UNRESOLVED: test4-unresolved-xfail
+flaky | UNRESOLVED: test5-unresolved-disappears
+flaky | UNRESOLVED: test6-unresolved-unsupported
+flaky | UNRESOLVED: test7-unresolved-untested
+flaky | UNRESOLVED: test9-unresolved-kfail
+
+Running /foo/bar/testsuite/tests-10.exp ...
+flaky | KFAIL: test1-kfail-pass
+flaky | KFAIL: test2-kfail-xpass
+flaky | KFAIL: test3-kfail-fail
+flaky | KFAIL: test4-kfail-xfail
+flaky | KFAIL: test5-kfail-disappears
+flaky | KFAIL: test6-kfail-unsupported
+flaky | KFAIL: test7-kfail-untested
+flaky | KFAIL: test8-kfail-unresolved
diff --git a/testsuite/sum-1-flaky.txt b/testsuite/sum-1-flaky.txt
new file mode 100644
index 0000000..0320c64
--- /dev/null
+++ b/testsuite/sum-1-flaky.txt
@@ -0,0 +1,117 @@
+# Same as sum-1.txt with non-changing entries removed (e.g. pass -> pass)
+
+ === gcc-compare-results tests ===
+
+Running target mytarget
+Running /foo/bar/testsuite/tests-1.exp ...
+#PASS: test1-pass-pass
+PASS: test2-pass-xpass
+PASS: test3-pass-fail
+PASS: test4-pass-xfail
+PASS: test5-pass-disappears
+PASS: test6-pass-unsupported
+PASS: test7-pass-untested
+PASS: test8-pass-unresolved
+PASS: test9-pass-kfail
+
+Running /foo/bar/testsuite/tests-2.exp ...
+XPASS: test1-xpass-pass
+#XPASS: test2-xpass-xpass
+XPASS: test3-xpass-fail
+XPASS: test4-xpass-xfail
+XPASS: test5-xpass-disappears
+XPASS: test6-xpass-unsupported
+XPASS: test7-xpass-untested
+XPASS: test8-xpass-unresolved
+XPASS: test9-xpass-kfail
+
+Running /foo/bar/testsuite/tests-3.exp ...
+FAIL: test1-fail-pass
+FAIL: test2-fail-xpass
+#FAIL: test3-fail-fail
+FAIL: test4-fail-xfail
+FAIL: test5-fail-disappears
+FAIL: test6-fail-unsupported
+FAIL: test7-fail-untested
+FAIL: test8-fail-unresolved
+FAIL: test9-fail-kfail
+
+Running /foo/bar/testsuite/tests-4.exp ...
+XFAIL: test1-xfail-pass
+XFAIL: test2-xfail-xpass
+XFAIL: test3-xfail-fail
+#XFAIL: test4-xfail-xfail
+XFAIL: test5-xfail-disappears
+XFAIL: test6-xfail-unsupported
+XFAIL: test7-xfail-untested
+XFAIL: test8-xfail-unresolved
+XFAIL: test9-xfail-kfail
+
+Running /foo/bar/testsuite/tests-5.exp ...
+UNSUPPORTED: test1-unsupported-pass
+UNSUPPORTED: test2-unsupported-xpass
+UNSUPPORTED: test3-unsupported-fail
+UNSUPPORTED: test4-unsupported-xfail
+UNSUPPORTED: test5-unsupported-disappears
+UNSUPPORTED: test6-unsupported-untested
+UNSUPPORTED: test7-unsupported-unresolved
+#UNSUPPORTED: test8-unsupported-unsupported
+UNSUPPORTED: test9-unsupported-kfail
+
+Running /foo/bar/testsuite/tests-6.exp ...
+UNTESTED: test1-untested-pass
+UNTESTED: test2-untested-xpass
+UNTESTED: test3-untested-fail
+UNTESTED: test4-untested-xfail
+UNTESTED: test5-untested-disappears
+UNTESTED: test6-untested-unsupported
+UNTESTED: test7-untested-unresolved
+#UNTESTED: test8-untested-untested
+UNTESTED: test9-untested-kfail
+
+Running /foo/bar/testsuite/tests-7.exp ...
+UNRESOLVED: test1-unresolved-pass
+UNRESOLVED: test2-unresolved-xpass
+UNRESOLVED: test3-unresolved-fail
+UNRESOLVED: test4-unresolved-xfail
+UNRESOLVED: test5-unresolved-disappears
+UNRESOLVED: test6-unresolved-unsupported
+UNRESOLVED: test7-unresolved-untested
+#UNRESOLVED: test8-unresolved-unresolved
+UNRESOLVED: test9-unresolved-kfail
+
+Running /foo/bar/testsuite/tests-8.exp ...
+#PASS: test1-pass-and-fails
+#FAIL: test1-pass-and-fails
+
+Running /foo/bar/testsuite/tests-9.exp ...
+#FAIL: test1-fail-pass execution
+
+#ERROR: (DejaGnu) proc "scan-dump-tree-not fail_test optimized 1" does not exist.
+
+Running /foo/bar/testsuite/tests-10.exp ...
+KFAIL: test1-kfail-pass
+KFAIL: test2-kfail-xpass
+KFAIL: test3-kfail-fail
+KFAIL: test4-kfail-xfail
+KFAIL: test5-kfail-disappears
+KFAIL: test6-kfail-unsupported
+KFAIL: test7-kfail-untested
+KFAIL: test8-kfail-unresolved
+#KFAIL: test9-kfail-kfail
+
+#Running /foo/bar/testsuite/hw-dep.exp ...
+#PASS: test1-hwdep-pass-pass
+#PASS: test2-hwdep-pass-xpass
+#PASS: test3-hwdep-pass-fail
+#PASS: test4-hwdep-pass-xfail
+#PASS: test5-hwdep-pass-disappears
+#PASS: test6-hwdep-pass-unsupported
+#PASS: test7-hwdep-pass-untested
+#PASS: test8-hwdep-pass-unresolved
+#PASS: test9-hwdep-pass-kfail
+
+#Running /foo/bar/testsuite/duplicate.exp ...
+#PASS: test1-dup-pass-disappears
+#PASS: test1-dup-pass-disappears
+#PASS: test2-dup-pass-appears
diff --git a/testsuite/sum-2-flaky.txt b/testsuite/sum-2-flaky.txt
new file mode 100644
index 0000000..890fe1a
--- /dev/null
+++ b/testsuite/sum-2-flaky.txt
@@ -0,0 +1,125 @@
+# Same as sum-2.txt with non-changing entries removed (e.g. pass -> pass)
+
+ === gcc-compare-results tests ===
+
+Running target mytarget
+Running /foo/bar/testsuite/tests-1.exp ...
+#PASS: test1-pass-pass
+XPASS: test2-pass-xpass
+FAIL: test3-pass-fail
+XFAIL: test4-pass-xfail
+UNSUPPORTED: test6-pass-unsupported
+UNTESTED: test7-pass-untested
+UNRESOLVED: test8-pass-unresolved
+KFAIL: test9-pass-kfail
+
+Running /foo/bar/testsuite/tests-2.exp ...
+PASS: test1-xpass-pass
+#XPASS: test2-xpass-xpass
+FAIL: test3-xpass-fail
+XFAIL: test4-xpass-xfail
+UNSUPPORTED: test6-xpass-unsupported
+UNTESTED: test7-xpass-untested
+UNRESOLVED: test8-xpass-unresolved
+KFAIL: test9-xpass-kfail
+
+Running /foo/bar/testsuite/tests-3.exp ...
+PASS: test1-fail-pass
+XPASS: test2-fail-xpass
+#FAIL: test3-fail-fail
+XFAIL: test4-fail-xfail
+UNSUPPORTED: test6-fail-unsupported
+UNTESTED: test7-fail-untested
+UNRESOLVED: test8-fail-unresolved
+KFAIL: test9-fail-kfail
+
+Running /foo/bar/testsuite/tests-4.exp ...
+PASS: test1-xfail-pass
+XPASS: test2-xfail-xpass
+FAIL: test3-xfail-fail
+#XFAIL: test4-xfail-xfail
+UNSUPPORTED: test6-xfail-unsupported
+UNTESTED: test7-xfail-untested
+UNRESOLVED: test8-xfail-unresolved
+KFAIL: test9-xfail-kfail
+
+Running /foo/bar/testsuite/tests-appear.exp ...
+#PASS: test1-pass-appears
+#XPASS: test2-xpass-appears
+#FAIL: test3-fail-appears
+#XFAIL: test4-xfail-appears
+#KFAIL: test5-kfail-appears
+
+Running /foo/bar/testsuite/tests-5.exp ...
+PASS: test1-unsupported-pass
+XPASS: test2-unsupported-xpass
+FAIL: test3-unsupported-fail
+XFAIL: test4-unsupported-xfail
+UNTESTED: test6-unsupported-untested
+UNRESOLVED: test7-unsupported-unresolved
+#UNSUPPORTED: test8-unsupported-unsupported
+KFAIL: test9-unsupported-kfail
+
+Running /foo/bar/testsuite/tests-6.exp ...
+PASS: test1-untested-pass
+XPASS: test2-untested-xpass
+FAIL: test3-untested-fail
+XFAIL: test4-untested-xfail
+UNSUPPORTED: test6-untested-unsupported
+UNRESOLVED: test7-untested-unresolved
+#UNTESTED: test8-untested-untested
+KFAIL: test9-untested-kfail
+
+Running /foo/bar/testsuite/tests-7.exp ...
+PASS: test1-unresolved-pass
+XPASS: test2-unresolved-xpass
+FAIL: test3-unresolved-fail
+XFAIL: test4-unresolved-xfail
+UNSUPPORTED: test6-unresolved-unsupported
+UNTESTED: test7-unresolved-untested
+#UNRESOLVED: test8-unresolved-unresolved
+KFAIL: test9-unresolved-kfail
+
+Running /foo/bar/testsuite/tests-appear2.exp ...
+#PASS: test1-pass-appears
+#XPASS: test2-xpass-appears
+#FAIL: test3-fail-appears
+#XFAIL: test4-xfail-appears
+#UNSUPPORTED: test5-unsupported-appears
+#UNTESTED: test6-untested-appears
+#UNRESOLVED: test7-unresolved-appears
+#KFAIL: test8-kfail-appears
+
+Running /foo/bar/testsuite/tests-8.exp ...
+#PASS: test1-pass-and-fails
+#FAIL: test1-pass-and-fails
+
+Running /foo/bar/testsuite/tests-9.exp ...
+#PASS: test1-fail-pass execution
+
+#ERROR: (DejaGnu) proc "scan-dump-tree-not fail_test optimized 3" does not exist.
+
+Running /foo/bar/testsuite/tests-10.exp ...
+PASS: test1-kfail-pass
+XPASS: test2-kfail-xpass
+FAIL: test3-kfail-fail
+XFAIL: test4-kfail-xfail
+UNSUPPORTED: test6-kfail-unsupported
+UNTESTED: test7-kfail-untested
+UNRESOLVED: test8-kfail-unresolved
+#KFAIL: test9-kfail-kfail
+
+Running /foo/bar/testsuite/hw-dep.exp ...
+#PASS: test1-hwdep-pass-pass
+#XPASS: test2-hwdep-pass-xpass
+#FAIL: test3-hwdep-pass-fail
+#XFAIL: test4-hwdep-pass-xfail
+#UNSUPPORTED: test6-hwdep-pass-unsupported
+#UNTESTED: test7-hwdep-pass-untested
+#UNRESOLVED: test8-hwdep-pass-unresolved
+#KFAIL: test9-hwdep-pass-kfail
+
+Running /foo/bar/testsuite/duplicate.exp ...
+#PASS: test1-dup-pass-disappears
+#PASS: test2-dup-pass-appears
+#PASS: test2-dup-pass-appears