Add support for the KFAIL status
The GDB testsuite has some tests which fail with the KFAIL status, which is
when there's a bugzilla report filed for that given failure.
Change-Id: I94d3cb2ed6aa1432a956c75ce1754a051ae7c7b2
diff --git a/compare_dg_tests.pl b/compare_dg_tests.pl
index 772eca1..6eec385 100755
--- a/compare_dg_tests.pl
+++ b/compare_dg_tests.pl
@@ -22,6 +22,7 @@
my $PASS_XPASS = "PASS now XPASS [PASS =>XPASS]";
my $PASS_FAIL = "PASS now FAIL [PASS => FAIL]";
my $PASS_XFAIL = "PASS now XFAIL [PASS =>XFAIL]";
+my $PASS_KFAIL = "PASS now KFAIL [PASS =>KFAIL]";
my $PASS_UNSUPPORTED = "PASS now UNSUPPORTED [PASS =>UNSUP]";
my $PASS_UNRESOLVED = "PASS now UNRESOLVED [PASS =>UNRES]";
my $PASS_UNTESTED = "PASS now UNTESTED [PASS =>UNTES]";
@@ -33,6 +34,7 @@
my $XPASS_XPASS = "Still xpass [XPASS=>XPASS]";
my $XPASS_FAIL = "XPASS now FAIL [XPASS=> FAIL]";
my $XPASS_XFAIL = "XPASS now XFAIL [XPASS=>XFAIL]";
+my $XPASS_KFAIL = "XPASS now KFAIL [XPASS=>KFAIL]";
my $XPASS_UNSUPPORTED = "XPASS now UNSUPPORTED [XPASS=>UNSUP]";
my $XPASS_UNRESOLVED = "XPASS now UNRESOLVED [XPASS=>UNRES]";
my $XPASS_UNTESTED = "XPASS now UNTESTED [XPASS=>UNTES]";
@@ -43,6 +45,7 @@
my $FAIL_XPASS = "FAIL now XPASS [FAIL =>XPASS]";
my $FAIL_FAIL = "Still fails [FAIL => FAIL]";
my $FAIL_XFAIL = "FAIL now XFAIL [FAIL =>XFAIL]";
+my $FAIL_KFAIL = "FAIL now KFAIL [FAIL =>KFAIL]";
my $FAIL_UNSUPPORTED = "FAIL now UNSUPPORTED [FAIL =>UNSUP]";
my $FAIL_UNRESOLVED = "FAIL now UNRESOLVED [FAIL =>UNRES]";
my $FAIL_UNTESTED = "FAIL now UNTESTED [FAIL =>UNTES]";
@@ -53,16 +56,29 @@
my $XFAIL_XPASS = "XFAIL now XPASS [XFAIL=>XPASS]";
my $XFAIL_FAIL = "XFAIL now FAIL [XFAIL=> FAIL]";
my $XFAIL_XFAIL = "Still xfail [XFAIL=>XFAIL]";
+my $XFAIL_KFAIL = "XFAIL now KFAIL [XFAIL=>KFAIL]";
my $XFAIL_UNSUPPORTED = "XFAIL now UNSUPPORTED [XFAIL=>UNSUP]";
my $XFAIL_UNRESOLVED = "XFAIL now UNRESOLVED [XFAIL=>UNRES]";
my $XFAIL_UNTESTED = "XFAIL now UNTESTED [XFAIL=>UNTES]";
my $XFAIL_DISAPPEARS = "XFAIL disappears [XFAIL=> ]";
my $XFAIL_APPEARS = "XFAIL appears [ =>XFAIL]";
+my $KFAIL_PASS = "KFAIL now PASS [KFAIL=> PASS]";
+my $KFAIL_XPASS = "KFAIL now XPASS [KFAIL=>XPASS]";
+my $KFAIL_FAIL = "KFAIL now FAIL [KFAIL=> FAIL]";
+my $KFAIL_XFAIL = "KFAIL now XFAIL [KFAIL=>XFAIL]";
+my $KFAIL_KFAIL = "Still kfail [KFAIL=>KFAIL]";
+my $KFAIL_UNSUPPORTED = "KFAIL now UNSUPPORTED [KFAIL=>UNSUP]";
+my $KFAIL_UNRESOLVED = "KFAIL now UNRESOLVED [KFAIL=>UNRES]";
+my $KFAIL_UNTESTED = "KFAIL now UNTESTED [KFAIL=>UNTES]";
+my $KFAIL_DISAPPEARS = "KFAIL disappears [KFAIL=> ]";
+my $KFAIL_APPEARS = "KFAIL appears [ =>KFAIL]";
+
my $UNSUPPORTED_PASS = "UNSUPPORTED now PASS [UNSUP=> PASS]";
my $UNSUPPORTED_XPASS = "UNSUPPORTED now XPASS [UNSUP=>XPASS]";
my $UNSUPPORTED_FAIL = "UNSUPPORTED now FAIL [UNSUP=> FAIL]";
my $UNSUPPORTED_XFAIL = "UNSUPPORTED now XFAIL [UNSUP=>XFAIL]";
+my $UNSUPPORTED_KFAIL = "UNSUPPORTED now KFAIL [UNSUP=>KFAIL]";
my $UNSUPPORTED_UNSUPPORTED= "UNSUPP now UNSUPP [UNSUP=>UNSUP]";
my $UNSUPPORTED_UNRESOLVED= "UNSUPP now UNRESOLVED [UNSUP=>UNRES]";
my $UNSUPPORTED_UNTESTED = "UNSUPP now UNTESTED [UNSUP=>UNTES]";
@@ -73,6 +89,7 @@
my $UNTESTED_XPASS = "UNTESTED now XPASS [UNTES=>XPASS]";
my $UNTESTED_FAIL = "UNTESTED now FAIL [UNTES=> FAIL]";
my $UNTESTED_XFAIL = "UNTESTED now XFAIL [UNTES=>XFAIL]";
+my $UNTESTED_KFAIL = "UNTESTED now KFAIL [UNTES=>KFAIL]";
my $UNTESTED_UNSUPPORTED = "UNTESTED now UNSUPP [UNTES=>UNSUP]";
my $UNTESTED_UNRESOLVED = "UNTESTED now UNRESOLVED [UNTES=>UNRES]";
my $UNTESTED_UNTESTED = "UNTESTED now UNTESTED [UNTES=>UNTES]";
@@ -83,6 +100,7 @@
my $UNRESOLVED_XPASS = "UNRESOLVED now XPASS [UNRES=>XPASS]";
my $UNRESOLVED_FAIL = "UNRESOLVED now FAIL [UNRES=> FAIL]";
my $UNRESOLVED_XFAIL = "UNRESOLVED now XFAIL [UNRES=>XFAIL]";
+my $UNRESOLVED_KFAIL = "UNRESOLVED now KFAIL [UNRES=>KFAIL]";
my $UNRESOLVED_UNSUPPORTED= "UNRESOLVED now UNSUPP [UNRES=>UNSUP]";
my $UNRESOLVED_UNRESOLVED = "UNRESOLVED now UNRESOLVED [UNRES=>UNRES]";
my $UNRESOLVED_UNTESTED = "UNRESOLVED now UNTESTED [UNRES=>UNTES]";
@@ -108,6 +126,7 @@
{was=>"PASS", is=>"XPASS", cat=>$PASS_XPASS, report_hwdep=>"true"},
{was=>"PASS", is=>"FAIL", handler=>\&handle_pass_fail, report_hwdep=>"true"},
{was=>"PASS", is=>"XFAIL", cat=>$PASS_XFAIL, report_hwdep=>"true"},
+ {was=>"PASS", is=>"KFAIL", cat=>$PASS_KFAIL, report_hwdep=>"true"},
{was=>"PASS", is=>"UNSUPPORTED",cat=>$PASS_UNSUPPORTED, report_hwdep=>"false"},
{was=>"PASS", is=>"UNTESTED", cat=>$PASS_UNTESTED, report_hwdep=>"true"},
{was=>"PASS", is=>"UNRESOLVED",cat=>$PASS_UNRESOLVED, report_hwdep=>"true"},
@@ -118,6 +137,7 @@
{was=>"XPASS", is=>"XPASS", cat=>$XPASS_XPASS, report_hwdep=>"true"},
{was=>"XPASS", is=>"FAIL", cat=>$XPASS_FAIL, report_hwdep=>"true"},
{was=>"XPASS", is=>"XFAIL", cat=>$XPASS_XFAIL, report_hwdep=>"true"},
+ {was=>"XPASS", is=>"KFAIL", cat=>$XPASS_KFAIL, report_hwdep=>"true"},
{was=>"XPASS", is=>"UNSUPPORTED",cat=>$XPASS_UNSUPPORTED, report_hwdep=>"true"},
{was=>"XPASS", is=>"UNTESTED", cat=>$XPASS_UNTESTED, report_hwdep=>"true"},
{was=>"XPASS", is=>"UNRESOLVED",cat=>$XPASS_UNRESOLVED, report_hwdep=>"true"},
@@ -128,6 +148,7 @@
{was=>"FAIL", is=>"XPASS", cat=>$FAIL_XPASS, report_hwdep=>"true"},
{was=>"FAIL", is=>"FAIL", cat=>$FAIL_FAIL, report_hwdep=>"true"},
{was=>"FAIL", is=>"XFAIL", cat=>$FAIL_XFAIL, report_hwdep=>"true"},
+ {was=>"FAIL", is=>"KFAIL", cat=>$FAIL_KFAIL, report_hwdep=>"true"},
{was=>"FAIL", is=>"UNSUPPORTED",cat=>$FAIL_UNSUPPORTED, report_hwdep=>"true"},
{was=>"FAIL", is=>"UNTESTED", cat=>$FAIL_UNTESTED, report_hwdep=>"true"},
{was=>"FAIL", is=>"UNRESOLVED",cat=>$FAIL_UNRESOLVED, report_hwdep=>"true"},
@@ -138,16 +159,29 @@
{was=>"XFAIL", is=>"XPASS", cat=>$XFAIL_XPASS, report_hwdep=>"true"},
{was=>"XFAIL", is=>"FAIL", cat=>$XFAIL_FAIL, report_hwdep=>"true"},
{was=>"XFAIL", is=>"XFAIL", cat=>$XFAIL_XFAIL, report_hwdep=>"true"},
+ {was=>"XFAIL", is=>"KFAIL", cat=>$XFAIL_KFAIL, report_hwdep=>"true"},
{was=>"XFAIL", is=>"UNSUPPORTED",cat=>$XFAIL_UNSUPPORTED, report_hwdep=>"true"},
{was=>"XFAIL", is=>"UNTESTED", cat=>$XFAIL_UNTESTED, report_hwdep=>"true"},
{was=>"XFAIL", is=>"UNRESOLVED",cat=>$XFAIL_UNRESOLVED, report_hwdep=>"true"},
{was=>"XFAIL", is=>"NO_EXIST", cat=>$XFAIL_DISAPPEARS, report_hwdep=>"true"},
{was=>"NO_EXIST", is=>"XFAIL", cat=>$XFAIL_APPEARS, report_hwdep=>"true"},
+ {was=>"KFAIL", is=>"PASS", cat=>$KFAIL_PASS, report_hwdep=>"true"},
+ {was=>"KFAIL", is=>"XPASS", cat=>$KFAIL_XPASS, report_hwdep=>"true"},
+ {was=>"KFAIL", is=>"FAIL", cat=>$KFAIL_FAIL, report_hwdep=>"true"},
+ {was=>"KFAIL", is=>"XFAIL", cat=>$KFAIL_XFAIL, report_hwdep=>"true"},
+ {was=>"KFAIL", is=>"KFAIL", cat=>$KFAIL_KFAIL, report_hwdep=>"true"},
+ {was=>"KFAIL", is=>"UNSUPPORTED",cat=>$KFAIL_UNSUPPORTED, report_hwdep=>"true"},
+ {was=>"KFAIL", is=>"UNTESTED", cat=>$KFAIL_UNTESTED, report_hwdep=>"true"},
+ {was=>"KFAIL", is=>"UNRESOLVED",cat=>$KFAIL_UNRESOLVED, report_hwdep=>"true"},
+ {was=>"KFAIL", is=>"NO_EXIST", cat=>$KFAIL_DISAPPEARS, report_hwdep=>"true"},
+ {was=>"NO_EXIST", is=>"KFAIL", cat=>$KFAIL_APPEARS, report_hwdep=>"true"},
+
{was=>"UNSUPPORTED", is=>"PASS", cat=>$UNSUPPORTED_PASS, report_hwdep=>"true"},
{was=>"UNSUPPORTED", is=>"XPASS", cat=>$UNSUPPORTED_XPASS, report_hwdep=>"true"},
{was=>"UNSUPPORTED", is=>"FAIL", cat=>$UNSUPPORTED_FAIL, report_hwdep=>"true"},
{was=>"UNSUPPORTED", is=>"XFAIL", cat=>$UNSUPPORTED_XFAIL, report_hwdep=>"true"},
+ {was=>"UNSUPPORTED", is=>"KFAIL", cat=>$UNSUPPORTED_KFAIL, report_hwdep=>"true"},
{was=>"UNSUPPORTED", is=>"UNSUPPORTED",cat=>$UNSUPPORTED_UNSUPPORTED, report_hwdep=>"true"},
{was=>"UNSUPPORTED", is=>"UNTESTED", cat=>$UNSUPPORTED_UNTESTED, report_hwdep=>"true"},
{was=>"UNSUPPORTED", is=>"UNRESOLVED",cat=>$UNSUPPORTED_UNRESOLVED, report_hwdep=>"true"},
@@ -158,6 +192,7 @@
{was=>"UNTESTED", is=>"XPASS", cat=>$UNTESTED_XPASS, report_hwdep=>"true"},
{was=>"UNTESTED", is=>"FAIL", cat=>$UNTESTED_FAIL, report_hwdep=>"true"},
{was=>"UNTESTED", is=>"XFAIL", cat=>$UNTESTED_XFAIL, report_hwdep=>"true"},
+ {was=>"UNTESTED", is=>"KFAIL", cat=>$UNTESTED_KFAIL, report_hwdep=>"true"},
{was=>"UNTESTED", is=>"UNSUPPORTED",cat=>$UNTESTED_UNSUPPORTED,report_hwdep=>"true"},
{was=>"UNTESTED", is=>"UNTESTED", cat=>$UNTESTED_UNTESTED, report_hwdep=>"true"},
{was=>"UNTESTED", is=>"UNRESOLVED",cat=>$UNTESTED_UNRESOLVED, report_hwdep=>"true"},
@@ -168,6 +203,7 @@
{was=>"UNRESOLVED", is=>"XPASS", cat=>$UNRESOLVED_XPASS, report_hwdep=>"true"},
{was=>"UNRESOLVED", is=>"FAIL", cat=>$UNRESOLVED_FAIL, report_hwdep=>"true"},
{was=>"UNRESOLVED", is=>"XFAIL", cat=>$UNRESOLVED_XFAIL, report_hwdep=>"true"},
+ {was=>"UNRESOLVED", is=>"KFAIL", cat=>$UNRESOLVED_KFAIL, report_hwdep=>"true"},
{was=>"UNRESOLVED", is=>"UNSUPPORTED",cat=>$UNRESOLVED_UNSUPPORTED, report_hwdep=>"true"},
{was=>"UNRESOLVED", is=>"UNTESTED", cat=>$UNRESOLVED_UNTESTED, report_hwdep=>"true"},
{was=>"UNRESOLVED", is=>"UNRESOLVED",cat=>$UNRESOLVED_UNRESOLVED, report_hwdep=>"true"},
@@ -256,8 +292,8 @@
sub empty_result()
{
- my %empty_result;# = {PASS=>0, FAIL=>0, XPASS=>0, XFAIL=>0, UNSUPPORTED=>0, UNTESTED=>0, UNRESOLVED=>0, ERROR=>0};
- $empty_result{PASS}=$empty_result{FAIL}=$empty_result{XPASS}=$empty_result{XFAIL}=0;
+ my %empty_result;# = {PASS=>0, FAIL=>0, XPASS=>0, XFAIL=>0, KFAIL=>0, UNSUPPORTED=>0, UNTESTED=>0, UNRESOLVED=>0, ERROR=>0};
+ $empty_result{PASS}=$empty_result{FAIL}=$empty_result{XPASS}=$empty_result{XFAIL}=$empty_result{KFAIL}=0;
$empty_result{UNSUPPORTED}=$empty_result{UNTESTED}=$empty_result{UNRESOLVED}=$empty_result{NO_EXIST}=$empty_result{ERROR}=0;
$empty_result{NB}=0;
return \%empty_result;
@@ -291,7 +327,7 @@
open SUMFILE, $sum_file or die $!;
while (<SUMFILE>)
{
- if (m/^(PASS|XPASS|FAIL|XFAIL|UNSUPPORTED|UNTESTED|UNRESOLVED|ERROR): *(.*)/)
+ if (m/^(PASS|XPASS|FAIL|XFAIL|KFAIL|UNSUPPORTED|UNTESTED|UNRESOLVED|ERROR): *(.*)/)
{
my ($diag,$tc) = ($1,$2);
my %tcresult;
@@ -421,6 +457,7 @@
@{$res->{$PASS_XPASS}} = ();
@{$res->{$PASS_FAIL}} = ();
@{$res->{$PASS_XFAIL}} = ();
+ @{$res->{$PASS_KFAIL}} = ();
@{$res->{$PASS_UNSUPPORTED}} = ();
@{$res->{$PASS_UNTESTED}} = ();
@{$res->{$PASS_UNRESOLVED}} = ();
@@ -432,6 +469,7 @@
@{$res->{$XPASS_XPASS}} = ();
@{$res->{$XPASS_FAIL}} = ();
@{$res->{$XPASS_XFAIL}} = ();
+ @{$res->{$XPASS_KFAIL}} = ();
@{$res->{$XPASS_UNSUPPORTED}} = ();
@{$res->{$XPASS_UNTESTED}} = ();
@{$res->{$XPASS_UNRESOLVED}} = ();
@@ -442,6 +480,7 @@
@{$res->{$FAIL_XPASS}} = ();
@{$res->{$FAIL_FAIL}} = ();
@{$res->{$FAIL_XFAIL}} = ();
+ @{$res->{$FAIL_KFAIL}} = ();
@{$res->{$FAIL_UNSUPPORTED}} = ();
@{$res->{$FAIL_UNTESTED}} = ();
@{$res->{$FAIL_UNRESOLVED}} = ();
@@ -452,16 +491,29 @@
@{$res->{$XFAIL_XPASS}} = ();
@{$res->{$XFAIL_FAIL}} = ();
@{$res->{$XFAIL_XFAIL}} = ();
+ @{$res->{$XFAIL_KFAIL}} = ();
@{$res->{$XFAIL_UNSUPPORTED}} = ();
@{$res->{$XFAIL_UNTESTED}} = ();
@{$res->{$XFAIL_UNRESOLVED}} = ();
@{$res->{$XFAIL_DISAPPEARS}} = ();
@{$res->{$XFAIL_APPEARS}} = ();
+ @{$res->{$KFAIL_PASS}} = ();
+ @{$res->{$KFAIL_XPASS}} = ();
+ @{$res->{$KFAIL_FAIL}} = ();
+ @{$res->{$KFAIL_XFAIL}} = ();
+ @{$res->{$KFAIL_KFAIL}} = ();
+ @{$res->{$KFAIL_UNSUPPORTED}} = ();
+ @{$res->{$KFAIL_UNTESTED}} = ();
+ @{$res->{$KFAIL_UNRESOLVED}} = ();
+ @{$res->{$KFAIL_DISAPPEARS}} = ();
+ @{$res->{$KFAIL_APPEARS}} = ();
+
@{$res->{$UNSUPPORTED_PASS}} = ();
@{$res->{$UNSUPPORTED_XPASS}} = ();
@{$res->{$UNSUPPORTED_FAIL}} = ();
@{$res->{$UNSUPPORTED_XFAIL}} = ();
+ @{$res->{$UNSUPPORTED_KFAIL}} = ();
@{$res->{$UNSUPPORTED_UNSUPPORTED}} = ();
@{$res->{$UNSUPPORTED_UNTESTED}} = ();
@{$res->{$UNSUPPORTED_UNRESOLVED}} = ();
@@ -472,6 +524,7 @@
@{$res->{$UNTESTED_XPASS}} = ();
@{$res->{$UNTESTED_FAIL}} = ();
@{$res->{$UNTESTED_XFAIL}} = ();
+ @{$res->{$UNTESTED_KFAIL}} = ();
@{$res->{$UNTESTED_UNSUPPORTED}} = ();
@{$res->{$UNTESTED_UNTESTED}} = ();
@{$res->{$UNTESTED_UNRESOLVED}} = ();
@@ -482,6 +535,7 @@
@{$res->{$UNRESOLVED_XPASS}} = ();
@{$res->{$UNRESOLVED_FAIL}} = ();
@{$res->{$UNRESOLVED_XFAIL}} = ();
+ @{$res->{$UNRESOLVED_KFAIL}} = ();
@{$res->{$UNRESOLVED_UNSUPPORTED}} = ();
@{$res->{$UNRESOLVED_UNTESTED}} = ();
@{$res->{$UNRESOLVED_UNRESOLVED}} = ();
@@ -619,6 +673,7 @@
# Compute the pass ratio as a sanity check
$ref_total = $ref->{PASS} +
$ref->{XFAIL} +
+ $ref->{KFAIL} +
$ref->{XPASS} +
$ref->{FAIL} +
$ref->{UNRESOLVED} +
@@ -634,6 +689,7 @@
$res_total = $res->{PASS} +
$res->{XFAIL} +
+ $res->{KFAIL} +
$res->{XPASS} +
$res->{FAIL} +
$res->{UNRESOLVED} +
@@ -663,6 +719,7 @@
printf " | %-40s | %7d | %7d |\n", "Errors [ERROR]", $ref->{ERROR}, $res->{ERROR};
printf " | %-40s | %7d | %7d |\n", "Unexpected passes [XPASS]", $ref->{XPASS}, $res->{XPASS};
printf " | %-40s | %7d | %7d |\n", "Expected fails [XFAIL]", $ref->{XFAIL}, $res->{XFAIL};
+ printf " | %-40s | %7d | %7d |\n", "Known fails [KFAIL]", $ref->{KFAIL}, $res->{KFAIL};
printf " | %-40s | %7d | %7d |\n", "Unresolved [UNRESOLVED]", $ref->{UNRESOLVED}, $res->{UNRESOLVED};
printf " | %-40s | %7d | %7d |\n", "Unsupported [UNSUPPORTED]", $ref->{UNSUPPORTED}, $res->{UNSUPPORTED};
printf " | %-40s | %7d | %7d |\n", "Untested [UNTESTED]", $ref->{UNTESTED}, $res->{UNTESTED};
@@ -704,23 +761,30 @@
#### REGRESSIONS ?
$rtotal = scalar(@{$res->{$PASS_XPASS}})
+scalar(@{$res->{$PASS_FAIL}})
+ +scalar(@{$res->{$PASS_KFAIL}})
+scalar(@{$res->{$PASS_UNRESOLVED}})
+scalar(@{$res->{$PASSED_NOW_TIMEOUTS}})
+scalar(@{$res->{$ERROR_APPEARS}})
+scalar(@{$res->{$XPASS_FAIL}})
+ +scalar(@{$res->{$XPASS_KFAIL}})
+scalar(@{$res->{$XPASS_APPEARS}})
+scalar(@{$res->{$XPASS_UNRESOLVED}})
+scalar(@{$res->{$FAIL_APPEARS}})
+scalar(@{$res->{$FAIL_UNRESOLVED}})
+scalar(@{$res->{$XFAIL_FAIL}})
+scalar(@{$res->{$XFAIL_UNRESOLVED}})
+ +scalar(@{$res->{$KFAIL_APPEARS}})
+ +scalar(@{$res->{$KFAIL_UNRESOLVED}})
+scalar(@{$res->{$UNSUPPORTED_FAIL}})
+ +scalar(@{$res->{$UNSUPPORTED_KFAIL}})
+scalar(@{$res->{$UNSUPPORTED_XPASS}})
+scalar(@{$res->{$UNSUPPORTED_UNRESOLVED}})
+scalar(@{$res->{$UNTESTED_FAIL}})
+ +scalar(@{$res->{$UNTESTED_KFAIL}})
+scalar(@{$res->{$UNTESTED_XPASS}})
+scalar(@{$res->{$UNTESTED_UNRESOLVED}})
+scalar(@{$res->{$UNRESOLVED_FAIL}})
+ +scalar(@{$res->{$UNRESOLVED_KFAIL}})
+scalar(@{$res->{$UNRESOLVED_XPASS}})
+scalar(@{$res->{$UNRESOLVED_APPEARS}});
@@ -732,23 +796,30 @@
printf " +------------------------------------------+---------+\n";
printf " | %-40s | %7d |\n", $PASS_XPASS, scalar(@{$res->{$PASS_XPASS}}) if (scalar(@{$res->{$PASS_XPASS}}));
printf " | %-40s | %7d |\n", $PASS_FAIL, scalar(@{$res->{$PASS_FAIL}}) if (scalar(@{$res->{$PASS_FAIL}}));
+ printf " | %-40s | %7d |\n", $PASS_KFAIL, scalar(@{$res->{$PASS_KFAIL}}) if (scalar(@{$res->{$PASS_KFAIL}}));
printf " | %-40s | %7d |\n", $PASS_UNRESOLVED, scalar(@{$res->{$PASS_UNRESOLVED}}) if (scalar(@{$res->{$PASS_UNRESOLVED}}));
printf " | %-40s | %7d |\n", $PASSED_NOW_TIMEOUTS, scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}) if (scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}));
printf " | %-40s | %7d |\n", $ERROR_APPEARS, scalar(@{$res->{$ERROR_APPEARS}}) if (scalar(@{$res->{$ERROR_APPEARS}}));
printf " | %-40s | %7d |\n", $XPASS_FAIL, scalar(@{$res->{$XPASS_FAIL}}) if (scalar(@{$res->{$XPASS_FAIL}}));
+ printf " | %-40s | %7d |\n", $XPASS_KFAIL, scalar(@{$res->{$XPASS_KFAIL}}) if (scalar(@{$res->{$XPASS_KFAIL}}));
printf " | %-40s | %7d |\n", $XPASS_APPEARS, scalar(@{$res->{$XPASS_APPEARS}}) if (scalar(@{$res->{$XPASS_APPEARS}}));
printf " | %-40s | %7d |\n", $XPASS_UNRESOLVED, scalar(@{$res->{$XPASS_UNRESOLVED}}) if (scalar(@{$res->{$XPASS_UNRESOLVED}}));
printf " | %-40s | %7d |\n", $FAIL_APPEARS, scalar(@{$res->{$FAIL_APPEARS}}) if (scalar(@{$res->{$FAIL_APPEARS}}));
printf " | %-40s | %7d |\n", $FAIL_UNRESOLVED, scalar(@{$res->{$FAIL_UNRESOLVED}}) if (scalar(@{$res->{$FAIL_UNRESOLVED}}));
printf " | %-40s | %7d |\n", $XFAIL_FAIL, scalar(@{$res->{$XFAIL_FAIL}}) if (scalar(@{$res->{$XFAIL_FAIL}}));
printf " | %-40s | %7d |\n", $XFAIL_UNRESOLVED, scalar(@{$res->{$XFAIL_UNRESOLVED}}) if (scalar(@{$res->{$XFAIL_UNRESOLVED}}));
+ printf " | %-40s | %7d |\n", $KFAIL_APPEARS, scalar(@{$res->{$KFAIL_APPEARS}}) if (scalar(@{$res->{$KFAIL_APPEARS}}));
+ printf " | %-40s | %7d |\n", $KFAIL_UNRESOLVED, scalar(@{$res->{$KFAIL_UNRESOLVED}}) if (scalar(@{$res->{$KFAIL_UNRESOLVED}}));
printf " | %-40s | %7d |\n", $UNSUPPORTED_FAIL, scalar(@{$res->{$UNSUPPORTED_FAIL}}) if (scalar(@{$res->{$UNSUPPORTED_FAIL}}));
+ printf " | %-40s | %7d |\n", $UNSUPPORTED_KFAIL, scalar(@{$res->{$UNSUPPORTED_KFAIL}}) if (scalar(@{$res->{$UNSUPPORTED_KFAIL}}));
printf " | %-40s | %7d |\n", $UNSUPPORTED_XPASS, scalar(@{$res->{$UNSUPPORTED_XPASS}}) if (scalar(@{$res->{$UNSUPPORTED_XPASS}}));
printf " | %-40s | %7d |\n", $UNSUPPORTED_UNRESOLVED, scalar(@{$res->{$UNSUPPORTED_UNRESOLVED}}) if (scalar(@{$res->{$UNSUPPORTED_UNRESOLVED}}));
printf " | %-40s | %7d |\n", $UNTESTED_FAIL, scalar(@{$res->{$UNTESTED_FAIL}}) if (scalar(@{$res->{$UNTESTED_FAIL}}));
+ printf " | %-40s | %7d |\n", $UNTESTED_KFAIL, scalar(@{$res->{$UNTESTED_KFAIL}}) if (scalar(@{$res->{$UNTESTED_KFAIL}}));
printf " | %-40s | %7d |\n", $UNTESTED_XPASS, scalar(@{$res->{$UNTESTED_XPASS}}) if (scalar(@{$res->{$UNTESTED_XPASS}}));
printf " | %-40s | %7d |\n", $UNTESTED_UNRESOLVED, scalar(@{$res->{$UNTESTED_UNRESOLVED}}) if (scalar(@{$res->{$UNTESTED_UNRESOLVED}}));
printf " | %-40s | %7d |\n", $UNRESOLVED_FAIL, scalar(@{$res->{$UNRESOLVED_FAIL}}) if (scalar(@{$res->{$UNRESOLVED_FAIL}}));
+ printf " | %-40s | %7d |\n", $UNRESOLVED_KFAIL, scalar(@{$res->{$UNRESOLVED_KFAIL}}) if (scalar(@{$res->{$UNRESOLVED_KFAIL}}));
printf " | %-40s | %7d |\n", $UNRESOLVED_XPASS, scalar(@{$res->{$UNRESOLVED_XPASS}}) if (scalar(@{$res->{$UNRESOLVED_XPASS}}));
printf " | %-40s | %7d |\n", $UNRESOLVED_APPEARS, scalar(@{$res->{$UNRESOLVED_APPEARS}}) if (scalar(@{$res->{$UNRESOLVED_APPEARS}}));
printf " +------------------------------------------+---------+\n";
@@ -760,23 +831,30 @@
{
print_tclist($res, $PASS_XPASS, @{$res->{$PASS_XPASS}});
print_tclist($res, $PASS_FAIL, @{$res->{$PASS_FAIL}});
+ print_tclist($res, $PASS_KFAIL, @{$res->{$PASS_KFAIL}});
print_tclist($res, $PASS_UNRESOLVED, @{$res->{$PASS_UNRESOLVED}});
print_tclist($res, $PASSED_NOW_TIMEOUTS, @{$res->{$PASSED_NOW_TIMEOUTS}});
print_tclist($res, $ERROR_APPEARS, @{$res->{$ERROR_APPEARS}});
print_tclist($res, $XPASS_FAIL, @{$res->{$XPASS_FAIL}});
+ print_tclist($res, $XPASS_KFAIL, @{$res->{$XPASS_KFAIL}});
print_tclist($res, $XPASS_APPEARS, @{$res->{$XPASS_APPEARS}});
print_tclist($res, $XPASS_UNRESOLVED, @{$res->{$XPASS_UNRESOLVED}});
print_tclist($res, $FAIL_APPEARS, @{$res->{$FAIL_APPEARS}});
print_tclist($res, $FAIL_UNRESOLVED, @{$res->{$FAIL_UNRESOLVED}});
print_tclist($res, $XFAIL_FAIL, @{$res->{$XFAIL_FAIL}});
print_tclist($res, $XFAIL_UNRESOLVED, @{$res->{$XFAIL_UNRESOLVED}});
+ print_tclist($res, $KFAIL_APPEARS, @{$res->{$KFAIL_APPEARS}});
+ print_tclist($res, $KFAIL_UNRESOLVED, @{$res->{$KFAIL_UNRESOLVED}});
print_tclist($res, $UNSUPPORTED_FAIL, @{$res->{$UNSUPPORTED_FAIL}});
+ print_tclist($res, $UNSUPPORTED_KFAIL, @{$res->{$UNSUPPORTED_KFAIL}});
print_tclist($res, $UNSUPPORTED_XPASS, @{$res->{$UNSUPPORTED_XPASS}});
print_tclist($res, $UNSUPPORTED_UNRESOLVED, @{$res->{$UNSUPPORTED_UNRESOLVED}});
print_tclist($res, $UNTESTED_FAIL, @{$res->{$UNTESTED_FAIL}});
+ print_tclist($res, $UNTESTED_KFAIL, @{$res->{$UNTESTED_KFAIL}});
print_tclist($res, $UNTESTED_XPASS, @{$res->{$UNTESTED_XPASS}});
print_tclist($res, $UNTESTED_UNRESOLVED, @{$res->{$UNTESTED_UNRESOLVED}});
print_tclist($res, $UNRESOLVED_FAIL, @{$res->{$UNRESOLVED_FAIL}});
+ print_tclist($res, $UNRESOLVED_KFAIL, @{$res->{$UNRESOLVED_KFAIL}});
print_tclist($res, $UNRESOLVED_XPASS, @{$res->{$UNRESOLVED_XPASS}});
print_tclist($res, $UNRESOLVED_APPEARS, @{$res->{$UNRESOLVED_APPEARS}});
}
@@ -809,6 +887,12 @@
scalar(@{$res->{$XFAIL_UNSUPPORTED}})+
scalar(@{$res->{$XFAIL_UNTESTED}})+
scalar(@{$res->{$XFAIL_APPEARS}})+
+ scalar(@{$res->{$KFAIL_PASS}})+
+ scalar(@{$res->{$KFAIL_XPASS}})+
+ scalar(@{$res->{$KFAIL_XFAIL}})+
+ scalar(@{$res->{$KFAIL_UNSUPPORTED}})+
+ scalar(@{$res->{$KFAIL_UNTESTED}})+
+ scalar(@{$res->{$KFAIL_DISAPPEARS}})+
scalar(@{$res->{$UNSUPPORTED_PASS}})+
scalar(@{$res->{$UNSUPPORTED_XFAIL}})+
scalar(@{$res->{$UNSUPPORTED_UNTESTED}})+
@@ -851,6 +935,12 @@
printf " | %-40s | %7d |\n", $XFAIL_UNSUPPORTED, scalar(@{$res->{$XFAIL_UNSUPPORTED}}) if (scalar(@{$res->{$XFAIL_UNSUPPORTED}}));
printf " | %-40s | %7d |\n", $XFAIL_UNTESTED, scalar(@{$res->{$XFAIL_UNTESTED}}) if (scalar(@{$res->{$XFAIL_UNTESTED}}));
printf " | %-40s | %7d |\n", $XFAIL_APPEARS, scalar(@{$res->{$XFAIL_APPEARS}}) if (scalar(@{$res->{$XFAIL_APPEARS}}));
+ printf " | %-40s | %7d |\n", $KFAIL_PASS, scalar(@{$res->{$KFAIL_PASS}}) if (scalar(@{$res->{$KFAIL_PASS}}));
+ printf " | %-40s | %7d |\n", $KFAIL_XPASS, scalar(@{$res->{$KFAIL_XPASS}}) if (scalar(@{$res->{$KFAIL_XPASS}}));
+ printf " | %-40s | %7d |\n", $KFAIL_XFAIL, scalar(@{$res->{$KFAIL_XFAIL}}) if (scalar(@{$res->{$KFAIL_XFAIL}}));
+ printf " | %-40s | %7d |\n", $KFAIL_UNSUPPORTED, scalar(@{$res->{$KFAIL_UNSUPPORTED}}) if (scalar(@{$res->{$KFAIL_UNSUPPORTED}}));
+ printf " | %-40s | %7d |\n", $KFAIL_UNTESTED, scalar(@{$res->{$KFAIL_UNTESTED}}) if (scalar(@{$res->{$KFAIL_UNTESTED}}));
+ printf " | %-40s | %7d |\n", $KFAIL_DISAPPEARS, scalar(@{$res->{$KFAIL_DISAPPEARS}}) if (scalar(@{$res->{$KFAIL_DISAPPEARS}}));
printf " | %-40s | %7d |\n", $UNSUPPORTED_PASS, scalar(@{$res->{$UNSUPPORTED_PASS}}) if (scalar(@{$res->{$UNSUPPORTED_PASS}}));
printf " | %-40s | %7d |\n", $UNSUPPORTED_XFAIL, scalar(@{$res->{$UNSUPPORTED_XFAIL}}) if (scalar(@{$res->{$UNSUPPORTED_XFAIL}}));
printf " | %-40s | %7d |\n", $UNSUPPORTED_UNTESTED, scalar(@{$res->{$UNSUPPORTED_UNTESTED}}) if (scalar(@{$res->{$UNSUPPORTED_UNTESTED}}));
@@ -899,6 +989,12 @@
print_tclist($res, $XFAIL_UNSUPPORTED, @{$res->{$XFAIL_UNSUPPORTED}});
print_tclist($res, $XFAIL_UNTESTED, @{$res->{$XFAIL_UNTESTED}});
print_tclist($res, $XFAIL_APPEARS, @{$res->{$XFAIL_APPEARS}});
+ print_tclist($res, $KFAIL_PASS, @{$res->{$KFAIL_PASS}});
+ print_tclist($res, $KFAIL_XPASS, @{$res->{$KFAIL_XPASS}});
+ print_tclist($res, $KFAIL_XFAIL, @{$res->{$KFAIL_XFAIL}});
+ print_tclist($res, $KFAIL_UNSUPPORTED, @{$res->{$KFAIL_UNSUPPORTED}});
+ print_tclist($res, $KFAIL_UNTESTED, @{$res->{$KFAIL_UNTESTED}});
+ print_tclist($res, $KFAIL_DISAPPEARS, @{$res->{$KFAIL_DISAPPEARS}});
print_tclist($res, $UNSUPPORTED_PASS, @{$res->{$UNSUPPORTED_PASS}});
print_tclist($res, $UNSUPPORTED_XFAIL, @{$res->{$UNSUPPORTED_XFAIL}});
print_tclist($res, $UNSUPPORTED_UNTESTED, @{$res->{$UNSUPPORTED_UNTESTED}});
diff --git a/testsuite/expected-1-2-long.txt b/testsuite/expected-1-2-long.txt
index e87ff77..e69ddab 100644
--- a/testsuite/expected-1-2-long.txt
+++ b/testsuite/expected-1-2-long.txt
@@ -5,18 +5,19 @@
+---------+---------+
o RUN STATUS: | REF | RES |
+------------------------------------------+---------+---------+
- | Passes [PASS] | 20 | 15 |
- | Unexpected fails [FAIL] | 10 | 11 |
+ | Passes [PASS] | 22 | 16 |
+ | Unexpected fails [FAIL] | 11 | 12 |
| Errors [ERROR] | 1 | 1 |
- | Unexpected passes [XPASS] | 8 | 10 |
- | Expected fails [XFAIL] | 8 | 10 |
- | Unresolved [UNRESOLVED] | 8 | 9 |
- | Unsupported [UNSUPPORTED] | 8 | 9 |
- | Untested [UNTESTED] | 8 | 9 |
+ | Unexpected passes [XPASS] | 9 | 11 |
+ | Expected fails [XFAIL] | 9 | 11 |
+ | Known fails [KFAIL] | 9 | 11 |
+ | Unresolved [UNRESOLVED] | 9 | 10 |
+ | Unsupported [UNSUPPORTED] | 9 | 10 |
+ | Untested [UNTESTED] | 9 | 10 |
+------------------------------------------+---------+---------+
- REF PASS ratio: 0.400000
- RES PASS ratio: 0.342466
+ REF PASS ratio: 0.356322
+ RES PASS ratio: 0.296703
***** ERROR: REF PASS ratio is abnormally low *****
***** ERROR: RES PASS ratio is abnormally low *****
***** ERROR: No REF execution test PASSed. Check execution engine configuration. *****
@@ -26,26 +27,33 @@
+------------------------------------------+---------+
| PASS now XPASS [PASS =>XPASS] | 2 |
| PASS now FAIL [PASS => FAIL] | 2 |
+ | PASS now KFAIL [PASS =>KFAIL] | 2 |
| PASS now UNRESOLVED [PASS =>UNRES] | 2 |
| ERROR appears [ =>ERROR] | 1 |
| XPASS now FAIL [XPASS=> FAIL] | 1 |
+ | XPASS now KFAIL [XPASS=>KFAIL] | 1 |
| XPASS appears [ =>XPASS] | 2 |
| XPASS now UNRESOLVED [XPASS=>UNRES] | 1 |
| FAIL appears [ => FAIL] | 2 |
| FAIL now UNRESOLVED [FAIL =>UNRES] | 1 |
| XFAIL now FAIL [XFAIL=> FAIL] | 1 |
| XFAIL now UNRESOLVED [XFAIL=>UNRES] | 1 |
+ | KFAIL appears [ =>KFAIL] | 2 |
+ | KFAIL now UNRESOLVED [KFAIL=>UNRES] | 1 |
| UNSUPPORTED now FAIL [UNSUP=> FAIL] | 1 |
+ | UNSUPPORTED now KFAIL [UNSUP=>KFAIL] | 1 |
| UNSUPPORTED now XPASS [UNSUP=>XPASS] | 1 |
| UNSUPP now UNRESOLVED [UNSUP=>UNRES] | 1 |
| UNTESTED now FAIL [UNTES=> FAIL] | 1 |
+ | UNTESTED now KFAIL [UNTES=>KFAIL] | 1 |
| UNTESTED now XPASS [UNTES=>XPASS] | 1 |
| UNTESTED now UNRESOLVED [UNTES=>UNRES] | 1 |
| UNRESOLVED now FAIL [UNRES=> FAIL] | 1 |
+ | UNRESOLVED now KFAIL [UNRES=>KFAIL] | 1 |
| UNRESOLVED now XPASS [UNRES=>XPASS] | 1 |
| UNRESOLVED appears [ =>UNRES] | 1 |
+------------------------------------------+---------+
- | TOTAL_REGRESSIONS | 25 |
+ | TOTAL_REGRESSIONS | 34 |
+------------------------------------------+---------+
- PASS now XPASS [PASS =>XPASS]:
@@ -64,6 +72,14 @@
gcc-compare-results:tests-1.exp=test3-pass-fail
+ - PASS now KFAIL [PASS =>KFAIL]:
+
+ Executed from: gcc-compare-results:hw-dep.exp
+ gcc-compare-results:hw-dep.exp=test9-hwdep-pass-kfail
+ Executed from: gcc-compare-results:tests-1.exp
+ gcc-compare-results:tests-1.exp=test9-pass-kfail
+
+
- PASS now UNRESOLVED [PASS =>UNRES]:
Executed from: gcc-compare-results:hw-dep.exp
@@ -84,6 +100,12 @@
gcc-compare-results:tests-2.exp=test3-xpass-fail
+ - XPASS now KFAIL [XPASS=>KFAIL]:
+
+ Executed from: gcc-compare-results:tests-2.exp
+ gcc-compare-results:tests-2.exp=test9-xpass-kfail
+
+
- XPASS appears [ =>XPASS]:
Executed from: gcc-compare-results:tests-appear.exp
@@ -124,12 +146,32 @@
gcc-compare-results:tests-4.exp=test8-xfail-unresolved
+ - KFAIL appears [ =>KFAIL]:
+
+ Executed from: gcc-compare-results:tests-appear.exp
+ gcc-compare-results:tests-appear.exp=test5-kfail-appears
+ Executed from: gcc-compare-results:tests-appear2.exp
+ gcc-compare-results:tests-appear2.exp=test8-kfail-appears
+
+
+ - KFAIL now UNRESOLVED [KFAIL=>UNRES]:
+
+ Executed from: gcc-compare-results:tests-10.exp
+ gcc-compare-results:tests-10.exp=test8-kfail-unresolved
+
+
- UNSUPPORTED now FAIL [UNSUP=> FAIL]:
Executed from: gcc-compare-results:tests-5.exp
gcc-compare-results:tests-5.exp=test3-unsupported-fail
+ - UNSUPPORTED now KFAIL [UNSUP=>KFAIL]:
+
+ Executed from: gcc-compare-results:tests-5.exp
+ gcc-compare-results:tests-5.exp=test9-unsupported-kfail
+
+
- UNSUPPORTED now XPASS [UNSUP=>XPASS]:
Executed from: gcc-compare-results:tests-5.exp
@@ -148,6 +190,12 @@
gcc-compare-results:tests-6.exp=test3-untested-fail
+ - UNTESTED now KFAIL [UNTES=>KFAIL]:
+
+ Executed from: gcc-compare-results:tests-6.exp
+ gcc-compare-results:tests-6.exp=test9-untested-kfail
+
+
- UNTESTED now XPASS [UNTES=>XPASS]:
Executed from: gcc-compare-results:tests-6.exp
@@ -166,6 +214,12 @@
gcc-compare-results:tests-7.exp=test3-unresolved-fail
+ - UNRESOLVED now KFAIL [UNRES=>KFAIL]:
+
+ Executed from: gcc-compare-results:tests-7.exp
+ gcc-compare-results:tests-7.exp=test9-unresolved-kfail
+
+
- UNRESOLVED now XPASS [UNRES=>XPASS]:
Executed from: gcc-compare-results:tests-7.exp
@@ -203,6 +257,12 @@
| XFAIL now UNSUPPORTED [XFAIL=>UNSUP] | 1 |
| XFAIL now UNTESTED [XFAIL=>UNTES] | 1 |
| XFAIL appears [ =>XFAIL] | 2 |
+ | KFAIL now PASS [KFAIL=> PASS] | 1 |
+ | KFAIL now XPASS [KFAIL=>XPASS] | 1 |
+ | KFAIL now XFAIL [KFAIL=>XFAIL] | 1 |
+ | KFAIL now UNSUPPORTED [KFAIL=>UNSUP] | 1 |
+ | KFAIL now UNTESTED [KFAIL=>UNTES] | 1 |
+ | KFAIL disappears [KFAIL=> ] | 1 |
| UNSUPPORTED now PASS [UNSUP=> PASS] | 1 |
| UNSUPPORTED now XFAIL [UNSUP=>XFAIL] | 1 |
| UNSUPP now UNTESTED [UNSUP=>UNTES] | 1 |
@@ -221,7 +281,7 @@
| ERROR disappears [ERROR=> ] | 1 |
| HW dependent cases [HW-DEPENDENT] | 2 |
+------------------------------------------+---------+
- | TOTAL_IMPROVEMENTS_TO_BE_CHECKED | 45 |
+ | TOTAL_IMPROVEMENTS_TO_BE_CHECKED | 51 |
+------------------------------------------+---------+
- PASS disappears [PASS => ]:
@@ -370,6 +430,42 @@
gcc-compare-results:tests-appear2.exp=test4-xfail-appears
+ - KFAIL now PASS [KFAIL=> PASS]:
+
+ Executed from: gcc-compare-results:tests-10.exp
+ gcc-compare-results:tests-10.exp=test1-kfail-pass
+
+
+ - KFAIL now XPASS [KFAIL=>XPASS]:
+
+ Executed from: gcc-compare-results:tests-10.exp
+ gcc-compare-results:tests-10.exp=test2-kfail-xpass
+
+
+ - KFAIL now XFAIL [KFAIL=>XFAIL]:
+
+ Executed from: gcc-compare-results:tests-10.exp
+ gcc-compare-results:tests-10.exp=test4-kfail-xfail
+
+
+ - KFAIL now UNSUPPORTED [KFAIL=>UNSUP]:
+
+ Executed from: gcc-compare-results:tests-10.exp
+ gcc-compare-results:tests-10.exp=test6-kfail-unsupported
+
+
+ - KFAIL now UNTESTED [KFAIL=>UNTES]:
+
+ Executed from: gcc-compare-results:tests-10.exp
+ gcc-compare-results:tests-10.exp=test7-kfail-untested
+
+
+ - KFAIL disappears [KFAIL=> ]:
+
+ Executed from: gcc-compare-results:tests-10.exp
+ gcc-compare-results:tests-10.exp=test5-kfail-disappears
+
+
- UNSUPPORTED now PASS [UNSUP=> PASS]:
Executed from: gcc-compare-results:tests-5.exp
diff --git a/testsuite/expected-1-2.txt b/testsuite/expected-1-2.txt
index 7c84055..8857e88 100644
--- a/testsuite/expected-1-2.txt
+++ b/testsuite/expected-1-2.txt
@@ -5,18 +5,19 @@
+---------+---------+
o RUN STATUS: | REF | RES |
+------------------------------------------+---------+---------+
- | Passes [PASS] | 20 | 15 |
- | Unexpected fails [FAIL] | 10 | 11 |
+ | Passes [PASS] | 22 | 16 |
+ | Unexpected fails [FAIL] | 11 | 12 |
| Errors [ERROR] | 1 | 1 |
- | Unexpected passes [XPASS] | 8 | 10 |
- | Expected fails [XFAIL] | 8 | 10 |
- | Unresolved [UNRESOLVED] | 8 | 9 |
- | Unsupported [UNSUPPORTED] | 8 | 9 |
- | Untested [UNTESTED] | 8 | 9 |
+ | Unexpected passes [XPASS] | 9 | 11 |
+ | Expected fails [XFAIL] | 9 | 11 |
+ | Known fails [KFAIL] | 9 | 11 |
+ | Unresolved [UNRESOLVED] | 9 | 10 |
+ | Unsupported [UNSUPPORTED] | 9 | 10 |
+ | Untested [UNTESTED] | 9 | 10 |
+------------------------------------------+---------+---------+
- REF PASS ratio: 0.400000
- RES PASS ratio: 0.342466
+ REF PASS ratio: 0.356322
+ RES PASS ratio: 0.296703
***** ERROR: REF PASS ratio is abnormally low *****
***** ERROR: RES PASS ratio is abnormally low *****
***** ERROR: No REF execution test PASSed. Check execution engine configuration. *****
@@ -26,26 +27,33 @@
+------------------------------------------+---------+
| PASS now XPASS [PASS =>XPASS] | 2 |
| PASS now FAIL [PASS => FAIL] | 2 |
+ | PASS now KFAIL [PASS =>KFAIL] | 2 |
| PASS now UNRESOLVED [PASS =>UNRES] | 2 |
| ERROR appears [ =>ERROR] | 1 |
| XPASS now FAIL [XPASS=> FAIL] | 1 |
+ | XPASS now KFAIL [XPASS=>KFAIL] | 1 |
| XPASS appears [ =>XPASS] | 2 |
| XPASS now UNRESOLVED [XPASS=>UNRES] | 1 |
| FAIL appears [ => FAIL] | 2 |
| FAIL now UNRESOLVED [FAIL =>UNRES] | 1 |
| XFAIL now FAIL [XFAIL=> FAIL] | 1 |
| XFAIL now UNRESOLVED [XFAIL=>UNRES] | 1 |
+ | KFAIL appears [ =>KFAIL] | 2 |
+ | KFAIL now UNRESOLVED [KFAIL=>UNRES] | 1 |
| UNSUPPORTED now FAIL [UNSUP=> FAIL] | 1 |
+ | UNSUPPORTED now KFAIL [UNSUP=>KFAIL] | 1 |
| UNSUPPORTED now XPASS [UNSUP=>XPASS] | 1 |
| UNSUPP now UNRESOLVED [UNSUP=>UNRES] | 1 |
| UNTESTED now FAIL [UNTES=> FAIL] | 1 |
+ | UNTESTED now KFAIL [UNTES=>KFAIL] | 1 |
| UNTESTED now XPASS [UNTES=>XPASS] | 1 |
| UNTESTED now UNRESOLVED [UNTES=>UNRES] | 1 |
| UNRESOLVED now FAIL [UNRES=> FAIL] | 1 |
+ | UNRESOLVED now KFAIL [UNRES=>KFAIL] | 1 |
| UNRESOLVED now XPASS [UNRES=>XPASS] | 1 |
| UNRESOLVED appears [ =>UNRES] | 1 |
+------------------------------------------+---------+
- | TOTAL_REGRESSIONS | 25 |
+ | TOTAL_REGRESSIONS | 34 |
+------------------------------------------+---------+
@@ -73,6 +81,12 @@
| XFAIL now UNSUPPORTED [XFAIL=>UNSUP] | 1 |
| XFAIL now UNTESTED [XFAIL=>UNTES] | 1 |
| XFAIL appears [ =>XFAIL] | 2 |
+ | KFAIL now PASS [KFAIL=> PASS] | 1 |
+ | KFAIL now XPASS [KFAIL=>XPASS] | 1 |
+ | KFAIL now XFAIL [KFAIL=>XFAIL] | 1 |
+ | KFAIL now UNSUPPORTED [KFAIL=>UNSUP] | 1 |
+ | KFAIL now UNTESTED [KFAIL=>UNTES] | 1 |
+ | KFAIL disappears [KFAIL=> ] | 1 |
| UNSUPPORTED now PASS [UNSUP=> PASS] | 1 |
| UNSUPPORTED now XFAIL [UNSUP=>XFAIL] | 1 |
| UNSUPP now UNTESTED [UNSUP=>UNTES] | 1 |
@@ -91,7 +105,7 @@
| ERROR disappears [ERROR=> ] | 1 |
| HW dependent cases [HW-DEPENDENT] | 2 |
+------------------------------------------+---------+
- | TOTAL_IMPROVEMENTS_TO_BE_CHECKED | 45 |
+ | TOTAL_IMPROVEMENTS_TO_BE_CHECKED | 51 |
+------------------------------------------+---------+
diff --git a/testsuite/sum-1.txt b/testsuite/sum-1.txt
index 6c7620d..99b1dce 100644
--- a/testsuite/sum-1.txt
+++ b/testsuite/sum-1.txt
@@ -10,6 +10,7 @@
PASS: test6-pass-unsupported
PASS: test7-pass-untested
PASS: test8-pass-unresolved
+PASS: test9-pass-kfail
Running tests-2.exp ...
XPASS: test1-xpass-pass
@@ -20,6 +21,7 @@
XPASS: test6-xpass-unsupported
XPASS: test7-xpass-untested
XPASS: test8-xpass-unresolved
+XPASS: test9-xpass-kfail
Running tests-3.exp ...
FAIL: test1-fail-pass
@@ -30,6 +32,7 @@
FAIL: test6-fail-unsupported
FAIL: test7-fail-untested
FAIL: test8-fail-unresolved
+FAIL: test9-fail-kfail
Running tests-4.exp ...
XFAIL: test1-xfail-pass
@@ -40,6 +43,7 @@
XFAIL: test6-xfail-unsupported
XFAIL: test7-xfail-untested
XFAIL: test8-xfail-unresolved
+XFAIL: test9-xfail-kfail
Running tests-5.exp ...
UNSUPPORTED: test1-unsupported-pass
@@ -50,6 +54,7 @@
UNSUPPORTED: test6-unsupported-untested
UNSUPPORTED: test7-unsupported-unresolved
UNSUPPORTED: test8-unsupported-unsupported
+UNSUPPORTED: test9-unsupported-kfail
Running tests-6.exp ...
UNTESTED: test1-untested-pass
@@ -60,6 +65,7 @@
UNTESTED: test6-untested-unsupported
UNTESTED: test7-untested-unresolved
UNTESTED: test8-untested-untested
+UNTESTED: test9-untested-kfail
Running tests-7.exp ...
UNRESOLVED: test1-unresolved-pass
@@ -70,6 +76,7 @@
UNRESOLVED: test6-unresolved-unsupported
UNRESOLVED: test7-unresolved-untested
UNRESOLVED: test8-unresolved-unresolved
+UNRESOLVED: test9-unresolved-kfail
Running tests-8.exp ...
PASS: test1-pass-and-fails
@@ -80,6 +87,17 @@
ERROR: (DejaGnu) proc "scan-dump-tree-not fail_test optimized 1" does not exist.
+Running tests-10.exp ...
+KFAIL: test1-kfail-pass
+KFAIL: test2-kfail-xpass
+KFAIL: test3-kfail-fail
+KFAIL: test4-kfail-xfail
+KFAIL: test5-kfail-disappears
+KFAIL: test6-kfail-unsupported
+KFAIL: test7-kfail-untested
+KFAIL: test8-kfail-unresolved
+KFAIL: test9-kfail-kfail
+
Running hw-dep.exp ...
PASS: test1-hwdep-pass-pass
PASS: test2-hwdep-pass-xpass
@@ -89,6 +107,7 @@
PASS: test6-hwdep-pass-unsupported
PASS: test7-hwdep-pass-untested
PASS: test8-hwdep-pass-unresolved
+PASS: test9-hwdep-pass-kfail
Running duplicate.exp ...
PASS: test1-dup-pass-disappears
diff --git a/testsuite/sum-2.txt b/testsuite/sum-2.txt
index 68cee55..9f251ca 100644
--- a/testsuite/sum-2.txt
+++ b/testsuite/sum-2.txt
@@ -9,6 +9,7 @@
UNSUPPORTED: test6-pass-unsupported
UNTESTED: test7-pass-untested
UNRESOLVED: test8-pass-unresolved
+KFAIL: test9-pass-kfail
Running tests-2.exp ...
PASS: test1-xpass-pass
@@ -18,6 +19,7 @@
UNSUPPORTED: test6-xpass-unsupported
UNTESTED: test7-xpass-untested
UNRESOLVED: test8-xpass-unresolved
+KFAIL: test9-xpass-kfail
Running tests-3.exp ...
PASS: test1-fail-pass
@@ -27,6 +29,7 @@
UNSUPPORTED: test6-fail-unsupported
UNTESTED: test7-fail-untested
UNRESOLVED: test8-fail-unresolved
+KFAIL: test9-fail-kfail
Running tests-4.exp ...
PASS: test1-xfail-pass
@@ -36,12 +39,14 @@
UNSUPPORTED: test6-xfail-unsupported
UNTESTED: test7-xfail-untested
UNRESOLVED: test8-xfail-unresolved
+KFAIL: test9-xfail-kfail
Running tests-appear.exp ...
PASS: test1-pass-appears
XPASS: test2-xpass-appears
FAIL: test3-fail-appears
XFAIL: test4-xfail-appears
+KFAIL: test5-kfail-appears
Running tests-5.exp ...
PASS: test1-unsupported-pass
@@ -51,6 +56,7 @@
UNTESTED: test6-unsupported-untested
UNRESOLVED: test7-unsupported-unresolved
UNSUPPORTED: test8-unsupported-unsupported
+KFAIL: test9-unsupported-kfail
Running tests-6.exp ...
PASS: test1-untested-pass
@@ -60,6 +66,7 @@
UNSUPPORTED: test6-untested-unsupported
UNRESOLVED: test7-untested-unresolved
UNTESTED: test8-untested-untested
+KFAIL: test9-untested-kfail
Running tests-7.exp ...
PASS: test1-unresolved-pass
@@ -69,6 +76,7 @@
UNSUPPORTED: test6-unresolved-unsupported
UNTESTED: test7-unresolved-untested
UNRESOLVED: test8-unresolved-unresolved
+KFAIL: test9-unresolved-kfail
Running tests-appear2.exp ...
PASS: test1-pass-appears
@@ -78,6 +86,7 @@
UNSUPPORTED: test5-unsupported-appears
UNTESTED: test6-untested-appears
UNRESOLVED: test7-unresolved-appears
+KFAIL: test8-kfail-appears
Running tests-8.exp ...
PASS: test1-pass-and-fails
@@ -88,6 +97,16 @@
ERROR: (DejaGnu) proc "scan-dump-tree-not fail_test optimized 3" does not exist.
+Running tests-10.exp ...
+PASS: test1-kfail-pass
+XPASS: test2-kfail-xpass
+FAIL: test3-kfail-fail
+XFAIL: test4-kfail-xfail
+UNSUPPORTED: test6-kfail-unsupported
+UNTESTED: test7-kfail-untested
+UNRESOLVED: test8-kfail-unresolved
+KFAIL: test9-kfail-kfail
+
Running hw-dep.exp ...
PASS: test1-hwdep-pass-pass
XPASS: test2-hwdep-pass-xpass
@@ -96,6 +115,7 @@
UNSUPPORTED: test6-hwdep-pass-unsupported
UNTESTED: test7-hwdep-pass-untested
UNRESOLVED: test8-hwdep-pass-unresolved
+KFAIL: test9-hwdep-pass-kfail
Running duplicate.exp ...
PASS: test1-dup-pass-disappears