Handle all combinations of status changes.
Change-Id: I8ad095a0a8544ac33a344fe1aa3750653d57b5e7
diff --git a/compare_dg_tests.pl b/compare_dg_tests.pl
index 1b19687..37ee377 100755
--- a/compare_dg_tests.pl
+++ b/compare_dg_tests.pl
@@ -22,6 +22,9 @@
my $PASS_XPASS = "PASS now XPASS [PASS =>XPASS]";
my $PASS_FAIL = "PASS now FAIL [PASS => FAIL]";
my $PASS_XFAIL = "PASS now XFAIL [PASS =>XFAIL]";
+my $PASS_UNSUPPORTED = "PASS now UNSUPPORTED [PASS =>UNSUP]";
+my $PASS_UNRESOLVED = "PASS now UNRESOLVED [PASS =>UNRES]";
+my $PASS_UNTESTED = "PASS now UNTESTED [PASS =>UNTES]";
my $PASS_DISAPPEARS = "PASS disappears [PASS => ]";
my $PASS_APPEARS = "New PASS [ => PASS]";
my $PASSED_NOW_TIMEOUTS = "Timeout [PASS =>T.OUT]";
@@ -30,6 +33,9 @@
my $XPASS_XPASS = "Still xpass [XPASS=>XPASS]";
my $XPASS_FAIL = "XPASS now FAIL [XPASS=> FAIL]";
my $XPASS_XFAIL = "XPASS now XFAIL [XPASS=>XFAIL]";
+my $XPASS_UNSUPPORTED = "XPASS now UNSUPPORTED [XPASS=>UNSUP]";
+my $XPASS_UNRESOLVED = "XPASS now UNRESOLVED [XPASS=>UNRES]";
+my $XPASS_UNTESTED = "XPASS now UNTESTED [XPASS=>UNTES]";
my $XPASS_DISAPPEARS = "XPASS disappears [XPASS=> ]";
my $XPASS_APPEARS = "XPASS appears [ =>XPASS]";
@@ -37,6 +43,9 @@
my $FAIL_XPASS = "FAIL now XPASS [FAIL =>XPASS]";
my $FAIL_FAIL = "Still fails [FAIL => FAIL]";
my $FAIL_XFAIL = "FAIL now XFAIL [FAIL =>XFAIL]";
+my $FAIL_UNSUPPORTED = "FAIL now UNSUPPORTED [FAIL =>UNSUP]";
+my $FAIL_UNRESOLVED = "FAIL now UNRESOLVED [FAIL =>UNRES]";
+my $FAIL_UNTESTED = "FAIL now UNTESTED [FAIL =>UNTES]";
my $FAIL_DISAPPEARS = "FAIL disappears [FAIL => ]";
my $FAIL_APPEARS = "FAIL appears [ => FAIL]";
@@ -44,6 +53,9 @@
my $XFAIL_XPASS = "XFAIL now XPASS [XFAIL=>XPASS]";
my $XFAIL_FAIL = "XFAIL now FAIL [XFAIL=> FAIL]";
my $XFAIL_XFAIL = "Still xfail [XFAIL=>XFAIL]";
+my $XFAIL_UNSUPPORTED = "XFAIL now UNSUPPORTED [XFAIL=>UNSUP]";
+my $XFAIL_UNRESOLVED = "XFAIL now UNRESOLVED [XFAIL=>UNRES]";
+my $XFAIL_UNTESTED = "XFAIL now UNTESTED [XFAIL=>UNTES]";
my $XFAIL_DISAPPEARS = "XFAIL disappears [XFAIL=> ]";
my $XFAIL_APPEARS = "XFAIL appears [ =>XFAIL]";
@@ -51,6 +63,9 @@
my $UNSUPPORTED_XPASS = "UNSUPPORTED now XPASS [UNSUP=>XPASS]";
my $UNSUPPORTED_FAIL = "UNSUPPORTED now FAIL [UNSUP=> FAIL]";
my $UNSUPPORTED_XFAIL = "UNSUPPORTED now XFAIL [UNSUP=>XFAIL]";
+my $UNSUPPORTED_UNSUPPORTED= "UNSUPP now UNSUPP [UNSUP=>UNSUP]";
+my $UNSUPPORTED_UNRESOLVED= "UNSUPP now UNRESOLVED [UNSUP=>UNRES]";
+my $UNSUPPORTED_UNTESTED = "UNSUPP now UNTESTED [UNSUP=>UNTES]";
my $UNSUPPORTED_DISAPPEARS= "UNSUPPORTED disappears [UNSUP=> ]";
my $UNSUPPORTED_APPEARS = "UNSUPPORTED appears [ =>UNSUP]";
@@ -58,6 +73,9 @@
my $UNTESTED_XPASS = "UNTESTED now XPASS [UNTES=>XPASS]";
my $UNTESTED_FAIL = "UNTESTED now FAIL [UNTES=> FAIL]";
my $UNTESTED_XFAIL = "UNTESTED now XFAIL [UNTES=>XFAIL]";
+my $UNTESTED_UNSUPPORTED = "UNTESTED now UNSUPP [UNTES=>UNSUP]";
+my $UNTESTED_UNRESOLVED = "UNTESTED now UNRESOLVED [UNTES=>UNRES]";
+my $UNTESTED_UNTESTED = "UNTESTED now UNTESTED [UNTES=>UNTES]";
my $UNTESTED_DISAPPEARS = "UNTESTED disappears [UNTES=> ]";
my $UNTESTED_APPEARS = "UNTESTED appears [ =>UNTES]";
@@ -65,6 +83,9 @@
my $UNRESOLVED_XPASS = "UNRESOLVED now XPASS [UNRES=>XPASS]";
my $UNRESOLVED_FAIL = "UNRESOLVED now FAIL [UNRES=> FAIL]";
my $UNRESOLVED_XFAIL = "UNRESOLVED now XFAIL [UNRES=>XFAIL]";
+my $UNRESOLVED_UNSUPPORTED= "UNRESOLVED now UNSUPP [UNRES=>UNSUP]";
+my $UNRESOLVED_UNRESOLVED = "UNRESOLVED now UNRESOLVED [UNRES=>UNRES]";
+my $UNRESOLVED_UNTESTED = "UNRESOLVED now UNTESTED [UNRES=>UNTES]";
my $UNRESOLVED_DISAPPEARS = "UNRESOLVED disappears [UNRES=> ]";
my $UNRESOLVED_APPEARS = "UNRESOLVED appears [ =>UNRES]";
@@ -76,6 +97,9 @@
{was=>"PASS", is=>"XPASS", cat=>$PASS_XPASS},
{was=>"PASS", is=>"FAIL", handler=>\&handle_pass_fail},
{was=>"PASS", is=>"XFAIL", cat=>$PASS_XFAIL},
+ {was=>"PASS", is=>"UNSUPPORTED",cat=>$PASS_UNSUPPORTED},
+ {was=>"PASS", is=>"UNTESTED", cat=>$PASS_UNTESTED},
+ {was=>"PASS", is=>"UNRESOLVED",cat=>$PASS_UNRESOLVED},
{was=>"PASS", is=>"NO_EXIST", cat=>$PASS_DISAPPEARS},
{was=>"NO_EXIST", is=>"PASS", cat=>$PASS_APPEARS},
@@ -83,6 +107,9 @@
{was=>"XPASS", is=>"XPASS", cat=>$XPASS_XPASS},
{was=>"XPASS", is=>"FAIL", cat=>$XPASS_FAIL},
{was=>"XPASS", is=>"XFAIL", cat=>$XPASS_XFAIL},
+ {was=>"XPASS", is=>"UNSUPPORTED",cat=>$XPASS_UNSUPPORTED},
+ {was=>"XPASS", is=>"UNTESTED", cat=>$XPASS_UNTESTED},
+ {was=>"XPASS", is=>"UNRESOLVED",cat=>$XPASS_UNRESOLVED},
{was=>"XPASS", is=>"NO_EXIST", cat=>$XPASS_DISAPPEARS},
{was=>"NO_EXIST", is=>"XPASS", cat=>$XPASS_APPEARS},
@@ -90,6 +117,9 @@
{was=>"FAIL", is=>"XPASS", cat=>$FAIL_XPASS},
{was=>"FAIL", is=>"FAIL", cat=>$FAIL_FAIL},
{was=>"FAIL", is=>"XFAIL", cat=>$FAIL_XFAIL},
+ {was=>"FAIL", is=>"UNSUPPORTED",cat=>$FAIL_UNSUPPORTED},
+ {was=>"FAIL", is=>"UNTESTED", cat=>$FAIL_UNTESTED},
+ {was=>"FAIL", is=>"UNRESOLVED",cat=>$FAIL_UNRESOLVED},
{was=>"FAIL", is=>"NO_EXIST", cat=>$FAIL_DISAPPEARS},
{was=>"NO_EXIST", is=>"FAIL", cat=>$FAIL_APPEARS},
@@ -97,13 +127,19 @@
{was=>"XFAIL", is=>"XPASS", cat=>$XFAIL_XPASS},
{was=>"XFAIL", is=>"FAIL", cat=>$XFAIL_FAIL},
{was=>"XFAIL", is=>"XFAIL", cat=>$XFAIL_XFAIL},
+ {was=>"XFAIL", is=>"UNSUPPORTED",cat=>$XFAIL_UNSUPPORTED},
+ {was=>"XFAIL", is=>"UNTESTED", cat=>$XFAIL_UNTESTED},
+ {was=>"XFAIL", is=>"UNRESOLVED",cat=>$XFAIL_UNRESOLVED},
{was=>"XFAIL", is=>"NO_EXIST", cat=>$XFAIL_DISAPPEARS},
{was=>"NO_EXIST", is=>"XFAIL", cat=>$XFAIL_APPEARS},
- {was=>"UNSUPPORTED", is=>"PASS", cat=>$UNSUPPORTED_PASS},
+ {was=>"UNSUPPORTED", is=>"PASS", cat=>$UNSUPPORTED_PASS},
{was=>"UNSUPPORTED", is=>"XPASS", cat=>$UNSUPPORTED_XPASS},
- {was=>"UNSUPPORTED", is=>"FAIL", cat=>$UNSUPPORTED_FAIL},
+ {was=>"UNSUPPORTED", is=>"FAIL", cat=>$UNSUPPORTED_FAIL},
{was=>"UNSUPPORTED", is=>"XFAIL", cat=>$UNSUPPORTED_XFAIL},
+ {was=>"UNSUPPORTED", is=>"UNSUPPORTED",cat=>$UNSUPPORTED_UNSUPPORTED},
+ {was=>"UNSUPPORTED", is=>"UNTESTED", cat=>$UNSUPPORTED_UNTESTED},
+ {was=>"UNSUPPORTED", is=>"UNRESOLVED",cat=>$UNSUPPORTED_UNRESOLVED},
{was=>"UNSUPPORTED", is=>"NO_EXIST", cat=>$UNSUPPORTED_DISAPPEARS},
{was=>"NO_EXIST", is=>"UNSUPPORTED", cat=>$UNSUPPORTED_APPEARS},
@@ -111,6 +147,9 @@
{was=>"UNTESTED", is=>"XPASS", cat=>$UNTESTED_XPASS},
{was=>"UNTESTED", is=>"FAIL", cat=>$UNTESTED_FAIL},
{was=>"UNTESTED", is=>"XFAIL", cat=>$UNTESTED_XFAIL},
+ {was=>"UNTESTED", is=>"UNSUPPORTED",cat=>$UNTESTED_UNSUPPORTED},
+ {was=>"UNTESTED", is=>"UNTESTED", cat=>$UNTESTED_UNTESTED},
+ {was=>"UNTESTED", is=>"UNRESOLVED",cat=>$UNTESTED_UNRESOLVED},
{was=>"UNTESTED", is=>"NO_EXIST", cat=>$UNTESTED_DISAPPEARS},
{was=>"NO_EXIST", is=>"UNTESTED", cat=>$UNTESTED_APPEARS},
@@ -118,6 +157,9 @@
{was=>"UNRESOLVED", is=>"XPASS", cat=>$UNRESOLVED_XPASS},
{was=>"UNRESOLVED", is=>"FAIL", cat=>$UNRESOLVED_FAIL},
{was=>"UNRESOLVED", is=>"XFAIL", cat=>$UNRESOLVED_XFAIL},
+ {was=>"UNRESOLVED", is=>"UNSUPPORTED",cat=>$UNRESOLVED_UNSUPPORTED},
+ {was=>"UNRESOLVED", is=>"UNTESTED", cat=>$UNRESOLVED_UNTESTED},
+ {was=>"UNRESOLVED", is=>"UNRESOLVED",cat=>$UNRESOLVED_UNRESOLVED},
{was=>"UNRESOLVED", is=>"NO_EXIST", cat=>$UNRESOLVED_DISAPPEARS},
{was=>"NO_EXIST", is=>"UNRESOLVED", cat=>$UNRESOLVED_APPEARS},
@@ -315,6 +357,9 @@
@{$res->{$PASS_XPASS}} = ();
@{$res->{$PASS_FAIL}} = ();
@{$res->{$PASS_XFAIL}} = ();
+ @{$res->{$PASS_UNSUPPORTED}} = ();
+ @{$res->{$PASS_UNTESTED}} = ();
+ @{$res->{$PASS_UNRESOLVED}} = ();
@{$res->{$PASS_DISAPPEARS}} = ();
@{$res->{$PASS_APPEARS}} = ();
@{$res->{$PASSED_NOW_TIMEOUTS}} = ();
@@ -323,6 +368,9 @@
@{$res->{$XPASS_XPASS}} = ();
@{$res->{$XPASS_FAIL}} = ();
@{$res->{$XPASS_XFAIL}} = ();
+ @{$res->{$XPASS_UNSUPPORTED}} = ();
+ @{$res->{$XPASS_UNTESTED}} = ();
+ @{$res->{$XPASS_UNRESOLVED}} = ();
@{$res->{$XPASS_DISAPPEARS}} = ();
@{$res->{$XPASS_APPEARS}} = ();
@@ -330,6 +378,9 @@
@{$res->{$FAIL_XPASS}} = ();
@{$res->{$FAIL_FAIL}} = ();
@{$res->{$FAIL_XFAIL}} = ();
+ @{$res->{$FAIL_UNSUPPORTED}} = ();
+ @{$res->{$FAIL_UNTESTED}} = ();
+ @{$res->{$FAIL_UNRESOLVED}} = ();
@{$res->{$FAIL_DISAPPEARS}} = ();
@{$res->{$FAIL_APPEARS}} = ();
@@ -337,6 +388,9 @@
@{$res->{$XFAIL_XPASS}} = ();
@{$res->{$XFAIL_FAIL}} = ();
@{$res->{$XFAIL_XFAIL}} = ();
+ @{$res->{$XFAIL_UNSUPPORTED}} = ();
+ @{$res->{$XFAIL_UNTESTED}} = ();
+ @{$res->{$XFAIL_UNRESOLVED}} = ();
@{$res->{$XFAIL_DISAPPEARS}} = ();
@{$res->{$XFAIL_APPEARS}} = ();
@@ -344,6 +398,9 @@
@{$res->{$UNSUPPORTED_XPASS}} = ();
@{$res->{$UNSUPPORTED_FAIL}} = ();
@{$res->{$UNSUPPORTED_XFAIL}} = ();
+ @{$res->{$UNSUPPORTED_UNSUPPORTED}} = ();
+ @{$res->{$UNSUPPORTED_UNTESTED}} = ();
+ @{$res->{$UNSUPPORTED_UNRESOLVED}} = ();
@{$res->{$UNSUPPORTED_DISAPPEARS}} = ();
@{$res->{$UNSUPPORTED_APPEARS}} = ();
@@ -351,6 +408,9 @@
@{$res->{$UNTESTED_XPASS}} = ();
@{$res->{$UNTESTED_FAIL}} = ();
@{$res->{$UNTESTED_XFAIL}} = ();
+ @{$res->{$UNTESTED_UNSUPPORTED}} = ();
+ @{$res->{$UNTESTED_UNTESTED}} = ();
+ @{$res->{$UNTESTED_UNRESOLVED}} = ();
@{$res->{$UNTESTED_DISAPPEARS}} = ();
@{$res->{$UNTESTED_APPEARS}} = ();
@@ -358,6 +418,9 @@
@{$res->{$UNRESOLVED_XPASS}} = ();
@{$res->{$UNRESOLVED_FAIL}} = ();
@{$res->{$UNRESOLVED_XFAIL}} = ();
+ @{$res->{$UNRESOLVED_UNSUPPORTED}} = ();
+ @{$res->{$UNRESOLVED_UNTESTED}} = ();
+ @{$res->{$UNRESOLVED_UNRESOLVED}} = ();
@{$res->{$UNRESOLVED_DISAPPEARS}} = ();
@{$res->{$UNRESOLVED_APPEARS}} = ();
@@ -528,23 +591,29 @@
}
#### REGRESSIONS ?
- $rtotal = scalar(@{$res->{$PASS_FAIL}})
- +scalar(@{$res->{$XPASS_FAIL}})
+ $rtotal = scalar(@{$res->{$PASS_XPASS}})
+ +scalar(@{$res->{$PASS_FAIL}})
+scalar(@{$res->{$PASS_DISAPPEARS}})
- +scalar(@{$res->{$XPASS_DISAPPEARS}})
- +scalar(@{$res->{$FAIL_APPEARS}})
+ +scalar(@{$res->{$PASS_UNRESOLVED}})
+ +scalar(@{$res->{$PASSED_NOW_TIMEOUTS}})
+ +scalar(@{$res->{$XPASS_FAIL}})
+scalar(@{$res->{$XPASS_APPEARS}})
- +scalar(@{$res->{$PASS_XPASS}})
+ +scalar(@{$res->{$XPASS_UNRESOLVED}})
+ +scalar(@{$res->{$FAIL_APPEARS}})
+ +scalar(@{$res->{$FAIL_UNRESOLVED}})
+scalar(@{$res->{$XFAIL_FAIL}})
+scalar(@{$res->{$XFAIL_DISAPPEARS}})
+ +scalar(@{$res->{$XFAIL_UNRESOLVED}})
+scalar(@{$res->{$UNSUPPORTED_FAIL}})
+scalar(@{$res->{$UNSUPPORTED_XPASS}})
+ +scalar(@{$res->{$UNSUPPORTED_UNRESOLVED}})
+scalar(@{$res->{$UNTESTED_FAIL}})
+scalar(@{$res->{$UNTESTED_XPASS}})
+ +scalar(@{$res->{$UNTESTED_UNRESOLVED}})
+scalar(@{$res->{$UNRESOLVED_FAIL}})
+scalar(@{$res->{$UNRESOLVED_XPASS}})
+scalar(@{$res->{$UNRESOLVED_APPEARS}})
- +scalar(@{$res->{$PASSED_NOW_TIMEOUTS}});
+ ;
$quiet_reg=1 if ($short and not $rtotal);
@@ -552,23 +621,28 @@
{
printf "\n$col_red"."o REGRESSIONS:\n";
printf " +------------------------------------------+---------+\n";
- printf " | %-40s | %7d |\n", $PASS_FAIL, scalar(@{$res->{$PASS_FAIL}}) if (scalar(@{$res->{$PASS_FAIL}}));
- printf " | %-40s | %7d |\n", $XPASS_FAIL, scalar(@{$res->{$XPASS_FAIL}}) if (scalar(@{$res->{$XPASS_FAIL}}));
- printf " | %-40s | %7d |\n", $PASS_DISAPPEARS, scalar(@{$res->{$PASS_DISAPPEARS}}) if (scalar(@{$res->{$PASS_DISAPPEARS}}));
- printf " | %-40s | %7d |\n", $XPASS_DISAPPEARS, scalar(@{$res->{$XPASS_DISAPPEARS}}) if (scalar(@{$res->{$XPASS_DISAPPEARS}}));
- printf " | %-40s | %7d |\n", $FAIL_APPEARS, scalar(@{$res->{$FAIL_APPEARS}}) if (scalar(@{$res->{$FAIL_APPEARS}}));
- printf " | %-40s | %7d |\n", $XPASS_APPEARS, scalar(@{$res->{$XPASS_APPEARS}}) if (scalar(@{$res->{$XPASS_APPEARS}}));
printf " | %-40s | %7d |\n", $PASS_XPASS, scalar(@{$res->{$PASS_XPASS}}) if (scalar(@{$res->{$PASS_XPASS}}));
+ printf " | %-40s | %7d |\n", $PASS_FAIL, scalar(@{$res->{$PASS_FAIL}}) if (scalar(@{$res->{$PASS_FAIL}}));
+ printf " | %-40s | %7d |\n", $PASS_DISAPPEARS, scalar(@{$res->{$PASS_DISAPPEARS}}) if (scalar(@{$res->{$PASS_DISAPPEARS}}));
+ printf " | %-40s | %7d |\n", $PASS_UNRESOLVED, scalar(@{$res->{$PASS_UNRESOLVED}}) if (scalar(@{$res->{$PASS_UNRESOLVED}}));
+ printf " | %-40s | %7d |\n", $PASSED_NOW_TIMEOUTS, scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}) if (scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}));
+ printf " | %-40s | %7d |\n", $XPASS_FAIL, scalar(@{$res->{$XPASS_FAIL}}) if (scalar(@{$res->{$XPASS_FAIL}}));
+ printf " | %-40s | %7d |\n", $XPASS_APPEARS, scalar(@{$res->{$XPASS_APPEARS}}) if (scalar(@{$res->{$XPASS_APPEARS}}));
+ printf " | %-40s | %7d |\n", $XPASS_UNRESOLVED, scalar(@{$res->{$XPASS_UNRESOLVED}}) if (scalar(@{$res->{$XPASS_UNRESOLVED}}));
+ printf " | %-40s | %7d |\n", $FAIL_APPEARS, scalar(@{$res->{$FAIL_APPEARS}}) if (scalar(@{$res->{$FAIL_APPEARS}}));
+ printf " | %-40s | %7d |\n", $FAIL_UNRESOLVED, scalar(@{$res->{$FAIL_UNRESOLVED}}) if (scalar(@{$res->{$FAIL_UNRESOLVED}}));
printf " | %-40s | %7d |\n", $XFAIL_FAIL, scalar(@{$res->{$XFAIL_FAIL}}) if (scalar(@{$res->{$XFAIL_FAIL}}));
printf " | %-40s | %7d |\n", $XFAIL_DISAPPEARS, scalar(@{$res->{$XFAIL_DISAPPEARS}}) if (scalar(@{$res->{$XFAIL_DISAPPEARS}}));
+ printf " | %-40s | %7d |\n", $XFAIL_UNRESOLVED, scalar(@{$res->{$XFAIL_UNRESOLVED}}) if (scalar(@{$res->{$XFAIL_UNRESOLVED}}));
printf " | %-40s | %7d |\n", $UNSUPPORTED_FAIL, scalar(@{$res->{$UNSUPPORTED_FAIL}}) if (scalar(@{$res->{$UNSUPPORTED_FAIL}}));
printf " | %-40s | %7d |\n", $UNSUPPORTED_XPASS, scalar(@{$res->{$UNSUPPORTED_XPASS}}) if (scalar(@{$res->{$UNSUPPORTED_XPASS}}));
+ printf " | %-40s | %7d |\n", $UNSUPPORTED_UNRESOLVED, scalar(@{$res->{$UNSUPPORTED_UNRESOLVED}}) if (scalar(@{$res->{$UNSUPPORTED_UNRESOLVED}}));
printf " | %-40s | %7d |\n", $UNTESTED_FAIL, scalar(@{$res->{$UNTESTED_FAIL}}) if (scalar(@{$res->{$UNTESTED_FAIL}}));
printf " | %-40s | %7d |\n", $UNTESTED_XPASS, scalar(@{$res->{$UNTESTED_XPASS}}) if (scalar(@{$res->{$UNTESTED_XPASS}}));
+ printf " | %-40s | %7d |\n", $UNTESTED_UNRESOLVED, scalar(@{$res->{$UNTESTED_UNRESOLVED}}) if (scalar(@{$res->{$UNTESTED_UNRESOLVED}}));
printf " | %-40s | %7d |\n", $UNRESOLVED_FAIL, scalar(@{$res->{$UNRESOLVED_FAIL}}) if (scalar(@{$res->{$UNRESOLVED_FAIL}}));
printf " | %-40s | %7d |\n", $UNRESOLVED_XPASS, scalar(@{$res->{$UNRESOLVED_XPASS}}) if (scalar(@{$res->{$UNRESOLVED_XPASS}}));
printf " | %-40s | %7d |\n", $UNRESOLVED_APPEARS, scalar(@{$res->{$UNRESOLVED_APPEARS}}) if (scalar(@{$res->{$UNRESOLVED_APPEARS}}));
- printf " | %-40s | %7d |\n", $PASSED_NOW_TIMEOUTS, scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}) if (scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}));
printf " +------------------------------------------+---------+\n";
printf " | %-40s | %7d |\n", "TOTAL_REGRESSIONS", $rtotal;
printf " +------------------------------------------+---------+\n";
@@ -576,23 +650,28 @@
if ($long)
{
- print_tclist($PASS_FAIL, @{$res->{$PASS_FAIL}});
- print_tclist($XPASS_FAIL, @{$res->{$XPASS_FAIL}});
- print_tclist($PASS_DISAPPEARS, @{$res->{$PASS_DISAPPEARS}});
- print_tclist($XPASS_DISAPPEARS, @{$res->{$XPASS_DISAPPEARS}});
- print_tclist($FAIL_APPEARS, @{$res->{$FAIL_APPEARS}});
- print_tclist($XPASS_APPEARS, @{$res->{$XPASS_APPEARS}});
print_tclist($PASS_XPASS, @{$res->{$PASS_XPASS}});
+ print_tclist($PASS_FAIL, @{$res->{$PASS_FAIL}});
+ print_tclist($PASS_DISAPPEARS, @{$res->{$PASS_DISAPPEARS}});
+ print_tclist($PASS_UNRESOLVED, @{$res->{$PASS_UNRESOLVED}});
+ print_tclist($PASSED_NOW_TIMEOUTS, @{$res->{$PASSED_NOW_TIMEOUTS}});
+ print_tclist($XPASS_FAIL, @{$res->{$XPASS_FAIL}});
+ print_tclist($XPASS_APPEARS, @{$res->{$XPASS_APPEARS}});
+ print_tclist($XPASS_UNRESOLVED, @{$res->{$XPASS_UNRESOLVED}});
+ print_tclist($FAIL_APPEARS, @{$res->{$FAIL_APPEARS}});
+ print_tclist($FAIL_UNRESOLVED, @{$res->{$FAIL_UNRESOLVED}});
print_tclist($XFAIL_FAIL, @{$res->{$XFAIL_FAIL}});
print_tclist($XFAIL_DISAPPEARS, @{$res->{$XFAIL_DISAPPEARS}});
+ print_tclist($XFAIL_UNRESOLVED, @{$res->{$XFAIL_UNRESOLVED}});
print_tclist($UNSUPPORTED_FAIL, @{$res->{$UNSUPPORTED_FAIL}});
print_tclist($UNSUPPORTED_XPASS, @{$res->{$UNSUPPORTED_XPASS}});
+ print_tclist($UNSUPPORTED_UNRESOLVED, @{$res->{$UNSUPPORTED_UNRESOLVED}});
print_tclist($UNTESTED_FAIL, @{$res->{$UNTESTED_FAIL}});
print_tclist($UNTESTED_XPASS, @{$res->{$UNTESTED_XPASS}});
+ print_tclist($UNTESTED_UNRESOLVED, @{$res->{$UNTESTED_UNRESOLVED}});
print_tclist($UNRESOLVED_FAIL, @{$res->{$UNRESOLVED_FAIL}});
print_tclist($UNRESOLVED_XPASS, @{$res->{$UNRESOLVED_XPASS}});
print_tclist($UNRESOLVED_APPEARS, @{$res->{$UNRESOLVED_APPEARS}});
- print_tclist($PASSED_NOW_TIMEOUTS, @{$res->{$PASSED_NOW_TIMEOUTS}});
}
printf "$col_reset\n";
}
@@ -600,53 +679,80 @@
#### MINOR TO BE CHECKED ?
if (not $quiet and not $short)
{
- $total_better = scalar(@{$res->{$XPASS_PASS}})+
- scalar(@{$res->{$XFAIL_PASS}})+
- scalar(@{$res->{$XFAIL_XPASS}})+
+ $total_better =
+ scalar(@{$res->{$PASS_XFAIL}})+
+ scalar(@{$res->{$PASS_APPEARS}})+
+ scalar(@{$res->{$PASS_UNSUPPORTED}})+
+ scalar(@{$res->{$PASS_UNTESTED}})+
+ scalar(@{$res->{$XPASS_PASS}})+
+ scalar(@{$res->{$XPASS_XFAIL}})+
+ scalar(@{$res->{$XPASS_UNSUPPORTED}})+
+ scalar(@{$res->{$XPASS_UNTESTED}})+
+ scalar(@{$res->{$XPASS_DISAPPEARS}})+
scalar(@{$res->{$FAIL_PASS}})+
scalar(@{$res->{$FAIL_XPASS}})+
- scalar(@{$res->{$PASS_APPEARS}})+
- scalar(@{$res->{$FAIL_DISAPPEARS}})+
- scalar(@{$res->{$XFAIL_APPEARS}})+
scalar(@{$res->{$FAIL_XFAIL}})+
- scalar(@{$res->{$PASS_XFAIL}})+
- scalar(@{$res->{$XPASS_XFAIL}})+
+ scalar(@{$res->{$FAIL_UNSUPPORTED}})+
+ scalar(@{$res->{$FAIL_UNTESTED}})+
+ scalar(@{$res->{$FAIL_DISAPPEARS}})+
+ scalar(@{$res->{$XFAIL_PASS}})+
+ scalar(@{$res->{$XFAIL_XPASS}})+
+ scalar(@{$res->{$XFAIL_UNSUPPORTED}})+
+ scalar(@{$res->{$XFAIL_UNTESTED}})+
+ scalar(@{$res->{$XFAIL_APPEARS}})+
scalar(@{$res->{$UNSUPPORTED_PASS}})+
scalar(@{$res->{$UNSUPPORTED_XFAIL}})+
+ scalar(@{$res->{$UNSUPPORTED_UNTESTED}})+
scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}})+
scalar(@{$res->{$UNSUPPORTED_APPEARS}})+
scalar(@{$res->{$UNTESTED_PASS}})+
scalar(@{$res->{$UNTESTED_XFAIL}})+
+ scalar(@{$res->{$UNTESTED_UNSUPPORTED}})+
scalar(@{$res->{$UNTESTED_DISAPPEARS}})+
scalar(@{$res->{$UNTESTED_APPEARS}})+
scalar(@{$res->{$UNRESOLVED_PASS}})+
scalar(@{$res->{$UNRESOLVED_XFAIL}})+
+ scalar(@{$res->{$UNRESOLVED_UNSUPPORTED}})+
+ scalar(@{$res->{$UNRESOLVED_UNTESTED}})+
scalar(@{$res->{$UNRESOLVED_DISAPPEARS}})+
scalar(@{$res->{$UNHANDLED_CASES}});
printf "$col_pink"."o MINOR TO BE CHECKED:\n";
printf " +------------------------------------------+---------+\n";
- printf " | %-40s | %7d |\n", $XFAIL_APPEARS, scalar(@{$res->{$XFAIL_APPEARS}}) if (scalar(@{$res->{$XFAIL_APPEARS}}));
- printf " | %-40s | %7d |\n", $FAIL_XFAIL, scalar(@{$res->{$FAIL_XFAIL}}) if (scalar(@{$res->{$FAIL_XFAIL}}));
printf " | %-40s | %7d |\n", $PASS_XFAIL, scalar(@{$res->{$PASS_XFAIL}}) if (scalar(@{$res->{$PASS_XFAIL}}));
- printf " | %-40s | %7d |\n", $XPASS_XFAIL, scalar(@{$res->{$XPASS_XFAIL}}) if (scalar(@{$res->{$XPASS_XFAIL}}));
- printf " | %-40s | %7d |\n", $FAIL_DISAPPEARS, scalar(@{$res->{$FAIL_DISAPPEARS}}) if (scalar(@{$res->{$FAIL_DISAPPEARS}}));
+ printf " | %-40s | %7d |\n", $PASS_APPEARS, scalar(@{$res->{$PASS_APPEARS}}) if (scalar(@{$res->{$PASS_APPEARS}}));
+ printf " | %-40s | %7d |\n", $PASS_UNSUPPORTED, scalar(@{$res->{$PASS_UNSUPPORTED}}) if (scalar(@{$res->{$PASS_UNSUPPORTED}}));
+ printf " | %-40s | %7d |\n", $PASS_UNTESTED, scalar(@{$res->{$PASS_UNTESTED}}) if (scalar(@{$res->{$PASS_UNTESTED}}));
printf " | %-40s | %7d |\n", $XPASS_PASS, scalar(@{$res->{$XPASS_PASS}}) if (scalar(@{$res->{$XPASS_PASS}}));
- printf " | %-40s | %7d |\n", $XFAIL_PASS, scalar(@{$res->{$XFAIL_PASS}}) if (scalar(@{$res->{$XFAIL_PASS}}));
- printf " | %-40s | %7d |\n", $XFAIL_XPASS, scalar(@{$res->{$XFAIL_XPASS}}) if (scalar(@{$res->{$XFAIL_XPASS}}));
+ printf " | %-40s | %7d |\n", $XPASS_XFAIL, scalar(@{$res->{$XPASS_XFAIL}}) if (scalar(@{$res->{$XPASS_XFAIL}}));
+ printf " | %-40s | %7d |\n", $XPASS_UNSUPPORTED, scalar(@{$res->{$XPASS_UNSUPPORTED}}) if (scalar(@{$res->{$XPASS_UNSUPPORTED}}));
+ printf " | %-40s | %7d |\n", $XPASS_UNTESTED, scalar(@{$res->{$XPASS_UNTESTED}}) if (scalar(@{$res->{$XPASS_UNTESTED}}));
+ printf " | %-40s | %7d |\n", $XPASS_DISAPPEARS, scalar(@{$res->{$XPASS_DISAPPEARS}}) if (scalar(@{$res->{$XPASS_DISAPPEARS}}));
printf " | %-40s | %7d |\n", $FAIL_PASS, scalar(@{$res->{$FAIL_PASS}}) if (scalar(@{$res->{$FAIL_PASS}}));
printf " | %-40s | %7d |\n", $FAIL_XPASS, scalar(@{$res->{$FAIL_XPASS}}) if (scalar(@{$res->{$FAIL_XPASS}}));
- printf " | %-40s | %7d |\n", $PASS_APPEARS, scalar(@{$res->{$PASS_APPEARS}}) if (scalar(@{$res->{$PASS_APPEARS}}));
+ printf " | %-40s | %7d |\n", $FAIL_XFAIL, scalar(@{$res->{$FAIL_XFAIL}}) if (scalar(@{$res->{$FAIL_XFAIL}}));
+ printf " | %-40s | %7d |\n", $FAIL_UNSUPPORTED, scalar(@{$res->{$FAIL_UNSUPPORTED}}) if (scalar(@{$res->{$FAIL_UNSUPPORTED}}));
+ printf " | %-40s | %7d |\n", $FAIL_UNTESTED, scalar(@{$res->{$FAIL_UNTESTED}}) if (scalar(@{$res->{$FAIL_UNTESTED}}));
+ printf " | %-40s | %7d |\n", $FAIL_DISAPPEARS, scalar(@{$res->{$FAIL_DISAPPEARS}}) if (scalar(@{$res->{$FAIL_DISAPPEARS}}));
+ printf " | %-40s | %7d |\n", $XFAIL_PASS, scalar(@{$res->{$XFAIL_PASS}}) if (scalar(@{$res->{$XFAIL_PASS}}));
+ printf " | %-40s | %7d |\n", $XFAIL_XPASS, scalar(@{$res->{$XFAIL_XPASS}}) if (scalar(@{$res->{$XFAIL_XPASS}}));
+ printf " | %-40s | %7d |\n", $XFAIL_UNSUPPORTED, scalar(@{$res->{$XFAIL_UNSUPPORTED}}) if (scalar(@{$res->{$XFAIL_UNSUPPORTED}}));
+ printf " | %-40s | %7d |\n", $XFAIL_UNTESTED, scalar(@{$res->{$XFAIL_UNTESTED}}) if (scalar(@{$res->{$XFAIL_UNTESTED}}));
+ printf " | %-40s | %7d |\n", $XFAIL_APPEARS, scalar(@{$res->{$XFAIL_APPEARS}}) if (scalar(@{$res->{$XFAIL_APPEARS}}));
printf " | %-40s | %7d |\n", $UNSUPPORTED_PASS, scalar(@{$res->{$UNSUPPORTED_PASS}}) if (scalar(@{$res->{$UNSUPPORTED_PASS}}));
printf " | %-40s | %7d |\n", $UNSUPPORTED_XFAIL, scalar(@{$res->{$UNSUPPORTED_XFAIL}}) if (scalar(@{$res->{$UNSUPPORTED_XFAIL}}));
+ printf " | %-40s | %7d |\n", $UNSUPPORTED_UNTESTED, scalar(@{$res->{$UNSUPPORTED_UNTESTED}}) if (scalar(@{$res->{$UNSUPPORTED_UNTESTED}}));
printf " | %-40s | %7d |\n", $UNSUPPORTED_DISAPPEARS, scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}}) if (scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}}));
printf " | %-40s | %7d |\n", $UNSUPPORTED_APPEARS, scalar(@{$res->{$UNSUPPORTED_APPEARS}}) if (scalar(@{$res->{$UNSUPPORTED_APPEARS}}));
printf " | %-40s | %7d |\n", $UNTESTED_PASS, scalar(@{$res->{$UNTESTED_PASS}}) if (scalar(@{$res->{$UNTESTED_PASS}}));
printf " | %-40s | %7d |\n", $UNTESTED_XFAIL, scalar(@{$res->{$UNTESTED_XFAIL}}) if (scalar(@{$res->{$UNTESTED_XFAIL}}));
+ printf " | %-40s | %7d |\n", $UNTESTED_UNSUPPORTED, scalar(@{$res->{$UNTESTED_UNSUPPORTED}}) if (scalar(@{$res->{$UNTESTED_UNSUPPORTED}}));
printf " | %-40s | %7d |\n", $UNTESTED_DISAPPEARS, scalar(@{$res->{$UNTESTED_DISAPPEARS}}) if (scalar(@{$res->{$UNTESTED_DISAPPEARS}}));
printf " | %-40s | %7d |\n", $UNTESTED_APPEARS, scalar(@{$res->{$UNTESTED_APPEARS}}) if (scalar(@{$res->{$UNTESTED_APPEARS}}));
printf " | %-40s | %7d |\n", $UNRESOLVED_PASS, scalar(@{$res->{$UNRESOLVED_PASS}}) if (scalar(@{$res->{$UNRESOLVED_PASS}}));
printf " | %-40s | %7d |\n", $UNRESOLVED_XFAIL, scalar(@{$res->{$UNRESOLVED_XFAIL}}) if (scalar(@{$res->{$UNRESOLVED_XFAIL}}));
+ printf " | %-40s | %7d |\n", $UNRESOLVED_UNSUPPORTED, scalar(@{$res->{$UNRESOLVED_UNSUPPORTED}}) if (scalar(@{$res->{$UNRESOLVED_UNSUPPORTED}}));
+ printf " | %-40s | %7d |\n", $UNRESOLVED_UNTESTED, scalar(@{$res->{$UNRESOLVED_UNTESTED}}) if (scalar(@{$res->{$UNRESOLVED_UNTESTED}}));
printf " | %-40s | %7d |\n", $UNRESOLVED_DISAPPEARS, scalar(@{$res->{$UNRESOLVED_DISAPPEARS}}) if (scalar(@{$res->{$UNRESOLVED_DISAPPEARS}}));
printf " | %-40s | %7d |\n", $UNHANDLED_CASES, scalar(@{$res->{$UNHANDLED_CASES}}) if (scalar(@{$res->{$UNHANDLED_CASES}}));
printf " | %-40s | %7d |\n", $UNSTABLE_CASES, scalar(@{$res->{$UNSTABLE_CASES}}) if (scalar(@{$res->{$UNSTABLE_CASES}}));
@@ -657,30 +763,45 @@
if ($long)
{
+ print_tclist($PASS_XFAIL, @{$res->{$PASS_XFAIL}});
+ print_tclist($PASS_APPEARS, @{$res->{$PASS_APPEARS}});
+ print_tclist($PASS_UNSUPPORTED, @{$res->{$PASS_UNSUPPORTED}});
+ print_tclist($PASS_UNTESTED, @{$res->{$PASS_UNTESTED}});
print_tclist($XPASS_PASS, @{$res->{$XPASS_PASS}});
+ print_tclist($XPASS_XFAIL, @{$res->{$XPASS_XFAIL}});
+ print_tclist($XPASS_UNSUPPORTED, @{$res->{$XPASS_UNSUPPORTED}});
+ print_tclist($XPASS_UNTESTED, @{$res->{$XPASS_UNTESTED}});
+ print_tclist($XPASS_DISAPPEARS, @{$res->{$XPASS_DISAPPEARS}});
print_tclist($XFAIL_PASS, @{$res->{$XFAIL_PASS}});
print_tclist($XFAIL_XPASS, @{$res->{$XFAIL_XPASS}});
print_tclist($FAIL_PASS, @{$res->{$FAIL_PASS}});
print_tclist($FAIL_XPASS, @{$res->{$FAIL_XPASS}});
- print_tclist($FAIL_DISAPPEARS, @{$res->{$FAIL_DISAPPEARS}});
- print_tclist($XFAIL_APPEARS, @{$res->{$XFAIL_APPEARS}});
print_tclist($FAIL_XFAIL, @{$res->{$FAIL_XFAIL}});
- print_tclist($PASS_XFAIL, @{$res->{$PASS_XFAIL}});
- print_tclist($XPASS_XFAIL, @{$res->{$XPASS_XFAIL}});
- print_tclist($UNHANDLED_CASES, @{$res->{$UNHANDLED_CASES}});
+ print_tclist($FAIL_UNSUPPORTED, @{$res->{$FAIL_UNSUPPORTED}});
+ print_tclist($FAIL_UNTESTED, @{$res->{$FAIL_UNTESTED}});
+ print_tclist($FAIL_DISAPPEARS, @{$res->{$FAIL_DISAPPEARS}});
+ print_tclist($XFAIL_PASS, @{$res->{$XFAIL_PASS}});
+ print_tclist($XFAIL_XPASS, @{$res->{$XFAIL_XPASS}});
+ print_tclist($XFAIL_UNSUPPORTED, @{$res->{$XFAIL_UNSUPPORTED}});
+ print_tclist($XFAIL_UNTESTED, @{$res->{$XFAIL_UNTESTED}});
+ print_tclist($XFAIL_APPEARS, @{$res->{$XFAIL_APPEARS}});
print_tclist($UNSUPPORTED_PASS, @{$res->{$UNSUPPORTED_PASS}});
print_tclist($UNSUPPORTED_XFAIL, @{$res->{$UNSUPPORTED_XFAIL}});
+ print_tclist($UNSUPPORTED_UNTESTED, @{$res->{$UNSUPPORTED_UNTESTED}});
print_tclist($UNSUPPORTED_DISAPPEARS, @{$res->{$UNSUPPORTED_DISAPPEARS}});
print_tclist($UNSUPPORTED_APPEARS, @{$res->{$UNSUPPORTED_APPEARS}});
print_tclist($UNTESTED_PASS, @{$res->{$UNTESTED_PASS}});
print_tclist($UNTESTED_XFAIL, @{$res->{$UNTESTED_XFAIL}});
+ print_tclist($UNTESTED_UNSUPPORTED, @{$res->{$UNTESTED_UNSUPPORTED}});
print_tclist($UNTESTED_DISAPPEARS, @{$res->{$UNTESTED_DISAPPEARS}});
print_tclist($UNTESTED_APPEARS, @{$res->{$UNTESTED_APPEARS}});
print_tclist($UNRESOLVED_PASS, @{$res->{$UNRESOLVED_PASS}});
print_tclist($UNRESOLVED_XFAIL, @{$res->{$UNRESOLVED_XFAIL}});
+ print_tclist($UNRESOLVED_UNSUPPORTED, @{$res->{$UNRESOLVED_UNSUPPORTED}});
+ print_tclist($UNRESOLVED_UNTESTED, @{$res->{$UNRESOLVED_UNTESTED}});
print_tclist($UNRESOLVED_DISAPPEARS, @{$res->{$UNRESOLVED_DISAPPEARS}});
+ print_tclist($UNHANDLED_CASES, @{$res->{$UNHANDLED_CASES}});
print_tclist($UNSTABLE_CASES, @{$res->{$UNSTABLE_CASES}});
- print_tclist($PASS_APPEARS, @{$res->{$PASS_APPEARS}});
}
printf "$col_reset\n";
}