| #!/usr/bin/env perl |
| use strict; |
| use warnings; |
| |
| use File::Glob; |
| use Getopt::Long; |
| use Term::ANSIColor qw(:constants); |
| use File::Basename; |
| use Cwd; |
| |
| my $app = $0; |
| |
| sub read_sum($); |
| sub read_unstable($); |
| sub dump_result($); |
| sub compare_results($$); |
| sub usage(); |
| sub print_compare_results_summary($$); |
| sub nothing($$$$); |
| |
| my $PASS_PASS = "Still passes [PASS => PASS]"; |
| my $PASS_XPASS = "PASS now XPASS [PASS =>XPASS]"; |
| my $PASS_FAIL = "PASS now FAIL [PASS => FAIL]"; |
| my $PASS_XFAIL = "PASS now XFAIL [PASS =>XFAIL]"; |
| my $PASS_UNSUPPORTED = "PASS now UNSUPPORTED [PASS =>UNSUP]"; |
| my $PASS_UNRESOLVED = "PASS now UNRESOLVED [PASS =>UNRES]"; |
| my $PASS_UNTESTED = "PASS now UNTESTED [PASS =>UNTES]"; |
| my $PASS_DISAPPEARS = "PASS disappears [PASS => ]"; |
| my $PASS_APPEARS = "New PASS [ => PASS]"; |
| my $PASSED_NOW_TIMEOUTS = "Timeout [PASS =>T.OUT]"; |
| |
| my $XPASS_PASS = "XPASS now PASS [XPASS=> PASS]"; |
| my $XPASS_XPASS = "Still xpass [XPASS=>XPASS]"; |
| my $XPASS_FAIL = "XPASS now FAIL [XPASS=> FAIL]"; |
| my $XPASS_XFAIL = "XPASS now XFAIL [XPASS=>XFAIL]"; |
| my $XPASS_UNSUPPORTED = "XPASS now UNSUPPORTED [XPASS=>UNSUP]"; |
| my $XPASS_UNRESOLVED = "XPASS now UNRESOLVED [XPASS=>UNRES]"; |
| my $XPASS_UNTESTED = "XPASS now UNTESTED [XPASS=>UNTES]"; |
| my $XPASS_DISAPPEARS = "XPASS disappears [XPASS=> ]"; |
| my $XPASS_APPEARS = "XPASS appears [ =>XPASS]"; |
| |
| my $FAIL_PASS = "FAIL now PASS [FAIL => PASS]"; |
| my $FAIL_XPASS = "FAIL now XPASS [FAIL =>XPASS]"; |
| my $FAIL_FAIL = "Still fails [FAIL => FAIL]"; |
| my $FAIL_XFAIL = "FAIL now XFAIL [FAIL =>XFAIL]"; |
| my $FAIL_UNSUPPORTED = "FAIL now UNSUPPORTED [FAIL =>UNSUP]"; |
| my $FAIL_UNRESOLVED = "FAIL now UNRESOLVED [FAIL =>UNRES]"; |
| my $FAIL_UNTESTED = "FAIL now UNTESTED [FAIL =>UNTES]"; |
| my $FAIL_DISAPPEARS = "FAIL disappears [FAIL => ]"; |
| my $FAIL_APPEARS = "FAIL appears [ => FAIL]"; |
| |
| my $XFAIL_PASS = "XFAIL now PASS [XFAIL=> PASS]"; |
| my $XFAIL_XPASS = "XFAIL now XPASS [XFAIL=>XPASS]"; |
| my $XFAIL_FAIL = "XFAIL now FAIL [XFAIL=> FAIL]"; |
| my $XFAIL_XFAIL = "Still xfail [XFAIL=>XFAIL]"; |
| my $XFAIL_UNSUPPORTED = "XFAIL now UNSUPPORTED [XFAIL=>UNSUP]"; |
| my $XFAIL_UNRESOLVED = "XFAIL now UNRESOLVED [XFAIL=>UNRES]"; |
| my $XFAIL_UNTESTED = "XFAIL now UNTESTED [XFAIL=>UNTES]"; |
| my $XFAIL_DISAPPEARS = "XFAIL disappears [XFAIL=> ]"; |
| my $XFAIL_APPEARS = "XFAIL appears [ =>XFAIL]"; |
| |
| my $UNSUPPORTED_PASS = "UNSUPPORTED now PASS [UNSUP=> PASS]"; |
| my $UNSUPPORTED_XPASS = "UNSUPPORTED now XPASS [UNSUP=>XPASS]"; |
| my $UNSUPPORTED_FAIL = "UNSUPPORTED now FAIL [UNSUP=> FAIL]"; |
| my $UNSUPPORTED_XFAIL = "UNSUPPORTED now XFAIL [UNSUP=>XFAIL]"; |
| my $UNSUPPORTED_UNSUPPORTED= "UNSUPP now UNSUPP [UNSUP=>UNSUP]"; |
| my $UNSUPPORTED_UNRESOLVED= "UNSUPP now UNRESOLVED [UNSUP=>UNRES]"; |
| my $UNSUPPORTED_UNTESTED = "UNSUPP now UNTESTED [UNSUP=>UNTES]"; |
| my $UNSUPPORTED_DISAPPEARS= "UNSUPPORTED disappears [UNSUP=> ]"; |
| my $UNSUPPORTED_APPEARS = "UNSUPPORTED appears [ =>UNSUP]"; |
| |
| my $UNTESTED_PASS = "UNTESTED now PASS [UNTES=> PASS]"; |
| my $UNTESTED_XPASS = "UNTESTED now XPASS [UNTES=>XPASS]"; |
| my $UNTESTED_FAIL = "UNTESTED now FAIL [UNTES=> FAIL]"; |
| my $UNTESTED_XFAIL = "UNTESTED now XFAIL [UNTES=>XFAIL]"; |
| my $UNTESTED_UNSUPPORTED = "UNTESTED now UNSUPP [UNTES=>UNSUP]"; |
| my $UNTESTED_UNRESOLVED = "UNTESTED now UNRESOLVED [UNTES=>UNRES]"; |
| my $UNTESTED_UNTESTED = "UNTESTED now UNTESTED [UNTES=>UNTES]"; |
| my $UNTESTED_DISAPPEARS = "UNTESTED disappears [UNTES=> ]"; |
| my $UNTESTED_APPEARS = "UNTESTED appears [ =>UNTES]"; |
| |
| my $UNRESOLVED_PASS = "UNRESOLVED now PASS [UNRES=> PASS]"; |
| my $UNRESOLVED_XPASS = "UNRESOLVED now XPASS [UNRES=>XPASS]"; |
| my $UNRESOLVED_FAIL = "UNRESOLVED now FAIL [UNRES=> FAIL]"; |
| my $UNRESOLVED_XFAIL = "UNRESOLVED now XFAIL [UNRES=>XFAIL]"; |
| my $UNRESOLVED_UNSUPPORTED= "UNRESOLVED now UNSUPP [UNRES=>UNSUP]"; |
| my $UNRESOLVED_UNRESOLVED = "UNRESOLVED now UNRESOLVED [UNRES=>UNRES]"; |
| my $UNRESOLVED_UNTESTED = "UNRESOLVED now UNTESTED [UNRES=>UNTES]"; |
| my $UNRESOLVED_DISAPPEARS = "UNRESOLVED disappears [UNRES=> ]"; |
| my $UNRESOLVED_APPEARS = "UNRESOLVED appears [ =>UNRES]"; |
| |
| my $ERROR_DISAPPEARS = "ERROR disappears [ERROR=> ]"; |
| my $ERROR_APPEARS = "ERROR appears [ =>ERROR]"; |
| |
| my $UNHANDLED_CASES = "Unhandled cases [ ..??.. ]"; |
| my $UNSTABLE_CASES = "Unstable cases, ignored [~RANDOM ]"; |
| my $HWDEP_CASES = "HW dependent cases [HW-DEPENDENT]"; |
| |
| # Handle status transitions: |
| # 'was' is the reference status |
| # 'is' is the current status |
| # 'cat' is the short name for the transition type |
| # 'handler' is a function handling the transition, if non-trivial |
| # report_hwdep is a boolean controlling whether to report the |
| # transition even if flagged as hardware-dependent |
| my @handler_list = ( |
| {was=>"PASS", is=>"PASS", cat=>$PASS_PASS, report_hwdep=>"false"}, |
| {was=>"PASS", is=>"XPASS", cat=>$PASS_XPASS, report_hwdep=>"true"}, |
| {was=>"PASS", is=>"FAIL", handler=>\&handle_pass_fail, report_hwdep=>"true"}, |
| {was=>"PASS", is=>"XFAIL", cat=>$PASS_XFAIL, report_hwdep=>"true"}, |
| {was=>"PASS", is=>"UNSUPPORTED",cat=>$PASS_UNSUPPORTED, report_hwdep=>"false"}, |
| {was=>"PASS", is=>"UNTESTED", cat=>$PASS_UNTESTED, report_hwdep=>"true"}, |
| {was=>"PASS", is=>"UNRESOLVED",cat=>$PASS_UNRESOLVED, report_hwdep=>"true"}, |
| {was=>"PASS", is=>"NO_EXIST", cat=>$PASS_DISAPPEARS, report_hwdep=>"false"}, |
| {was=>"NO_EXIST", is=>"PASS", cat=>$PASS_APPEARS, report_hwdep=>"false"}, |
| |
| {was=>"XPASS", is=>"PASS", cat=>$XPASS_PASS, report_hwdep=>"true"}, |
| {was=>"XPASS", is=>"XPASS", cat=>$XPASS_XPASS, report_hwdep=>"true"}, |
| {was=>"XPASS", is=>"FAIL", cat=>$XPASS_FAIL, report_hwdep=>"true"}, |
| {was=>"XPASS", is=>"XFAIL", cat=>$XPASS_XFAIL, report_hwdep=>"true"}, |
| {was=>"XPASS", is=>"UNSUPPORTED",cat=>$XPASS_UNSUPPORTED, report_hwdep=>"true"}, |
| {was=>"XPASS", is=>"UNTESTED", cat=>$XPASS_UNTESTED, report_hwdep=>"true"}, |
| {was=>"XPASS", is=>"UNRESOLVED",cat=>$XPASS_UNRESOLVED, report_hwdep=>"true"}, |
| {was=>"XPASS", is=>"NO_EXIST", cat=>$XPASS_DISAPPEARS, report_hwdep=>"true"}, |
| {was=>"NO_EXIST", is=>"XPASS", cat=>$XPASS_APPEARS, report_hwdep=>"true"}, |
| |
| {was=>"FAIL", is=>"PASS", cat=>$FAIL_PASS, report_hwdep=>"true"}, |
| {was=>"FAIL", is=>"XPASS", cat=>$FAIL_XPASS, report_hwdep=>"true"}, |
| {was=>"FAIL", is=>"FAIL", cat=>$FAIL_FAIL, report_hwdep=>"true"}, |
| {was=>"FAIL", is=>"XFAIL", cat=>$FAIL_XFAIL, report_hwdep=>"true"}, |
| {was=>"FAIL", is=>"UNSUPPORTED",cat=>$FAIL_UNSUPPORTED, report_hwdep=>"true"}, |
| {was=>"FAIL", is=>"UNTESTED", cat=>$FAIL_UNTESTED, report_hwdep=>"true"}, |
| {was=>"FAIL", is=>"UNRESOLVED",cat=>$FAIL_UNRESOLVED, report_hwdep=>"true"}, |
| {was=>"FAIL", is=>"NO_EXIST", cat=>$FAIL_DISAPPEARS, report_hwdep=>"true"}, |
| {was=>"NO_EXIST", is=>"FAIL", cat=>$FAIL_APPEARS, report_hwdep=>"true"}, |
| |
| {was=>"XFAIL", is=>"PASS", cat=>$XFAIL_PASS, report_hwdep=>"true"}, |
| {was=>"XFAIL", is=>"XPASS", cat=>$XFAIL_XPASS, report_hwdep=>"true"}, |
| {was=>"XFAIL", is=>"FAIL", cat=>$XFAIL_FAIL, report_hwdep=>"true"}, |
| {was=>"XFAIL", is=>"XFAIL", cat=>$XFAIL_XFAIL, report_hwdep=>"true"}, |
| {was=>"XFAIL", is=>"UNSUPPORTED",cat=>$XFAIL_UNSUPPORTED, report_hwdep=>"true"}, |
| {was=>"XFAIL", is=>"UNTESTED", cat=>$XFAIL_UNTESTED, report_hwdep=>"true"}, |
| {was=>"XFAIL", is=>"UNRESOLVED",cat=>$XFAIL_UNRESOLVED, report_hwdep=>"true"}, |
| {was=>"XFAIL", is=>"NO_EXIST", cat=>$XFAIL_DISAPPEARS, report_hwdep=>"true"}, |
| {was=>"NO_EXIST", is=>"XFAIL", cat=>$XFAIL_APPEARS, report_hwdep=>"true"}, |
| |
| {was=>"UNSUPPORTED", is=>"PASS", cat=>$UNSUPPORTED_PASS, report_hwdep=>"true"}, |
| {was=>"UNSUPPORTED", is=>"XPASS", cat=>$UNSUPPORTED_XPASS, report_hwdep=>"true"}, |
| {was=>"UNSUPPORTED", is=>"FAIL", cat=>$UNSUPPORTED_FAIL, report_hwdep=>"true"}, |
| {was=>"UNSUPPORTED", is=>"XFAIL", cat=>$UNSUPPORTED_XFAIL, report_hwdep=>"true"}, |
| {was=>"UNSUPPORTED", is=>"UNSUPPORTED",cat=>$UNSUPPORTED_UNSUPPORTED, report_hwdep=>"true"}, |
| {was=>"UNSUPPORTED", is=>"UNTESTED", cat=>$UNSUPPORTED_UNTESTED, report_hwdep=>"true"}, |
| {was=>"UNSUPPORTED", is=>"UNRESOLVED",cat=>$UNSUPPORTED_UNRESOLVED, report_hwdep=>"true"}, |
| {was=>"UNSUPPORTED", is=>"NO_EXIST", cat=>$UNSUPPORTED_DISAPPEARS, report_hwdep=>"true"}, |
| {was=>"NO_EXIST", is=>"UNSUPPORTED", cat=>$UNSUPPORTED_APPEARS, report_hwdep=>"true"}, |
| |
| {was=>"UNTESTED", is=>"PASS", cat=>$UNTESTED_PASS, report_hwdep=>"true"}, |
| {was=>"UNTESTED", is=>"XPASS", cat=>$UNTESTED_XPASS, report_hwdep=>"true"}, |
| {was=>"UNTESTED", is=>"FAIL", cat=>$UNTESTED_FAIL, report_hwdep=>"true"}, |
| {was=>"UNTESTED", is=>"XFAIL", cat=>$UNTESTED_XFAIL, report_hwdep=>"true"}, |
| {was=>"UNTESTED", is=>"UNSUPPORTED",cat=>$UNTESTED_UNSUPPORTED,report_hwdep=>"true"}, |
| {was=>"UNTESTED", is=>"UNTESTED", cat=>$UNTESTED_UNTESTED, report_hwdep=>"true"}, |
| {was=>"UNTESTED", is=>"UNRESOLVED",cat=>$UNTESTED_UNRESOLVED, report_hwdep=>"true"}, |
| {was=>"UNTESTED", is=>"NO_EXIST", cat=>$UNTESTED_DISAPPEARS, report_hwdep=>"true"}, |
| {was=>"NO_EXIST", is=>"UNTESTED", cat=>$UNTESTED_APPEARS, report_hwdep=>"true"}, |
| |
| {was=>"UNRESOLVED", is=>"PASS", cat=>$UNRESOLVED_PASS, report_hwdep=>"true"}, |
| {was=>"UNRESOLVED", is=>"XPASS", cat=>$UNRESOLVED_XPASS, report_hwdep=>"true"}, |
| {was=>"UNRESOLVED", is=>"FAIL", cat=>$UNRESOLVED_FAIL, report_hwdep=>"true"}, |
| {was=>"UNRESOLVED", is=>"XFAIL", cat=>$UNRESOLVED_XFAIL, report_hwdep=>"true"}, |
| {was=>"UNRESOLVED", is=>"UNSUPPORTED",cat=>$UNRESOLVED_UNSUPPORTED, report_hwdep=>"true"}, |
| {was=>"UNRESOLVED", is=>"UNTESTED", cat=>$UNRESOLVED_UNTESTED, report_hwdep=>"true"}, |
| {was=>"UNRESOLVED", is=>"UNRESOLVED",cat=>$UNRESOLVED_UNRESOLVED, report_hwdep=>"true"}, |
| {was=>"UNRESOLVED", is=>"NO_EXIST", cat=>$UNRESOLVED_DISAPPEARS, report_hwdep=>"true"}, |
| {was=>"NO_EXIST", is=>"UNRESOLVED", cat=>$UNRESOLVED_APPEARS, report_hwdep=>"true"}, |
| |
| {was=>"ERROR", is=>"NO_EXIST", cat=>$ERROR_DISAPPEARS, report_hwdep=>"true"}, |
| {was=>"NO_EXIST", is=>"ERROR", cat=>$ERROR_APPEARS, report_hwdep=>"true"}, |
| |
| # {was=>"NO_EXIST", is=>"NO_EXIST", handler=>\&handle_not_yet_supported} |
| ); |
| |
| ###################################################### |
| # TREAT ARGUMENTS |
| |
| my $verbose=0; |
| my $quiet=0; |
| my $long=0; |
| my $short=0; |
| my $debug=0; |
| my ($testroot, $basename); |
| my ($ref_file_name, $res_file_name); |
| my $nounstable=0; |
| my $unstablefile=0; |
| my @unstable_markers=(); |
| |
| my $no_hwdep=0; # Ignore hw-dependent flag (ie. consider all tests as *not* hw-dependent |
| my $hwdep_file=0; # File containing the list of hw-dependent tests |
| my @hwdep_markers=(); # Optional markers when parsing hwdep_file |
| |
| my $ratio_thresh = 0.95; # Warn if pass ratio is below this threshold |
| |
| GetOptions ("l" => \$long, |
| "s" => \$short, |
| "q" => \$quiet, |
| "v" => \$verbose, |
| "dbg" => \$debug, |
| "testroot=s" => \$testroot, |
| "basename=s" => \$basename, |
| "pass-thresh=f" => \$ratio_thresh, |
| "no-unstable" => \$nounstable, |
| "unstable-tests=s" => \$unstablefile, |
| "unstable-marker=s" => \@unstable_markers, |
| "no-hwdep" => \$no_hwdep, |
| "hwdep-tests=s" => \$hwdep_file, |
| "hwdep-marker=s" => \@hwdep_markers); |
| |
| $ref_file_name = $ARGV[0] if ($#ARGV == 1); |
| $res_file_name = $ARGV[1] if ($#ARGV == 1); |
| |
| $ref_file_name = $testroot."/expected_results/".$basename if ($testroot and $basename); |
| $res_file_name = $testroot."/testing/run/".$basename if ($testroot and $basename); |
| &usage if (not $ref_file_name or not $res_file_name); |
| |
| my ($col_boldred, $col_red, $col_boldgreen, $col_green, $col_boldpink, $col_pink, $col_reset) |
| = ("\033[31;1m","\033[31;3m","\033[32;1m","\033[32;3m","\033[35;1m","\033[35;2m","\033[0m"); |
| ($col_boldred, $col_red, $col_boldgreen, $col_green, $col_boldpink, $col_pink, $col_reset) |
| = ("","","","","","","") if (not I_am_interactive()); |
| |
| ###################################################### |
| # MAIN PROGRAM |
| # print "comparing $ref_file_name $res_file_name\n"; |
| |
| # If none of the 2 .sum exists, nothing to compare: exit early. |
| exit 0 if ( (! -e $ref_file_name) && (! -e $res_file_name )); |
| |
| my $ref = read_sum($ref_file_name) ; |
| my $res = read_sum($res_file_name) ; |
| my @unstablelist = (); |
| |
| @unstablelist = read_unstable($unstablefile) if ($unstablefile ne 0); |
| |
| # Import list of hw-dependent files, if provided |
| my @hwdep_list = (); |
| |
| @hwdep_list = read_unstable($hwdep_file) if ($hwdep_file ne 0); |
| |
| compare_results($ref, $res); |
| |
| my $final_result = print_compare_results_summary($ref, $res); |
| |
| exit $final_result; |
| |
| ###################################################### |
| # UTILITIES |
| |
| sub empty_result() |
| { |
| my %empty_result;# = {PASS=>0, FAIL=>0, XPASS=>0, XFAIL=>0, UNSUPPORTED=>0, UNTESTED=>0, UNRESOLVED=>0, ERROR=>0}; |
| $empty_result{PASS}=$empty_result{FAIL}=$empty_result{XPASS}=$empty_result{XFAIL}=0; |
| $empty_result{UNSUPPORTED}=$empty_result{UNTESTED}=$empty_result{UNRESOLVED}=$empty_result{NO_EXIST}=$empty_result{ERROR}=0; |
| $empty_result{NB}=0; |
| return \%empty_result; |
| } |
| sub I_am_interactive { |
| return -t STDIN && -t STDOUT; |
| } |
| sub usage() |
| { |
| print "Usage : $app <ref_file.sum> <result_file.sum>\n"; |
| exit 1; |
| } |
| |
| |
| ###################################################### |
| # PARSING |
| sub read_sum($) |
| { |
| my ($sum_file) = @_; |
| my $res = empty_result(); |
| my %testcases; |
| my %unsupported; |
| $res->{testcases} = \%testcases; |
| my $pending_timeout=0; |
| my $current_tool=""; |
| my $current_exp=""; |
| |
| $res->{EXEC}->{PASS} = 0; |
| $res->{EXEC}->{FAIL} = 0; |
| |
| open SUMFILE, $sum_file or die $!; |
| while (<SUMFILE>) |
| { |
| if (m/^(PASS|XPASS|FAIL|XFAIL|UNSUPPORTED|UNTESTED|UNRESOLVED|ERROR): *(.*)/) |
| { |
| my ($diag,$tc) = ($1,$2); |
| my %tcresult; |
| $tc =~ s/==[0-9]+== Shadow memory range interleaves with an existing memory mapping. ASan cannot proceed correctly./==<pid>== Shadow memory range interleaves with an existing memory mapping. ASan cannot proceed correctly./; |
| # Prefix test name wih .exp filename to help report |
| # regressions/run bisect. |
| $tc = "$current_tool:$current_exp=$tc"; |
| $testcases{$tc} = empty_result() if (not exists $testcases{$tc}); |
| $testcases{$tc}->{$diag}++; |
| $testcases{$tc}->{HAS_TIMED_OUT} = $pending_timeout; |
| $testcases{$tc}->{TOOL_EXP} = "$current_tool:$current_exp"; |
| $testcases{$tc}->{NB}++; |
| $pending_timeout = 0; |
| $res->{$diag}++; |
| # Count execution tests, for a sanity check |
| if ($tc =~ /execution/) |
| { |
| $res->{EXEC}->{$diag}++; |
| } |
| } |
| elsif (m/WARNING: program timed out/) |
| { |
| $pending_timeout = 1; |
| } |
| elsif (m/^(# of expected passes|# of unexpected failures|# of expected failures|# of known failures|# of unsupported tests|# of untested testcases)\s+(.*)/) |
| { |
| $res->{"summary - "."$1"} = $2; |
| } |
| elsif (m/^\/.*\/([^\/]+)\s+version\s+(.*)/) |
| { |
| $res->{tool} = $1; |
| $res->{version} = $2; |
| $res->{version} =~ s/ [-(].*//; |
| } |
| elsif (m/^Running target .*/) |
| { |
| } |
| elsif (m/^Running (.*) \.\.\.*/) |
| { |
| $current_exp=$1; |
| $current_exp =~ s|.*/testsuite/||; |
| } |
| elsif (m/^\t\t=== (.*) tests ===/) |
| { |
| $current_tool=$1; |
| } |
| elsif (m/^\t\t=== (.*) Summary ===/) |
| { |
| $current_tool=""; |
| $current_exp=""; |
| } |
| } |
| close SUMFILE; |
| return $res; |
| } |
| |
| # Parse list of unstable/hw-dependent tests |
| sub read_unstable($) |
| { |
| my ($unstable_file) = @_; |
| my @unstable_tests = (); |
| |
| open UNSTABLEFILE, $unstable_file or die $!; |
| while (<UNSTABLEFILE>) |
| { |
| # Skip lines starting with '#', or with spaces only |
| if ((/^#/) || (/^[ ]*$/)) |
| { |
| } |
| else |
| { |
| chomp; |
| |
| my $test = $_; |
| |
| # Check if line is of type: target:testname |
| if (/^(.*?):/) |
| { |
| foreach my $unstable_marker (@unstable_markers) |
| { |
| if ($unstable_marker eq $1) { |
| # If target matches the one supplied as script |
| # argument, add the testname to the list |
| $test =~ s/.*?://; |
| push @unstable_tests, $test; |
| } |
| } |
| } else { |
| push @unstable_tests, $test; |
| } |
| } |
| } |
| close UNSTABLEFILE; |
| return @unstable_tests; |
| } |
| |
| ###################################################### |
| # DIFFING |
| sub handle_pass_fail($$$$) |
| { |
| my ($ref, $res, $diag_diag, $tc) = @_; |
| if ($res->{testcases}->{$tc}->{HAS_TIMED_OUT}) |
| { |
| push @{$res->{$PASSED_NOW_TIMEOUTS}}, $tc; |
| } |
| else |
| { |
| # This transition is flagged as hw-dependent if: |
| # - the name is in the list |
| # - the transition should not be reported (but be flagged as hw-dependent instead) |
| # - --no-hwdep was not provided |
| if ((grep { (index $tc,$_)!=-1} @hwdep_list) |
| && ($diag_diag->{report_hwdep} eq "false") |
| && ($no_hwdep == 0)) { |
| push @{$res->{$HWDEP_CASES}}, $tc; |
| } else { |
| push @{$res->{$PASS_FAIL}}, $tc; |
| } |
| } |
| } |
| |
| sub compare_results($$) |
| { |
| my ($ref, $res) = @_; |
| |
| @{$res->{$PASS_PASS}} = (); |
| @{$res->{$PASS_XPASS}} = (); |
| @{$res->{$PASS_FAIL}} = (); |
| @{$res->{$PASS_XFAIL}} = (); |
| @{$res->{$PASS_UNSUPPORTED}} = (); |
| @{$res->{$PASS_UNTESTED}} = (); |
| @{$res->{$PASS_UNRESOLVED}} = (); |
| @{$res->{$PASS_DISAPPEARS}} = (); |
| @{$res->{$PASS_APPEARS}} = (); |
| @{$res->{$PASSED_NOW_TIMEOUTS}} = (); |
| |
| @{$res->{$XPASS_PASS}} = (); |
| @{$res->{$XPASS_XPASS}} = (); |
| @{$res->{$XPASS_FAIL}} = (); |
| @{$res->{$XPASS_XFAIL}} = (); |
| @{$res->{$XPASS_UNSUPPORTED}} = (); |
| @{$res->{$XPASS_UNTESTED}} = (); |
| @{$res->{$XPASS_UNRESOLVED}} = (); |
| @{$res->{$XPASS_DISAPPEARS}} = (); |
| @{$res->{$XPASS_APPEARS}} = (); |
| |
| @{$res->{$FAIL_PASS}} = (); |
| @{$res->{$FAIL_XPASS}} = (); |
| @{$res->{$FAIL_FAIL}} = (); |
| @{$res->{$FAIL_XFAIL}} = (); |
| @{$res->{$FAIL_UNSUPPORTED}} = (); |
| @{$res->{$FAIL_UNTESTED}} = (); |
| @{$res->{$FAIL_UNRESOLVED}} = (); |
| @{$res->{$FAIL_DISAPPEARS}} = (); |
| @{$res->{$FAIL_APPEARS}} = (); |
| |
| @{$res->{$XFAIL_PASS}} = (); |
| @{$res->{$XFAIL_XPASS}} = (); |
| @{$res->{$XFAIL_FAIL}} = (); |
| @{$res->{$XFAIL_XFAIL}} = (); |
| @{$res->{$XFAIL_UNSUPPORTED}} = (); |
| @{$res->{$XFAIL_UNTESTED}} = (); |
| @{$res->{$XFAIL_UNRESOLVED}} = (); |
| @{$res->{$XFAIL_DISAPPEARS}} = (); |
| @{$res->{$XFAIL_APPEARS}} = (); |
| |
| @{$res->{$UNSUPPORTED_PASS}} = (); |
| @{$res->{$UNSUPPORTED_XPASS}} = (); |
| @{$res->{$UNSUPPORTED_FAIL}} = (); |
| @{$res->{$UNSUPPORTED_XFAIL}} = (); |
| @{$res->{$UNSUPPORTED_UNSUPPORTED}} = (); |
| @{$res->{$UNSUPPORTED_UNTESTED}} = (); |
| @{$res->{$UNSUPPORTED_UNRESOLVED}} = (); |
| @{$res->{$UNSUPPORTED_DISAPPEARS}} = (); |
| @{$res->{$UNSUPPORTED_APPEARS}} = (); |
| |
| @{$res->{$UNTESTED_PASS}} = (); |
| @{$res->{$UNTESTED_XPASS}} = (); |
| @{$res->{$UNTESTED_FAIL}} = (); |
| @{$res->{$UNTESTED_XFAIL}} = (); |
| @{$res->{$UNTESTED_UNSUPPORTED}} = (); |
| @{$res->{$UNTESTED_UNTESTED}} = (); |
| @{$res->{$UNTESTED_UNRESOLVED}} = (); |
| @{$res->{$UNTESTED_DISAPPEARS}} = (); |
| @{$res->{$UNTESTED_APPEARS}} = (); |
| |
| @{$res->{$UNRESOLVED_PASS}} = (); |
| @{$res->{$UNRESOLVED_XPASS}} = (); |
| @{$res->{$UNRESOLVED_FAIL}} = (); |
| @{$res->{$UNRESOLVED_XFAIL}} = (); |
| @{$res->{$UNRESOLVED_UNSUPPORTED}} = (); |
| @{$res->{$UNRESOLVED_UNTESTED}} = (); |
| @{$res->{$UNRESOLVED_UNRESOLVED}} = (); |
| @{$res->{$UNRESOLVED_DISAPPEARS}} = (); |
| @{$res->{$UNRESOLVED_APPEARS}} = (); |
| |
| @{$res->{$ERROR_DISAPPEARS}} = (); |
| @{$res->{$ERROR_APPEARS}} = (); |
| |
| @{$res->{$UNHANDLED_CASES}} = (); |
| @{$res->{$UNSTABLE_CASES}} = (); |
| @{$res->{$HWDEP_CASES}} = (); |
| |
| #### MERGE REF AND RES |
| foreach my $key (sort (keys %{$res->{testcases}})) |
| { |
| if (not exists $ref->{testcases}->{$key}) { |
| $ref->{testcases}->{$key} = empty_result(); |
| $ref->{testcases}->{$key}->{NO_EXIST} = 1; |
| $ref->{testcases}->{$key}->{TOOL_EXP} = $res->{testcases}->{$key}->{TOOL_EXP}; |
| } |
| |
| # Consistency check, useful in case of duplicate test names |
| if ($ref->{testcases}->{$key}->{NB} != $res->{testcases}->{$key}->{NB}) { |
| print "Number of occurrences mismatch $key (ref $ref->{testcases}->{$key}->{NB} vs $res->{testcases}->{$key}->{NB})\n" if ($debug); |
| if ($ref->{testcases}->{$key}->{NB} > $res->{testcases}->{$key}->{NB}) { |
| $res->{testcases}->{$key}->{NO_EXIST} = 1; |
| } elsif ($ref->{testcases}->{$key}->{NB} < $res->{testcases}->{$key}->{NB}) { |
| $ref->{testcases}->{$key}->{NO_EXIST} = 1; |
| } |
| } |
| } |
| foreach my $key (keys %{$ref->{testcases}}) |
| { |
| if (not exists $res->{testcases}->{$key}) |
| { |
| $res->{testcases}->{$key} = empty_result(); |
| $res->{testcases}->{$key}->{NO_EXIST} = 1; |
| $res->{testcases}->{$key}->{TOOL_EXP} = $ref->{testcases}->{$key}->{TOOL_EXP}; |
| } |
| |
| # No need for similar consistency check: since we are |
| # checking for tests whose number of occurrences are |
| # different, they are present in both ref and res, so |
| # no need to check again here. |
| } |
| |
| #### ACTIONS FOR EACH CASES |
| my %unstable_found; |
| |
| foreach my $key (sort (keys %{$ref->{testcases}})) |
| { |
| foreach my $diag_diag (@handler_list) |
| { |
| if ($ref->{testcases}->{$key}->{$diag_diag->{was}} != $res->{testcases}->{$key}->{$diag_diag->{was}} |
| and $ref->{testcases}->{$key}->{$diag_diag->{was}} |
| and $res->{testcases}->{$key}->{$diag_diag->{is}}) |
| { |
| print "$key ref $diag_diag->{was}:$ref->{testcases}->{$key}->{$diag_diag->{was}}, res $diag_diag->{was}:$res->{testcases}->{$key}->{$diag_diag->{was}}, res $diag_diag->{is}:$res->{testcases}->{$key}->{$diag_diag->{is}} \n" if ($debug); |
| # If testcase is listed as 'unstable' mark it as such |
| # and skip other processing to avoid listing it twice. |
| { |
| if (grep { (index $key,$_)!=-1} @unstablelist) |
| { |
| print "[unstable] $key\n" if ($debug); |
| $unstable_found{$key}=1; |
| } |
| else { |
| print "[$diag_diag->{was} => $diag_diag->{is}] $key\n" if ($debug); |
| if ($diag_diag->{handler}) |
| { |
| $diag_diag->{handler} ($ref, $res, $diag_diag, $key); |
| } |
| else |
| { |
| # This transition is flagged as hw-dependent if: |
| # - the name is in the list |
| # - the transition should not be reported (but be flagged as hw-dependent instead) |
| # - --no-hwdep was not provided |
| if ((grep { (index $key,$_)!=-1} @hwdep_list) |
| && ($diag_diag->{report_hwdep} eq "false") |
| && ($no_hwdep == 0)) { |
| push @{$res->{$HWDEP_CASES}}, $key; |
| } else { |
| push @{$res->{$diag_diag->{cat}}}, $key; |
| } |
| } |
| } |
| } |
| } |
| } |
| } |
| push @{$res->{$UNSTABLE_CASES}}, (sort (keys (%unstable_found))) if ($nounstable == 0); |
| |
| } |
| |
| ###################################################### |
| # PRINTING |
| sub print_tclist($@) |
| { |
| # The 1st parameter is used to find the name of the .exp file |
| # associated with the testcase. |
| my ($this_res, $cat, @tclist) = @_; |
| my $this_tool_exp = ""; |
| if (scalar(@tclist)) |
| { |
| print " - ".$cat.":\n\n"; |
| foreach my $case (@tclist) |
| { |
| if ($this_tool_exp ne $this_res->{testcases}->{$case}->{TOOL_EXP}) { |
| $this_tool_exp = $this_res->{testcases}->{$case}->{TOOL_EXP}; |
| print " Executed from: $this_tool_exp\n"; |
| } |
| print " $case\n"; |
| } |
| print "\n\n"; |
| } |
| } |
| |
| sub print_compare_results_summary($$) |
| { |
| my ($ref, $res) = @_; |
| my $return_value=0; |
| my $total_better = 0; |
| my $rtotal = 0; |
| my $quiet_reg = $quiet; |
| my $ref_ratio = 0; |
| my $ref_total = 0; |
| my $res_ratio = 0; |
| my $res_total = 0; |
| my $ignore_exec = 0; |
| my $ref_has_exec_tests = 0; |
| my $res_has_exec_tests = 0; |
| |
| # Compute the pass ratio as a sanity check |
| $ref_total = $ref->{PASS} + |
| $ref->{XFAIL} + |
| $ref->{XPASS} + |
| $ref->{FAIL} + |
| $ref->{UNRESOLVED} + |
| $ref->{UNSUPPORTED} + |
| $ref->{UNTESTED}; |
| |
| # It is possible that no test is executed at all (for instance if |
| # RUNTESTFLAGS was too restrictive). Avoid division by 0. |
| if ($ref_total > 0) { |
| $ref_ratio = ($ref->{PASS} + $ref->{XFAIL}) / |
| $ref_total; |
| } |
| |
| $res_total = $res->{PASS} + |
| $res->{XFAIL} + |
| $res->{XPASS} + |
| $res->{FAIL} + |
| $res->{UNRESOLVED} + |
| $res->{UNSUPPORTED} + |
| $res->{UNTESTED}; |
| |
| if ($res_total > 0) { |
| $res_ratio = ($res->{PASS} + $res->{XFAIL}) / |
| $res_total; |
| } |
| |
| if (not $quiet) |
| { |
| printf "Comparing:\n"; |
| printf "REFERENCE:$ref_file_name\n"; |
| printf "CURRENT: $res_file_name\n\n"; |
| } |
| |
| #### TESTS STATUS |
| if (not $quiet and not $short) |
| { |
| printf " ` +---------+---------+\n"; |
| printf "o RUN STATUS: | REF | RES |\n"; |
| printf " +------------------------------------------+---------+---------+\n"; |
| printf " | %-40s | %7d | %7d |\n", "Passes [PASS]", $ref->{PASS}, $res->{PASS}; |
| printf " | %-40s | %7d | %7d |\n", "Unexpected fails [FAIL]", $ref->{FAIL}, $res->{FAIL}; |
| printf " | %-40s | %7d | %7d |\n", "Errors [ERROR]", $ref->{ERROR}, $res->{ERROR}; |
| printf " | %-40s | %7d | %7d |\n", "Unexpected passes [XPASS]", $ref->{XPASS}, $res->{XPASS}; |
| printf " | %-40s | %7d | %7d |\n", "Expected fails [XFAIL]", $ref->{XFAIL}, $res->{XFAIL}; |
| printf " | %-40s | %7d | %7d |\n", "Unresolved [UNRESOLVED]", $ref->{UNRESOLVED}, $res->{UNRESOLVED}; |
| printf " | %-40s | %7d | %7d |\n", "Unsupported [UNSUPPORTED]", $ref->{UNSUPPORTED}, $res->{UNSUPPORTED}; |
| printf " | %-40s | %7d | %7d |\n", "Untested [UNTESTED]", $ref->{UNTESTED}, $res->{UNTESTED}; |
| printf " +------------------------------------------+---------+---------+\n"; |
| printf "\n"; |
| |
| printf " REF PASS ratio: %f\n", $ref_ratio; |
| printf " RES PASS ratio: %f\n", $res_ratio; |
| if ($ref_ratio < $ratio_thresh) |
| { |
| printf " ***** ERROR: REF PASS ratio is abnormally low *****\n"; |
| } |
| if ($res_ratio < $ratio_thresh) |
| { |
| printf " ***** ERROR: RES PASS ratio is abnormally low *****\n"; |
| } |
| |
| # If both PASS and FAIL EXEC tests are zero, assume there is no |
| # execution test for this tool, and do not warn. |
| $ref_has_exec_tests = ($ref->{EXEC}->{PASS} + $ref->{EXEC}->{FAIL}) != 0; |
| if ($ref_has_exec_tests) |
| { |
| printf " ***** ERROR: No REF execution test PASSed. Check execution engine configuration. *****\n" if ($ref->{EXEC}->{PASS} == 0); |
| printf " ***** WARNING: No REF execution test FAILed. Check execution engine configuration. *****\n" if ($ref->{EXEC}->{FAIL} == 0); |
| } |
| |
| $res_has_exec_tests = ($res->{EXEC}->{PASS} + $res->{EXEC}->{FAIL}) != 0; |
| if ($res_has_exec_tests) |
| { |
| printf " ***** ERROR: No RES execution test PASSed. Check execution engine configuration. *****\n" if ($res->{EXEC}->{PASS} == 0); |
| printf " ***** WARNING: No RES execution test FAILed. Check execution engine configuration. *****\n" if ($res->{EXEC}->{FAIL} == 0); |
| } |
| |
| # Ignore number of execution tests when computing the return |
| # value, if both REF and RES have no execution test. |
| $ignore_exec = !$ref_has_exec_tests && !$res_has_exec_tests; |
| } |
| |
| #### REGRESSIONS ? |
| $rtotal = scalar(@{$res->{$PASS_XPASS}}) |
| +scalar(@{$res->{$PASS_FAIL}}) |
| +scalar(@{$res->{$PASS_UNRESOLVED}}) |
| +scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}) |
| +scalar(@{$res->{$ERROR_APPEARS}}) |
| +scalar(@{$res->{$XPASS_FAIL}}) |
| +scalar(@{$res->{$XPASS_APPEARS}}) |
| +scalar(@{$res->{$XPASS_UNRESOLVED}}) |
| +scalar(@{$res->{$FAIL_APPEARS}}) |
| +scalar(@{$res->{$FAIL_UNRESOLVED}}) |
| +scalar(@{$res->{$XFAIL_FAIL}}) |
| +scalar(@{$res->{$XFAIL_UNRESOLVED}}) |
| +scalar(@{$res->{$UNSUPPORTED_FAIL}}) |
| +scalar(@{$res->{$UNSUPPORTED_XPASS}}) |
| +scalar(@{$res->{$UNSUPPORTED_UNRESOLVED}}) |
| +scalar(@{$res->{$UNTESTED_FAIL}}) |
| +scalar(@{$res->{$UNTESTED_XPASS}}) |
| +scalar(@{$res->{$UNTESTED_UNRESOLVED}}) |
| +scalar(@{$res->{$UNRESOLVED_FAIL}}) |
| +scalar(@{$res->{$UNRESOLVED_XPASS}}) |
| +scalar(@{$res->{$UNRESOLVED_APPEARS}}); |
| |
| $quiet_reg=1 if ($short and not $rtotal); |
| |
| if (not $quiet_reg) |
| { |
| printf "\n$col_red"."o REGRESSIONS:\n"; |
| printf " +------------------------------------------+---------+\n"; |
| printf " | %-40s | %7d |\n", $PASS_XPASS, scalar(@{$res->{$PASS_XPASS}}) if (scalar(@{$res->{$PASS_XPASS}})); |
| printf " | %-40s | %7d |\n", $PASS_FAIL, scalar(@{$res->{$PASS_FAIL}}) if (scalar(@{$res->{$PASS_FAIL}})); |
| printf " | %-40s | %7d |\n", $PASS_UNRESOLVED, scalar(@{$res->{$PASS_UNRESOLVED}}) if (scalar(@{$res->{$PASS_UNRESOLVED}})); |
| printf " | %-40s | %7d |\n", $PASSED_NOW_TIMEOUTS, scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}) if (scalar(@{$res->{$PASSED_NOW_TIMEOUTS}})); |
| printf " | %-40s | %7d |\n", $ERROR_APPEARS, scalar(@{$res->{$ERROR_APPEARS}}) if (scalar(@{$res->{$ERROR_APPEARS}})); |
| printf " | %-40s | %7d |\n", $XPASS_FAIL, scalar(@{$res->{$XPASS_FAIL}}) if (scalar(@{$res->{$XPASS_FAIL}})); |
| printf " | %-40s | %7d |\n", $XPASS_APPEARS, scalar(@{$res->{$XPASS_APPEARS}}) if (scalar(@{$res->{$XPASS_APPEARS}})); |
| printf " | %-40s | %7d |\n", $XPASS_UNRESOLVED, scalar(@{$res->{$XPASS_UNRESOLVED}}) if (scalar(@{$res->{$XPASS_UNRESOLVED}})); |
| printf " | %-40s | %7d |\n", $FAIL_APPEARS, scalar(@{$res->{$FAIL_APPEARS}}) if (scalar(@{$res->{$FAIL_APPEARS}})); |
| printf " | %-40s | %7d |\n", $FAIL_UNRESOLVED, scalar(@{$res->{$FAIL_UNRESOLVED}}) if (scalar(@{$res->{$FAIL_UNRESOLVED}})); |
| printf " | %-40s | %7d |\n", $XFAIL_FAIL, scalar(@{$res->{$XFAIL_FAIL}}) if (scalar(@{$res->{$XFAIL_FAIL}})); |
| printf " | %-40s | %7d |\n", $XFAIL_UNRESOLVED, scalar(@{$res->{$XFAIL_UNRESOLVED}}) if (scalar(@{$res->{$XFAIL_UNRESOLVED}})); |
| printf " | %-40s | %7d |\n", $UNSUPPORTED_FAIL, scalar(@{$res->{$UNSUPPORTED_FAIL}}) if (scalar(@{$res->{$UNSUPPORTED_FAIL}})); |
| printf " | %-40s | %7d |\n", $UNSUPPORTED_XPASS, scalar(@{$res->{$UNSUPPORTED_XPASS}}) if (scalar(@{$res->{$UNSUPPORTED_XPASS}})); |
| printf " | %-40s | %7d |\n", $UNSUPPORTED_UNRESOLVED, scalar(@{$res->{$UNSUPPORTED_UNRESOLVED}}) if (scalar(@{$res->{$UNSUPPORTED_UNRESOLVED}})); |
| printf " | %-40s | %7d |\n", $UNTESTED_FAIL, scalar(@{$res->{$UNTESTED_FAIL}}) if (scalar(@{$res->{$UNTESTED_FAIL}})); |
| printf " | %-40s | %7d |\n", $UNTESTED_XPASS, scalar(@{$res->{$UNTESTED_XPASS}}) if (scalar(@{$res->{$UNTESTED_XPASS}})); |
| printf " | %-40s | %7d |\n", $UNTESTED_UNRESOLVED, scalar(@{$res->{$UNTESTED_UNRESOLVED}}) if (scalar(@{$res->{$UNTESTED_UNRESOLVED}})); |
| printf " | %-40s | %7d |\n", $UNRESOLVED_FAIL, scalar(@{$res->{$UNRESOLVED_FAIL}}) if (scalar(@{$res->{$UNRESOLVED_FAIL}})); |
| printf " | %-40s | %7d |\n", $UNRESOLVED_XPASS, scalar(@{$res->{$UNRESOLVED_XPASS}}) if (scalar(@{$res->{$UNRESOLVED_XPASS}})); |
| printf " | %-40s | %7d |\n", $UNRESOLVED_APPEARS, scalar(@{$res->{$UNRESOLVED_APPEARS}}) if (scalar(@{$res->{$UNRESOLVED_APPEARS}})); |
| printf " +------------------------------------------+---------+\n"; |
| printf " | %-40s | %7d |\n", "TOTAL_REGRESSIONS", $rtotal; |
| printf " +------------------------------------------+---------+\n"; |
| printf "\n"; |
| |
| if ($long) |
| { |
| print_tclist($res, $PASS_XPASS, @{$res->{$PASS_XPASS}}); |
| print_tclist($res, $PASS_FAIL, @{$res->{$PASS_FAIL}}); |
| print_tclist($res, $PASS_UNRESOLVED, @{$res->{$PASS_UNRESOLVED}}); |
| print_tclist($res, $PASSED_NOW_TIMEOUTS, @{$res->{$PASSED_NOW_TIMEOUTS}}); |
| print_tclist($res, $ERROR_APPEARS, @{$res->{$ERROR_APPEARS}}); |
| print_tclist($res, $XPASS_FAIL, @{$res->{$XPASS_FAIL}}); |
| print_tclist($res, $XPASS_APPEARS, @{$res->{$XPASS_APPEARS}}); |
| print_tclist($res, $XPASS_UNRESOLVED, @{$res->{$XPASS_UNRESOLVED}}); |
| print_tclist($res, $FAIL_APPEARS, @{$res->{$FAIL_APPEARS}}); |
| print_tclist($res, $FAIL_UNRESOLVED, @{$res->{$FAIL_UNRESOLVED}}); |
| print_tclist($res, $XFAIL_FAIL, @{$res->{$XFAIL_FAIL}}); |
| print_tclist($res, $XFAIL_UNRESOLVED, @{$res->{$XFAIL_UNRESOLVED}}); |
| print_tclist($res, $UNSUPPORTED_FAIL, @{$res->{$UNSUPPORTED_FAIL}}); |
| print_tclist($res, $UNSUPPORTED_XPASS, @{$res->{$UNSUPPORTED_XPASS}}); |
| print_tclist($res, $UNSUPPORTED_UNRESOLVED, @{$res->{$UNSUPPORTED_UNRESOLVED}}); |
| print_tclist($res, $UNTESTED_FAIL, @{$res->{$UNTESTED_FAIL}}); |
| print_tclist($res, $UNTESTED_XPASS, @{$res->{$UNTESTED_XPASS}}); |
| print_tclist($res, $UNTESTED_UNRESOLVED, @{$res->{$UNTESTED_UNRESOLVED}}); |
| print_tclist($res, $UNRESOLVED_FAIL, @{$res->{$UNRESOLVED_FAIL}}); |
| print_tclist($res, $UNRESOLVED_XPASS, @{$res->{$UNRESOLVED_XPASS}}); |
| print_tclist($res, $UNRESOLVED_APPEARS, @{$res->{$UNRESOLVED_APPEARS}}); |
| } |
| printf "$col_reset\n"; |
| } |
| |
| #### IMPROVEMENTS TO BE CHECKED ? |
| if (not $quiet and not $short) |
| { |
| $total_better = |
| scalar(@{$res->{$PASS_DISAPPEARS}})+ |
| scalar(@{$res->{$PASS_XFAIL}})+ |
| scalar(@{$res->{$PASS_APPEARS}})+ |
| scalar(@{$res->{$PASS_UNSUPPORTED}})+ |
| scalar(@{$res->{$PASS_UNTESTED}})+ |
| scalar(@{$res->{$XPASS_PASS}})+ |
| scalar(@{$res->{$XPASS_XFAIL}})+ |
| scalar(@{$res->{$XPASS_UNSUPPORTED}})+ |
| scalar(@{$res->{$XPASS_UNTESTED}})+ |
| scalar(@{$res->{$XPASS_DISAPPEARS}})+ |
| scalar(@{$res->{$FAIL_PASS}})+ |
| scalar(@{$res->{$FAIL_XPASS}})+ |
| scalar(@{$res->{$FAIL_XFAIL}})+ |
| scalar(@{$res->{$FAIL_UNSUPPORTED}})+ |
| scalar(@{$res->{$FAIL_UNTESTED}})+ |
| scalar(@{$res->{$FAIL_DISAPPEARS}})+ |
| scalar(@{$res->{$XFAIL_DISAPPEARS}})+ |
| scalar(@{$res->{$XFAIL_PASS}})+ |
| scalar(@{$res->{$XFAIL_XPASS}})+ |
| scalar(@{$res->{$XFAIL_UNSUPPORTED}})+ |
| scalar(@{$res->{$XFAIL_UNTESTED}})+ |
| scalar(@{$res->{$XFAIL_APPEARS}})+ |
| scalar(@{$res->{$UNSUPPORTED_PASS}})+ |
| scalar(@{$res->{$UNSUPPORTED_XFAIL}})+ |
| scalar(@{$res->{$UNSUPPORTED_UNTESTED}})+ |
| scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}})+ |
| scalar(@{$res->{$UNSUPPORTED_APPEARS}})+ |
| scalar(@{$res->{$UNTESTED_PASS}})+ |
| scalar(@{$res->{$UNTESTED_XFAIL}})+ |
| scalar(@{$res->{$UNTESTED_UNSUPPORTED}})+ |
| scalar(@{$res->{$UNTESTED_DISAPPEARS}})+ |
| scalar(@{$res->{$UNTESTED_APPEARS}})+ |
| scalar(@{$res->{$UNRESOLVED_PASS}})+ |
| scalar(@{$res->{$UNRESOLVED_XFAIL}})+ |
| scalar(@{$res->{$UNRESOLVED_UNSUPPORTED}})+ |
| scalar(@{$res->{$UNRESOLVED_UNTESTED}})+ |
| scalar(@{$res->{$UNRESOLVED_DISAPPEARS}})+ |
| scalar(@{$res->{$ERROR_DISAPPEARS}})+ |
| scalar(@{$res->{$UNHANDLED_CASES}}); |
| |
| printf "$col_pink"."o IMPROVEMENTS TO BE CHECKED:\n"; |
| printf " +------------------------------------------+---------+\n"; |
| printf " | %-40s | %7d |\n", $PASS_DISAPPEARS, scalar(@{$res->{$PASS_DISAPPEARS}}) if (scalar(@{$res->{$PASS_DISAPPEARS}})); |
| printf " | %-40s | %7d |\n", $PASS_XFAIL, scalar(@{$res->{$PASS_XFAIL}}) if (scalar(@{$res->{$PASS_XFAIL}})); |
| printf " | %-40s | %7d |\n", $PASS_APPEARS, scalar(@{$res->{$PASS_APPEARS}}) if (scalar(@{$res->{$PASS_APPEARS}})); |
| printf " | %-40s | %7d |\n", $PASS_UNSUPPORTED, scalar(@{$res->{$PASS_UNSUPPORTED}}) if (scalar(@{$res->{$PASS_UNSUPPORTED}})); |
| printf " | %-40s | %7d |\n", $PASS_UNTESTED, scalar(@{$res->{$PASS_UNTESTED}}) if (scalar(@{$res->{$PASS_UNTESTED}})); |
| printf " | %-40s | %7d |\n", $XPASS_PASS, scalar(@{$res->{$XPASS_PASS}}) if (scalar(@{$res->{$XPASS_PASS}})); |
| printf " | %-40s | %7d |\n", $XPASS_XFAIL, scalar(@{$res->{$XPASS_XFAIL}}) if (scalar(@{$res->{$XPASS_XFAIL}})); |
| printf " | %-40s | %7d |\n", $XPASS_UNSUPPORTED, scalar(@{$res->{$XPASS_UNSUPPORTED}}) if (scalar(@{$res->{$XPASS_UNSUPPORTED}})); |
| printf " | %-40s | %7d |\n", $XPASS_UNTESTED, scalar(@{$res->{$XPASS_UNTESTED}}) if (scalar(@{$res->{$XPASS_UNTESTED}})); |
| printf " | %-40s | %7d |\n", $XPASS_DISAPPEARS, scalar(@{$res->{$XPASS_DISAPPEARS}}) if (scalar(@{$res->{$XPASS_DISAPPEARS}})); |
| printf " | %-40s | %7d |\n", $FAIL_PASS, scalar(@{$res->{$FAIL_PASS}}) if (scalar(@{$res->{$FAIL_PASS}})); |
| printf " | %-40s | %7d |\n", $FAIL_XPASS, scalar(@{$res->{$FAIL_XPASS}}) if (scalar(@{$res->{$FAIL_XPASS}})); |
| printf " | %-40s | %7d |\n", $FAIL_XFAIL, scalar(@{$res->{$FAIL_XFAIL}}) if (scalar(@{$res->{$FAIL_XFAIL}})); |
| printf " | %-40s | %7d |\n", $FAIL_UNSUPPORTED, scalar(@{$res->{$FAIL_UNSUPPORTED}}) if (scalar(@{$res->{$FAIL_UNSUPPORTED}})); |
| printf " | %-40s | %7d |\n", $FAIL_UNTESTED, scalar(@{$res->{$FAIL_UNTESTED}}) if (scalar(@{$res->{$FAIL_UNTESTED}})); |
| printf " | %-40s | %7d |\n", $FAIL_DISAPPEARS, scalar(@{$res->{$FAIL_DISAPPEARS}}) if (scalar(@{$res->{$FAIL_DISAPPEARS}})); |
| printf " | %-40s | %7d |\n", $XFAIL_DISAPPEARS, scalar(@{$res->{$XFAIL_DISAPPEARS}}) if (scalar(@{$res->{$XFAIL_DISAPPEARS}})); |
| printf " | %-40s | %7d |\n", $XFAIL_PASS, scalar(@{$res->{$XFAIL_PASS}}) if (scalar(@{$res->{$XFAIL_PASS}})); |
| printf " | %-40s | %7d |\n", $XFAIL_XPASS, scalar(@{$res->{$XFAIL_XPASS}}) if (scalar(@{$res->{$XFAIL_XPASS}})); |
| printf " | %-40s | %7d |\n", $XFAIL_UNSUPPORTED, scalar(@{$res->{$XFAIL_UNSUPPORTED}}) if (scalar(@{$res->{$XFAIL_UNSUPPORTED}})); |
| printf " | %-40s | %7d |\n", $XFAIL_UNTESTED, scalar(@{$res->{$XFAIL_UNTESTED}}) if (scalar(@{$res->{$XFAIL_UNTESTED}})); |
| printf " | %-40s | %7d |\n", $XFAIL_APPEARS, scalar(@{$res->{$XFAIL_APPEARS}}) if (scalar(@{$res->{$XFAIL_APPEARS}})); |
| printf " | %-40s | %7d |\n", $UNSUPPORTED_PASS, scalar(@{$res->{$UNSUPPORTED_PASS}}) if (scalar(@{$res->{$UNSUPPORTED_PASS}})); |
| printf " | %-40s | %7d |\n", $UNSUPPORTED_XFAIL, scalar(@{$res->{$UNSUPPORTED_XFAIL}}) if (scalar(@{$res->{$UNSUPPORTED_XFAIL}})); |
| printf " | %-40s | %7d |\n", $UNSUPPORTED_UNTESTED, scalar(@{$res->{$UNSUPPORTED_UNTESTED}}) if (scalar(@{$res->{$UNSUPPORTED_UNTESTED}})); |
| printf " | %-40s | %7d |\n", $UNSUPPORTED_DISAPPEARS, scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}}) if (scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}})); |
| printf " | %-40s | %7d |\n", $UNSUPPORTED_APPEARS, scalar(@{$res->{$UNSUPPORTED_APPEARS}}) if (scalar(@{$res->{$UNSUPPORTED_APPEARS}})); |
| printf " | %-40s | %7d |\n", $UNTESTED_PASS, scalar(@{$res->{$UNTESTED_PASS}}) if (scalar(@{$res->{$UNTESTED_PASS}})); |
| printf " | %-40s | %7d |\n", $UNTESTED_XFAIL, scalar(@{$res->{$UNTESTED_XFAIL}}) if (scalar(@{$res->{$UNTESTED_XFAIL}})); |
| printf " | %-40s | %7d |\n", $UNTESTED_UNSUPPORTED, scalar(@{$res->{$UNTESTED_UNSUPPORTED}}) if (scalar(@{$res->{$UNTESTED_UNSUPPORTED}})); |
| printf " | %-40s | %7d |\n", $UNTESTED_DISAPPEARS, scalar(@{$res->{$UNTESTED_DISAPPEARS}}) if (scalar(@{$res->{$UNTESTED_DISAPPEARS}})); |
| printf " | %-40s | %7d |\n", $UNTESTED_APPEARS, scalar(@{$res->{$UNTESTED_APPEARS}}) if (scalar(@{$res->{$UNTESTED_APPEARS}})); |
| printf " | %-40s | %7d |\n", $UNRESOLVED_PASS, scalar(@{$res->{$UNRESOLVED_PASS}}) if (scalar(@{$res->{$UNRESOLVED_PASS}})); |
| printf " | %-40s | %7d |\n", $UNRESOLVED_XFAIL, scalar(@{$res->{$UNRESOLVED_XFAIL}}) if (scalar(@{$res->{$UNRESOLVED_XFAIL}})); |
| printf " | %-40s | %7d |\n", $UNRESOLVED_UNSUPPORTED, scalar(@{$res->{$UNRESOLVED_UNSUPPORTED}}) if (scalar(@{$res->{$UNRESOLVED_UNSUPPORTED}})); |
| printf " | %-40s | %7d |\n", $UNRESOLVED_UNTESTED, scalar(@{$res->{$UNRESOLVED_UNTESTED}}) if (scalar(@{$res->{$UNRESOLVED_UNTESTED}})); |
| printf " | %-40s | %7d |\n", $UNRESOLVED_DISAPPEARS, scalar(@{$res->{$UNRESOLVED_DISAPPEARS}}) if (scalar(@{$res->{$UNRESOLVED_DISAPPEARS}})); |
| printf " | %-40s | %7d |\n", $ERROR_DISAPPEARS, scalar(@{$res->{$ERROR_DISAPPEARS}}) if (scalar(@{$res->{$ERROR_DISAPPEARS}})); |
| printf " | %-40s | %7d |\n", $UNHANDLED_CASES, scalar(@{$res->{$UNHANDLED_CASES}}) if (scalar(@{$res->{$UNHANDLED_CASES}})); |
| printf " | %-40s | %7d |\n", $UNSTABLE_CASES, scalar(@{$res->{$UNSTABLE_CASES}}) if (scalar(@{$res->{$UNSTABLE_CASES}})); |
| printf " | %-40s | %7d |\n", $HWDEP_CASES, scalar(@{$res->{$HWDEP_CASES}}) if (scalar(@{$res->{$HWDEP_CASES}})); |
| printf " +------------------------------------------+---------+\n"; |
| printf " | %-40s | %7d |\n", "TOTAL_IMPROVEMENTS_TO_BE_CHECKED", $total_better + scalar(@{$res->{$UNSTABLE_CASES}});; |
| printf " +------------------------------------------+---------+\n"; |
| printf "\n"; |
| |
| if ($long) |
| { |
| print_tclist($res, $PASS_DISAPPEARS, @{$res->{$PASS_DISAPPEARS}}); |
| print_tclist($res, $PASS_XFAIL, @{$res->{$PASS_XFAIL}}); |
| print_tclist($res, $PASS_APPEARS, @{$res->{$PASS_APPEARS}}); |
| print_tclist($res, $PASS_UNSUPPORTED, @{$res->{$PASS_UNSUPPORTED}}); |
| print_tclist($res, $PASS_UNTESTED, @{$res->{$PASS_UNTESTED}}); |
| print_tclist($res, $XPASS_PASS, @{$res->{$XPASS_PASS}}); |
| print_tclist($res, $XPASS_XFAIL, @{$res->{$XPASS_XFAIL}}); |
| print_tclist($res, $XPASS_UNSUPPORTED, @{$res->{$XPASS_UNSUPPORTED}}); |
| print_tclist($res, $XPASS_UNTESTED, @{$res->{$XPASS_UNTESTED}}); |
| print_tclist($res, $XPASS_DISAPPEARS, @{$res->{$XPASS_DISAPPEARS}}); |
| print_tclist($res, $FAIL_PASS, @{$res->{$FAIL_PASS}}); |
| print_tclist($res, $FAIL_XPASS, @{$res->{$FAIL_XPASS}}); |
| print_tclist($res, $FAIL_XFAIL, @{$res->{$FAIL_XFAIL}}); |
| print_tclist($res, $FAIL_UNSUPPORTED, @{$res->{$FAIL_UNSUPPORTED}}); |
| print_tclist($res, $FAIL_UNTESTED, @{$res->{$FAIL_UNTESTED}}); |
| print_tclist($res, $FAIL_DISAPPEARS, @{$res->{$FAIL_DISAPPEARS}}); |
| print_tclist($res, $XFAIL_DISAPPEARS, @{$res->{$XFAIL_DISAPPEARS}}); |
| print_tclist($res, $XFAIL_PASS, @{$res->{$XFAIL_PASS}}); |
| print_tclist($res, $XFAIL_XPASS, @{$res->{$XFAIL_XPASS}}); |
| print_tclist($res, $XFAIL_UNSUPPORTED, @{$res->{$XFAIL_UNSUPPORTED}}); |
| print_tclist($res, $XFAIL_UNTESTED, @{$res->{$XFAIL_UNTESTED}}); |
| print_tclist($res, $XFAIL_APPEARS, @{$res->{$XFAIL_APPEARS}}); |
| print_tclist($res, $UNSUPPORTED_PASS, @{$res->{$UNSUPPORTED_PASS}}); |
| print_tclist($res, $UNSUPPORTED_XFAIL, @{$res->{$UNSUPPORTED_XFAIL}}); |
| print_tclist($res, $UNSUPPORTED_UNTESTED, @{$res->{$UNSUPPORTED_UNTESTED}}); |
| print_tclist($res, $UNSUPPORTED_DISAPPEARS, @{$res->{$UNSUPPORTED_DISAPPEARS}}); |
| print_tclist($res, $UNSUPPORTED_APPEARS, @{$res->{$UNSUPPORTED_APPEARS}}); |
| print_tclist($res, $UNTESTED_PASS, @{$res->{$UNTESTED_PASS}}); |
| print_tclist($res, $UNTESTED_XFAIL, @{$res->{$UNTESTED_XFAIL}}); |
| print_tclist($res, $UNTESTED_UNSUPPORTED, @{$res->{$UNTESTED_UNSUPPORTED}}); |
| print_tclist($res, $UNTESTED_DISAPPEARS, @{$res->{$UNTESTED_DISAPPEARS}}); |
| print_tclist($res, $UNTESTED_APPEARS, @{$res->{$UNTESTED_APPEARS}}); |
| print_tclist($res, $UNRESOLVED_PASS, @{$res->{$UNRESOLVED_PASS}}); |
| print_tclist($res, $UNRESOLVED_XFAIL, @{$res->{$UNRESOLVED_XFAIL}}); |
| print_tclist($res, $UNRESOLVED_UNSUPPORTED, @{$res->{$UNRESOLVED_UNSUPPORTED}}); |
| print_tclist($res, $UNRESOLVED_UNTESTED, @{$res->{$UNRESOLVED_UNTESTED}}); |
| print_tclist($res, $UNRESOLVED_DISAPPEARS, @{$res->{$UNRESOLVED_DISAPPEARS}}); |
| print_tclist($res, $ERROR_DISAPPEARS, @{$res->{$ERROR_DISAPPEARS}}); |
| print_tclist($res, $UNHANDLED_CASES, @{$res->{$UNHANDLED_CASES}}); |
| print_tclist($res, $UNSTABLE_CASES, @{$res->{$UNSTABLE_CASES}}); |
| print_tclist($res, $HWDEP_CASES, @{$res->{$HWDEP_CASES}}); |
| } |
| printf "$col_reset\n"; |
| } |
| |
| $return_value = 1 if ($total_better); |
| |
| $return_value = 2 if ($rtotal); |
| |
| # Error if there was no PASS (eg when sth went wrong and no .sum was generated) |
| $return_value = 2 if (($res->{PASS} + $res->{XFAIL}) == 0); |
| |
| # Error if every test passed, or the ratio was too low. This |
| # generally means the simulator is not configured well. |
| $return_value = 2 if (($ref_ratio == 100) || ($ref_ratio < $ratio_thresh)); |
| $return_value = 2 if (($res_ratio == 100) || ($res_ratio < $ratio_thresh)); |
| |
| # Error if no execution test passed. |
| # It is possible that no execution test fails, though. |
| $return_value = 2 if (($ignore_exec == 0) |
| && (($ref->{EXEC}->{PASS} == 0) |
| || ($res->{EXEC}->{PASS} == 0))); |
| |
| return $return_value; |
| } |