compare_dg_tests.pl

#!/usr/bin/env perl
use strict;
use warnings;

use File::Glob;
use Getopt::Long;
use Term::ANSIColor qw(:constants);
use File::Basename;
use Cwd;

my $app = $0;

sub read_sum($);
sub read_unstable($);
sub dump_result($);
sub compare_results($$);
sub usage();
sub print_compare_results_summary($$);
sub nothing($$$$);

my $PASS_PASS             = "Still passes              [PASS => PASS]";
my $PASS_XPASS            = "PASS now XPASS            [PASS =>XPASS]";
my $PASS_FAIL             = "PASS now FAIL             [PASS => FAIL]";
my $PASS_XFAIL            = "PASS now XFAIL            [PASS =>XFAIL]";
my $PASS_UNSUPPORTED      = "PASS now UNSUPPORTED      [PASS =>UNSUP]";
my $PASS_UNRESOLVED       = "PASS now UNRESOLVED       [PASS =>UNRES]";
my $PASS_UNTESTED         = "PASS now UNTESTED         [PASS =>UNTES]";
my $PASS_DISAPPEARS       = "PASS disappears           [PASS =>     ]";
my $PASS_APPEARS	  = "New PASS                  [     => PASS]";
my $PASSED_NOW_TIMEOUTS   = "Timeout                   [PASS =>T.OUT]";

my $XPASS_PASS            = "XPASS now PASS            [XPASS=> PASS]";
my $XPASS_XPASS           = "Still xpass               [XPASS=>XPASS]";
my $XPASS_FAIL            = "XPASS now FAIL            [XPASS=> FAIL]";
my $XPASS_XFAIL           = "XPASS now XFAIL           [XPASS=>XFAIL]";
my $XPASS_UNSUPPORTED     = "XPASS now UNSUPPORTED     [XPASS=>UNSUP]";
my $XPASS_UNRESOLVED      = "XPASS now UNRESOLVED      [XPASS=>UNRES]";
my $XPASS_UNTESTED        = "XPASS now UNTESTED        [XPASS=>UNTES]";
my $XPASS_DISAPPEARS      = "XPASS disappears          [XPASS=>     ]";
my $XPASS_APPEARS         = "XPASS appears             [     =>XPASS]";

my $FAIL_PASS             = "FAIL now PASS             [FAIL => PASS]";
my $FAIL_XPASS            = "FAIL now XPASS            [FAIL =>XPASS]";
my $FAIL_FAIL             = "Still fails               [FAIL => FAIL]";
my $FAIL_XFAIL            = "FAIL now XFAIL            [FAIL =>XFAIL]";
my $FAIL_UNSUPPORTED      = "FAIL now UNSUPPORTED      [FAIL =>UNSUP]";
my $FAIL_UNRESOLVED       = "FAIL now UNRESOLVED       [FAIL =>UNRES]";
my $FAIL_UNTESTED         = "FAIL now UNTESTED         [FAIL =>UNTES]";
my $FAIL_DISAPPEARS	  = "FAIL disappears           [FAIL =>     ]";
my $FAIL_APPEARS          = "FAIL appears              [     => FAIL]";

my $XFAIL_PASS            = "XFAIL now PASS            [XFAIL=> PASS]";
my $XFAIL_XPASS           = "XFAIL now XPASS           [XFAIL=>XPASS]";
my $XFAIL_FAIL            = "XFAIL now FAIL            [XFAIL=> FAIL]";
my $XFAIL_XFAIL           = "Still xfail               [XFAIL=>XFAIL]";
my $XFAIL_UNSUPPORTED     = "XFAIL now UNSUPPORTED     [XFAIL=>UNSUP]";
my $XFAIL_UNRESOLVED      = "XFAIL now UNRESOLVED      [XFAIL=>UNRES]";
my $XFAIL_UNTESTED        = "XFAIL now UNTESTED        [XFAIL=>UNTES]";
my $XFAIL_DISAPPEARS	  = "XFAIL disappears          [XFAIL=>     ]";
my $XFAIL_APPEARS         = "XFAIL appears             [     =>XFAIL]";

my $UNSUPPORTED_PASS      = "UNSUPPORTED now PASS      [UNSUP=> PASS]";
my $UNSUPPORTED_XPASS     = "UNSUPPORTED now XPASS     [UNSUP=>XPASS]";
my $UNSUPPORTED_FAIL      = "UNSUPPORTED now FAIL      [UNSUP=> FAIL]";
my $UNSUPPORTED_XFAIL     = "UNSUPPORTED now XFAIL     [UNSUP=>XFAIL]";
my $UNSUPPORTED_UNSUPPORTED= "UNSUPP now UNSUPP        [UNSUP=>UNSUP]";
my $UNSUPPORTED_UNRESOLVED= "UNSUPP now UNRESOLVED     [UNSUP=>UNRES]";
my $UNSUPPORTED_UNTESTED  = "UNSUPP now UNTESTED       [UNSUP=>UNTES]";
my $UNSUPPORTED_DISAPPEARS= "UNSUPPORTED disappears    [UNSUP=>     ]";
my $UNSUPPORTED_APPEARS   = "UNSUPPORTED appears       [     =>UNSUP]";

my $UNTESTED_PASS         = "UNTESTED now PASS         [UNTES=> PASS]";
my $UNTESTED_XPASS        = "UNTESTED now XPASS        [UNTES=>XPASS]";
my $UNTESTED_FAIL         = "UNTESTED now FAIL         [UNTES=> FAIL]";
my $UNTESTED_XFAIL        = "UNTESTED now XFAIL        [UNTES=>XFAIL]";
my $UNTESTED_UNSUPPORTED  = "UNTESTED now UNSUPP       [UNTES=>UNSUP]";
my $UNTESTED_UNRESOLVED   = "UNTESTED now UNRESOLVED   [UNTES=>UNRES]";
my $UNTESTED_UNTESTED     = "UNTESTED now UNTESTED     [UNTES=>UNTES]";
my $UNTESTED_DISAPPEARS   = "UNTESTED disappears       [UNTES=>     ]";
my $UNTESTED_APPEARS      = "UNTESTED appears          [     =>UNTES]";

my $UNRESOLVED_PASS       = "UNRESOLVED now PASS       [UNRES=> PASS]";
my $UNRESOLVED_XPASS      = "UNRESOLVED now XPASS      [UNRES=>XPASS]";
my $UNRESOLVED_FAIL       = "UNRESOLVED now FAIL       [UNRES=> FAIL]";
my $UNRESOLVED_XFAIL      = "UNRESOLVED now XFAIL      [UNRES=>XFAIL]";
my $UNRESOLVED_UNSUPPORTED= "UNRESOLVED now UNSUPP     [UNRES=>UNSUP]";
my $UNRESOLVED_UNRESOLVED = "UNRESOLVED now UNRESOLVED [UNRES=>UNRES]";
my $UNRESOLVED_UNTESTED   = "UNRESOLVED now UNTESTED   [UNRES=>UNTES]";
my $UNRESOLVED_DISAPPEARS = "UNRESOLVED disappears     [UNRES=>     ]";
my $UNRESOLVED_APPEARS    = "UNRESOLVED appears        [     =>UNRES]";

my $ERROR_DISAPPEARS      = "ERROR disappears          [ERROR=>     ]";
my $ERROR_APPEARS         = "ERROR appears             [     =>ERROR]";

my $UNHANDLED_CASES	  = "Unhandled cases           [   ..??..   ]";
my $UNSTABLE_CASES	  = "Unstable cases, ignored   [~RANDOM     ]";

my @handler_list = (
  {was=>"PASS",     is=>"PASS",     cat=>$PASS_PASS},
  {was=>"PASS",     is=>"XPASS",    cat=>$PASS_XPASS},
  {was=>"PASS",     is=>"FAIL",     handler=>\&handle_pass_fail},
  {was=>"PASS",     is=>"XFAIL",    cat=>$PASS_XFAIL},
  {was=>"PASS",     is=>"UNSUPPORTED",cat=>$PASS_UNSUPPORTED},
  {was=>"PASS",     is=>"UNTESTED", cat=>$PASS_UNTESTED},
  {was=>"PASS",     is=>"UNRESOLVED",cat=>$PASS_UNRESOLVED},
  {was=>"PASS",     is=>"NO_EXIST", cat=>$PASS_DISAPPEARS},
  {was=>"NO_EXIST", is=>"PASS",     cat=>$PASS_APPEARS},

  {was=>"XPASS",    is=>"PASS",     cat=>$XPASS_PASS},
  {was=>"XPASS",    is=>"XPASS",    cat=>$XPASS_XPASS},
  {was=>"XPASS",    is=>"FAIL",     cat=>$XPASS_FAIL},
  {was=>"XPASS",    is=>"XFAIL",    cat=>$XPASS_XFAIL},
  {was=>"XPASS",    is=>"UNSUPPORTED",cat=>$XPASS_UNSUPPORTED},
  {was=>"XPASS",    is=>"UNTESTED", cat=>$XPASS_UNTESTED},
  {was=>"XPASS",    is=>"UNRESOLVED",cat=>$XPASS_UNRESOLVED},
  {was=>"XPASS",    is=>"NO_EXIST", cat=>$XPASS_DISAPPEARS},
  {was=>"NO_EXIST", is=>"XPASS",    cat=>$XPASS_APPEARS},

  {was=>"FAIL",     is=>"PASS",     cat=>$FAIL_PASS},
  {was=>"FAIL",     is=>"XPASS",    cat=>$FAIL_XPASS},
  {was=>"FAIL",     is=>"FAIL",     cat=>$FAIL_FAIL},
  {was=>"FAIL",     is=>"XFAIL",    cat=>$FAIL_XFAIL},
  {was=>"FAIL",     is=>"UNSUPPORTED",cat=>$FAIL_UNSUPPORTED},
  {was=>"FAIL",     is=>"UNTESTED", cat=>$FAIL_UNTESTED},
  {was=>"FAIL",     is=>"UNRESOLVED",cat=>$FAIL_UNRESOLVED},
  {was=>"FAIL",     is=>"NO_EXIST", cat=>$FAIL_DISAPPEARS},
  {was=>"NO_EXIST", is=>"FAIL",     cat=>$FAIL_APPEARS},

  {was=>"XFAIL",    is=>"PASS",     cat=>$XFAIL_PASS},
  {was=>"XFAIL",    is=>"XPASS",    cat=>$XFAIL_XPASS},
  {was=>"XFAIL",    is=>"FAIL",     cat=>$XFAIL_FAIL},
  {was=>"XFAIL",    is=>"XFAIL",    cat=>$XFAIL_XFAIL},
  {was=>"XFAIL",    is=>"UNSUPPORTED",cat=>$XFAIL_UNSUPPORTED},
  {was=>"XFAIL",    is=>"UNTESTED", cat=>$XFAIL_UNTESTED},
  {was=>"XFAIL",    is=>"UNRESOLVED",cat=>$XFAIL_UNRESOLVED},
  {was=>"XFAIL",    is=>"NO_EXIST", cat=>$XFAIL_DISAPPEARS},
  {was=>"NO_EXIST", is=>"XFAIL",    cat=>$XFAIL_APPEARS},

  {was=>"UNSUPPORTED", is=>"PASS",  cat=>$UNSUPPORTED_PASS},
  {was=>"UNSUPPORTED", is=>"XPASS", cat=>$UNSUPPORTED_XPASS},
  {was=>"UNSUPPORTED", is=>"FAIL",  cat=>$UNSUPPORTED_FAIL},
  {was=>"UNSUPPORTED", is=>"XFAIL", cat=>$UNSUPPORTED_XFAIL},
  {was=>"UNSUPPORTED", is=>"UNSUPPORTED",cat=>$UNSUPPORTED_UNSUPPORTED},
  {was=>"UNSUPPORTED", is=>"UNTESTED", cat=>$UNSUPPORTED_UNTESTED},
  {was=>"UNSUPPORTED", is=>"UNRESOLVED",cat=>$UNSUPPORTED_UNRESOLVED},
  {was=>"UNSUPPORTED", is=>"NO_EXIST", cat=>$UNSUPPORTED_DISAPPEARS},
  {was=>"NO_EXIST", is=>"UNSUPPORTED", cat=>$UNSUPPORTED_APPEARS},

  {was=>"UNTESTED", is=>"PASS", cat=>$UNTESTED_PASS},
  {was=>"UNTESTED", is=>"XPASS", cat=>$UNTESTED_XPASS},
  {was=>"UNTESTED", is=>"FAIL", cat=>$UNTESTED_FAIL},
  {was=>"UNTESTED", is=>"XFAIL", cat=>$UNTESTED_XFAIL},
  {was=>"UNTESTED", is=>"UNSUPPORTED",cat=>$UNTESTED_UNSUPPORTED},
  {was=>"UNTESTED", is=>"UNTESTED", cat=>$UNTESTED_UNTESTED},
  {was=>"UNTESTED", is=>"UNRESOLVED",cat=>$UNTESTED_UNRESOLVED},
  {was=>"UNTESTED", is=>"NO_EXIST", cat=>$UNTESTED_DISAPPEARS},
  {was=>"NO_EXIST", is=>"UNTESTED",    cat=>$UNTESTED_APPEARS},

  {was=>"UNRESOLVED", is=>"PASS", cat=>$UNRESOLVED_PASS},
  {was=>"UNRESOLVED", is=>"XPASS", cat=>$UNRESOLVED_XPASS},
  {was=>"UNRESOLVED", is=>"FAIL", cat=>$UNRESOLVED_FAIL},
  {was=>"UNRESOLVED", is=>"XFAIL", cat=>$UNRESOLVED_XFAIL},
  {was=>"UNRESOLVED", is=>"UNSUPPORTED",cat=>$UNRESOLVED_UNSUPPORTED},
  {was=>"UNRESOLVED", is=>"UNTESTED", cat=>$UNRESOLVED_UNTESTED},
  {was=>"UNRESOLVED", is=>"UNRESOLVED",cat=>$UNRESOLVED_UNRESOLVED},
  {was=>"UNRESOLVED", is=>"NO_EXIST", cat=>$UNRESOLVED_DISAPPEARS},
  {was=>"NO_EXIST", is=>"UNRESOLVED",  cat=>$UNRESOLVED_APPEARS},

  {was=>"ERROR", is=>"NO_EXIST", cat=>$ERROR_DISAPPEARS},
  {was=>"NO_EXIST", is=>"ERROR",  cat=>$ERROR_APPEARS},

#  {was=>"NO_EXIST", is=>"NO_EXIST", handler=>\&handle_not_yet_supported}
);

######################################################
# TREAT ARGUMENTS

my $verbose=0;
my $quiet=0;
my $long=0;
my $short=0;
my $debug=0;
my ($testroot, $basename);
my ($ref_file_name, $res_file_name);
my $nounstable=0;
my $unstablefile=0;
my @unstable_markers=();
my $ratio_thresh = 0.95; # Warn if pass ratio is below this threshold

GetOptions ("l"           => \$long,
            "s"           => \$short,
            "q"           => \$quiet,
            "v"           => \$verbose,
            "dbg"         => \$debug,
            "testroot=s"  => \$testroot,
            "basename=s"  => \$basename,
            "pass-thresh=f" => \$ratio_thresh,
            "no-unstable"  => \$nounstable,
            "unstable-tests=s" => \$unstablefile,
            "unstable-marker=s"    => \@unstable_markers);

$ref_file_name = $ARGV[0] if ($#ARGV == 1);
$res_file_name = $ARGV[1] if ($#ARGV == 1);

$ref_file_name = $testroot."/expected_results/".$basename if ($testroot and $basename);
$res_file_name = $testroot."/testing/run/".$basename if ($testroot and $basename);
&usage if (not $ref_file_name or not $res_file_name);

my ($col_boldred, $col_red, $col_boldgreen, $col_green, $col_boldpink, $col_pink, $col_reset)
    = ("\033[31;1m","\033[31;3m","\033[32;1m","\033[32;3m","\033[35;1m","\033[35;2m","\033[0m");
($col_boldred, $col_red, $col_boldgreen, $col_green, $col_boldpink, $col_pink, $col_reset)
    = ("","","","","","","") if (not I_am_interactive());

######################################################
# MAIN PROGRAM
# print "comparing $ref_file_name $res_file_name\n";

# If none of the 2 .sum exists, nothing to compare: exit early.
exit 0 if ( (! -e $ref_file_name) && (! -e $res_file_name ));

my $ref = read_sum($ref_file_name) ;
my $res = read_sum($res_file_name) ;
my @unstablelist = ();

@unstablelist = read_unstable($unstablefile) if ($unstablefile ne 0);

compare_results($ref, $res);

my $final_result = print_compare_results_summary($ref, $res);

exit $final_result;

######################################################
# UTILITIES

sub empty_result()
{
   my %empty_result;# = {PASS=>0, FAIL=>0, XPASS=>0, XFAIL=>0, UNSUPPORTED=>0, UNTESTED=>0, UNRESOLVED=>0, ERROR=>0};
   $empty_result{PASS}=$empty_result{FAIL}=$empty_result{XPASS}=$empty_result{XFAIL}=0;
   $empty_result{UNSUPPORTED}=$empty_result{UNTESTED}=$empty_result{UNRESOLVED}=$empty_result{NO_EXIST}=$empty_result{ERROR}=0;
   return \%empty_result;
}
sub I_am_interactive {
  return -t STDIN && -t STDOUT;
}
sub usage()
{
	print "Usage : $app <ref_file.sum> <result_file.sum>\n";
	exit 1;
}


######################################################
# PARSING
sub read_sum($)
{
   my ($sum_file) = @_;
   my $res = empty_result();
   my %testcases;
   my %unsupported;
   $res->{testcases} = \%testcases;
   my $pending_timeout=0;
   my $current_exp="";

   $res->{EXEC}->{PASS} = 0;
   $res->{EXEC}->{FAIL} = 0;

   open SUMFILE, $sum_file or die $!;
   while (<SUMFILE>)
   {
	  if (m/^(PASS|XPASS|FAIL|XFAIL|UNSUPPORTED|UNTESTED|UNRESOLVED|ERROR): *(.*)/)
	  {
	  	my ($diag,$tc) = ($1,$2);
		my %tcresult;
		$tc =~ s/==[0-9]+== Shadow memory range interleaves with an existing memory mapping. ASan cannot proceed correctly./==<pid>== Shadow memory range interleaves with an existing memory mapping. ASan cannot proceed correctly./;
		$testcases{$tc} = empty_result() if (not exists $testcases{$tc});
		$testcases{$tc}->{$diag}++;
		$testcases{$tc}->{HAS_TIMED_OUT} = $pending_timeout;
		$testcases{$tc}->{EXP} = $current_exp;
		$pending_timeout = 0;
		$res->{$diag}++;
		# Count execution tests, for a sanity check
		if ($tc =~ /execution/)
		{
		    $res->{EXEC}->{$diag}++;
		}
	  }
	  elsif (m/WARNING: program timed out/) 
	  {
	  	$pending_timeout = 1;
	  }
	  elsif (m/^(# of expected passes|# of unexpected failures|# of expected failures|# of known failures|# of unsupported tests|# of untested testcases)\s+(.*)/)
	  {
	  	 $res->{"summary - "."$1"} = $2;
	  }
	  elsif (m/^\/.*\/([^\/]+)\s+version\s+(.*)/)
	  {
	  	 $res->{tool} = $1;
	  	 $res->{version} = $2;
		 $res->{version} =~ s/ [-(].*//;
	  }
	  elsif (m/^Running target .*/)
	  {
	  }
	  elsif (m/^Running (.*) \.\.\.*/)
	  {
	      $current_exp=$1;
	      $current_exp =~ s|.*/testsuite/||;
	  }
   }
   close SUMFILE;
   return $res;
}

# Parse list on unstable tests
sub read_unstable($)
{
   my ($unstable_file) = @_;
   my @unstable_tests = ();

   open UNSTABLEFILE, $unstable_file or die $!;
   while (<UNSTABLEFILE>)
   {
       # Skip lines starting with '#', or with spaces only
       if ((/^#/) || (/^[ 	]*$/))
       {
       }
       else
       {
	   chomp;

	   my $test = $_;

	   # Check if line is of type: target:testname
	   if (/^(.*?):/)
	   {
	       foreach my $unstable_marker (@unstable_markers)
	       {
		   if ($unstable_marker eq $1) {
		       # If target matches the one supplied as script
		       # argument, add the testname to the list
		       $test =~ s/.*?://;
		       push @unstable_tests, $test;
		   }
	       }
	   } else {
	       push @unstable_tests, $test;
	   }
       }
   }
   close UNSTABLEFILE;
   return @unstable_tests;
}

######################################################
# DIFFING
sub handle_pass_fail($$$$)
{
   my ($ref, $res, $diag_diag, $tc) = @_;
   if ($res->{testcases}->{$tc}->{HAS_TIMED_OUT})
   {
     push @{$res->{$PASSED_NOW_TIMEOUTS}}, $tc;
   }
   else
   {
     push @{$res->{$PASS_FAIL}}, $tc;
   }
}

sub compare_results($$)
{
   my ($ref, $res) = @_;

   @{$res->{$PASS_PASS}} = ();
   @{$res->{$PASS_XPASS}} = ();
   @{$res->{$PASS_FAIL}} = ();
   @{$res->{$PASS_XFAIL}} = ();
   @{$res->{$PASS_UNSUPPORTED}} = ();
   @{$res->{$PASS_UNTESTED}} = ();
   @{$res->{$PASS_UNRESOLVED}} = ();
   @{$res->{$PASS_DISAPPEARS}} = ();
   @{$res->{$PASS_APPEARS}} = ();
   @{$res->{$PASSED_NOW_TIMEOUTS}} = ();

   @{$res->{$XPASS_PASS}} = ();
   @{$res->{$XPASS_XPASS}} = ();
   @{$res->{$XPASS_FAIL}} = ();
   @{$res->{$XPASS_XFAIL}} = ();
   @{$res->{$XPASS_UNSUPPORTED}} = ();
   @{$res->{$XPASS_UNTESTED}} = ();
   @{$res->{$XPASS_UNRESOLVED}} = ();
   @{$res->{$XPASS_DISAPPEARS}} = ();
   @{$res->{$XPASS_APPEARS}} = ();

   @{$res->{$FAIL_PASS}} = ();
   @{$res->{$FAIL_XPASS}} = ();
   @{$res->{$FAIL_FAIL}} = ();
   @{$res->{$FAIL_XFAIL}} = ();
   @{$res->{$FAIL_UNSUPPORTED}} = ();
   @{$res->{$FAIL_UNTESTED}} = ();
   @{$res->{$FAIL_UNRESOLVED}} = ();
   @{$res->{$FAIL_DISAPPEARS}} = ();
   @{$res->{$FAIL_APPEARS}} = ();

   @{$res->{$XFAIL_PASS}} = ();
   @{$res->{$XFAIL_XPASS}} = ();
   @{$res->{$XFAIL_FAIL}} = ();
   @{$res->{$XFAIL_XFAIL}} = ();
   @{$res->{$XFAIL_UNSUPPORTED}} = ();
   @{$res->{$XFAIL_UNTESTED}} = ();
   @{$res->{$XFAIL_UNRESOLVED}} = ();
   @{$res->{$XFAIL_DISAPPEARS}} = ();
   @{$res->{$XFAIL_APPEARS}} = ();

   @{$res->{$UNSUPPORTED_PASS}} = ();
   @{$res->{$UNSUPPORTED_XPASS}} = ();
   @{$res->{$UNSUPPORTED_FAIL}} = ();
   @{$res->{$UNSUPPORTED_XFAIL}} = ();
   @{$res->{$UNSUPPORTED_UNSUPPORTED}} = ();
   @{$res->{$UNSUPPORTED_UNTESTED}} = ();
   @{$res->{$UNSUPPORTED_UNRESOLVED}} = ();
   @{$res->{$UNSUPPORTED_DISAPPEARS}} = ();
   @{$res->{$UNSUPPORTED_APPEARS}} = ();

   @{$res->{$UNTESTED_PASS}} = ();
   @{$res->{$UNTESTED_XPASS}} = ();
   @{$res->{$UNTESTED_FAIL}} = ();
   @{$res->{$UNTESTED_XFAIL}} = ();
   @{$res->{$UNTESTED_UNSUPPORTED}} = ();
   @{$res->{$UNTESTED_UNTESTED}} = ();
   @{$res->{$UNTESTED_UNRESOLVED}} = ();
   @{$res->{$UNTESTED_DISAPPEARS}} = ();
   @{$res->{$UNTESTED_APPEARS}} = ();

   @{$res->{$UNRESOLVED_PASS}} = ();
   @{$res->{$UNRESOLVED_XPASS}} = ();
   @{$res->{$UNRESOLVED_FAIL}} = ();
   @{$res->{$UNRESOLVED_XFAIL}} = ();
   @{$res->{$UNRESOLVED_UNSUPPORTED}} = ();
   @{$res->{$UNRESOLVED_UNTESTED}} = ();
   @{$res->{$UNRESOLVED_UNRESOLVED}} = ();
   @{$res->{$UNRESOLVED_DISAPPEARS}} = ();
   @{$res->{$UNRESOLVED_APPEARS}} = ();

   @{$res->{$ERROR_DISAPPEARS}} = ();
   @{$res->{$ERROR_APPEARS}} = ();

   @{$res->{$UNHANDLED_CASES}} = ();
   @{$res->{$UNSTABLE_CASES}} = ();

   #### MERGE REF AND RES
   foreach my $key (sort (keys %{$res->{testcases}}))
   {
		if (not exists $ref->{testcases}->{$key}) {
		   $ref->{testcases}->{$key} = empty_result();
		   $ref->{testcases}->{$key}->{NO_EXIST} = 1;
		}
   }
   foreach my $key (keys %{$ref->{testcases}})
   {
   		if (not exists $res->{testcases}->{$key})
		{
		   $res->{testcases}->{$key} = empty_result();
		   $res->{testcases}->{$key}->{NO_EXIST} = 1;
		}
   }

   #### ACTIONS FOR EACH CASES
   my %unstable_found;

   foreach my $key (sort (keys %{$ref->{testcases}}))
   {
       foreach my $diag_diag (@handler_list)
       {
	   if ($ref->{testcases}->{$key}->{$diag_diag->{was}} != $res->{testcases}->{$key}->{$diag_diag->{was}}
	       and $res->{testcases}->{$key}->{$diag_diag->{is}})
	   {
	   print "$key ref was:$ref->{testcases}->{$key}->{$diag_diag->{was}}, res was:$res->{testcases}->{$key}->{$diag_diag->{was}}, res is:$res->{testcases}->{$key}->{$diag_diag->{is}} \n" if ($debug);
	       # If testcase is listed as 'unstable' mark it as
	       # such and skip other processing.
	       {
		   if (grep { (index $key,$_)!=-1} @unstablelist)
		   {
		       print "[unstable] $key\n" if ($debug);
		       $unstable_found{$key}=1;
		   }
		   else {
		       print "[$diag_diag->{was} => $diag_diag->{is}] $key\n" if ($debug);
		       if ($diag_diag->{handler})
		       {
			   $diag_diag->{handler} ($ref, $res, $diag_diag, $key);
		       }
		       else
		       {
			   push @{$res->{$diag_diag->{cat}}}, $key;
		       }
		   }
	       }
	   }
       }
   }
   push @{$res->{$UNSTABLE_CASES}}, (sort (keys (%unstable_found)))  if ($nounstable == 0);

}

######################################################
# PRINTING
sub print_tclist($@)
{
   # The 1st parameter is used to find the name of the .exp file
   # associated with the testcase. In the case where the testcase
   # disappeared in the current results compared to the reference,
   # make sure to pass the reference results as 1st parameter,
   # otherwise there will be an undefined reference.
   my ($this_res, $cat, @tclist) = @_;
   my $this_exp = "";
   if (scalar(@tclist))
   {
       print "  - ".$cat.":\n\n";
       foreach my $case (@tclist)
       {
	   if ($this_exp ne $this_res->{testcases}->{$case}->{EXP}) {
	       $this_exp = $this_res->{testcases}->{$case}->{EXP};
	       print "  Executed from: $this_exp\n";
	   }
	   print "    $case\n";
       }
       print "\n\n";
   }
}

sub print_compare_results_summary($$)
{
   my ($ref, $res) = @_;
   my $return_value=0;
   my $total_better = 0;
   my $rtotal = 0;
   my $quiet_reg = $quiet;
   my $ref_ratio = 0;
   my $ref_total = 0;
   my $res_ratio = 0;
   my $res_total = 0;
   my $ignore_exec = 0;
   my $ref_has_exec_tests = 0;
   my $res_has_exec_tests = 0;

   # Compute the pass ratio as a sanity check
   $ref_total = $ref->{PASS} +
       $ref->{XFAIL} +
       $ref->{XPASS} +
       $ref->{FAIL} +
       $ref->{UNRESOLVED} +
       $ref->{UNSUPPORTED} +
       $ref->{UNTESTED};

   # It is possible that no test is executed at all (for instance if
   # RUNTESTFLAGS was too restrictive). Avoid division by 0.
   if ($ref_total > 0) {
       $ref_ratio = ($ref->{PASS} + $ref->{XFAIL}) /
	   $ref_total;
   }

   $res_total = $res->{PASS} +
       $res->{XFAIL} +
       $res->{XPASS} +
       $res->{FAIL} +
       $res->{UNRESOLVED} +
       $res->{UNSUPPORTED} +
       $res->{UNTESTED};

   if ($res_total > 0) {
       $res_ratio = ($res->{PASS} + $res->{XFAIL}) /
	   $res_total;
   }

   if (not $quiet)
   {
       printf "Comparing:\n";
       printf "REFERENCE:$ref_file_name\n";
       printf "CURRENT:  $res_file_name\n\n";
   }

   #### TESTS STATUS
   if (not $quiet and not $short)
   {
       printf "              `                              +---------+---------+\n";
       printf "o  RUN STATUS:                               |   REF   |   RES   |\n";
       printf "  +------------------------------------------+---------+---------+\n";
       printf "  | %-40s | %7d | %7d |\n", "Passes                            [PASS]", $ref->{PASS}, $res->{PASS};
       printf "  | %-40s | %7d | %7d |\n", "Unexpected fails                  [FAIL]", $ref->{FAIL}, $res->{FAIL};
       printf "  | %-40s | %7d | %7d |\n", "Errors                           [ERROR]", $ref->{ERROR}, $res->{ERROR};
       printf "  | %-40s | %7d | %7d |\n", "Unexpected passes                [XPASS]", $ref->{XPASS}, $res->{XPASS};
       printf "  | %-40s | %7d | %7d |\n", "Expected fails                   [XFAIL]", $ref->{XFAIL}, $res->{XFAIL};
       printf "  | %-40s | %7d | %7d |\n", "Unresolved                  [UNRESOLVED]", $ref->{UNRESOLVED}, $res->{UNRESOLVED};
       printf "  | %-40s | %7d | %7d |\n", "Unsupported                [UNSUPPORTED]", $ref->{UNSUPPORTED}, $res->{UNSUPPORTED};
       printf "  | %-40s | %7d | %7d |\n", "Untested                      [UNTESTED]", $ref->{UNTESTED}, $res->{UNTESTED};
       printf "  +------------------------------------------+---------+---------+\n";
       printf "\n";

       printf "    REF PASS ratio: %f\n", $ref_ratio;
       printf "    RES PASS ratio: %f\n", $res_ratio;
       if ($ref_ratio < $ratio_thresh)
       {
	   printf "    ***** ERROR: REF PASS ratio is abnormally low *****\n";
       }
       if ($res_ratio < $ratio_thresh)
       {
	   printf "    ***** ERROR: RES PASS ratio is abnormally low *****\n";
       }

       # If both PASS and FAIL EXEC tests are zero, assume there is no
       # execution test for this tool, and do not warn.
       $ref_has_exec_tests = ($ref->{EXEC}->{PASS} + $ref->{EXEC}->{FAIL}) != 0;
       if ($ref_has_exec_tests)
       {
	   printf "    ***** ERROR: No REF execution test PASSed. Check execution engine configuration. *****\n" if ($ref->{EXEC}->{PASS} == 0);
	   printf "    ***** WARNING: No REF execution test FAILed. Check execution engine configuration. *****\n" if ($ref->{EXEC}->{FAIL} == 0);
       }

       $res_has_exec_tests = ($res->{EXEC}->{PASS} + $res->{EXEC}->{FAIL}) != 0;
       if ($res_has_exec_tests)
       {
	   printf "    ***** ERROR: No RES execution test PASSed. Check execution engine configuration. *****\n" if ($res->{EXEC}->{PASS} == 0);
	   printf "    ***** WARNING: No RES execution test FAILed. Check execution engine configuration. *****\n" if ($res->{EXEC}->{FAIL} == 0);
       }

       # Ignore number of execution tests when computing the return
       # value, if both REF and RES have no execution test.
       $ignore_exec = !$ref_has_exec_tests && !$res_has_exec_tests;
   }

   #### REGRESSIONS ?
   $rtotal = scalar(@{$res->{$PASS_XPASS}})
       +scalar(@{$res->{$PASS_FAIL}})
       +scalar(@{$res->{$PASS_UNRESOLVED}})
       +scalar(@{$res->{$PASSED_NOW_TIMEOUTS}})
       +scalar(@{$res->{$XPASS_FAIL}})
       +scalar(@{$res->{$XPASS_APPEARS}})
       +scalar(@{$res->{$XPASS_UNRESOLVED}})
       +scalar(@{$res->{$FAIL_APPEARS}})
       +scalar(@{$res->{$FAIL_UNRESOLVED}})
       +scalar(@{$res->{$XFAIL_FAIL}})
       +scalar(@{$res->{$XFAIL_DISAPPEARS}})
       +scalar(@{$res->{$XFAIL_UNRESOLVED}})
       +scalar(@{$res->{$UNSUPPORTED_FAIL}})
       +scalar(@{$res->{$UNSUPPORTED_XPASS}})
       +scalar(@{$res->{$UNSUPPORTED_UNRESOLVED}})
       +scalar(@{$res->{$UNTESTED_FAIL}})
       +scalar(@{$res->{$UNTESTED_XPASS}})
       +scalar(@{$res->{$UNTESTED_UNRESOLVED}})
       +scalar(@{$res->{$UNRESOLVED_FAIL}})
       +scalar(@{$res->{$UNRESOLVED_XPASS}})
       +scalar(@{$res->{$UNRESOLVED_APPEARS}})
       +scalar(@{$res->{$ERROR_APPEARS}});

   $quiet_reg=1 if ($short and not $rtotal);

   if (not $quiet_reg)
   {
       printf "\n$col_red"."o  REGRESSIONS:\n";
       printf "  +------------------------------------------+---------+\n";
       printf "  | %-40s | %7d |\n", $PASS_XPASS, scalar(@{$res->{$PASS_XPASS}}) if (scalar(@{$res->{$PASS_XPASS}}));
       printf "  | %-40s | %7d |\n", $PASS_FAIL, scalar(@{$res->{$PASS_FAIL}}) if (scalar(@{$res->{$PASS_FAIL}}));
       printf "  | %-40s | %7d |\n", $PASS_UNRESOLVED, scalar(@{$res->{$PASS_UNRESOLVED}}) if (scalar(@{$res->{$PASS_UNRESOLVED}}));
       printf "  | %-40s | %7d |\n", $PASSED_NOW_TIMEOUTS, scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}) if (scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}));
       printf "  | %-40s | %7d |\n", $ERROR_APPEARS, scalar(@{$res->{$ERROR_APPEARS}}) if (scalar(@{$res->{$ERROR_APPEARS}}));
       printf "  | %-40s | %7d |\n", $XPASS_FAIL, scalar(@{$res->{$XPASS_FAIL}}) if (scalar(@{$res->{$XPASS_FAIL}}));
       printf "  | %-40s | %7d |\n", $XPASS_APPEARS, scalar(@{$res->{$XPASS_APPEARS}}) if (scalar(@{$res->{$XPASS_APPEARS}}));
       printf "  | %-40s | %7d |\n", $XPASS_UNRESOLVED, scalar(@{$res->{$XPASS_UNRESOLVED}}) if (scalar(@{$res->{$XPASS_UNRESOLVED}}));
       printf "  | %-40s | %7d |\n", $FAIL_APPEARS, scalar(@{$res->{$FAIL_APPEARS}}) if (scalar(@{$res->{$FAIL_APPEARS}}));
       printf "  | %-40s | %7d |\n", $FAIL_UNRESOLVED, scalar(@{$res->{$FAIL_UNRESOLVED}}) if (scalar(@{$res->{$FAIL_UNRESOLVED}}));
       printf "  | %-40s | %7d |\n", $XFAIL_FAIL, scalar(@{$res->{$XFAIL_FAIL}}) if (scalar(@{$res->{$XFAIL_FAIL}}));
       printf "  | %-40s | %7d |\n", $XFAIL_DISAPPEARS, scalar(@{$res->{$XFAIL_DISAPPEARS}}) if (scalar(@{$res->{$XFAIL_DISAPPEARS}}));
       printf "  | %-40s | %7d |\n", $XFAIL_UNRESOLVED, scalar(@{$res->{$XFAIL_UNRESOLVED}}) if (scalar(@{$res->{$XFAIL_UNRESOLVED}}));
       printf "  | %-40s | %7d |\n", $UNSUPPORTED_FAIL, scalar(@{$res->{$UNSUPPORTED_FAIL}}) if (scalar(@{$res->{$UNSUPPORTED_FAIL}}));
       printf "  | %-40s | %7d |\n", $UNSUPPORTED_XPASS, scalar(@{$res->{$UNSUPPORTED_XPASS}}) if (scalar(@{$res->{$UNSUPPORTED_XPASS}}));
       printf "  | %-40s | %7d |\n", $UNSUPPORTED_UNRESOLVED, scalar(@{$res->{$UNSUPPORTED_UNRESOLVED}}) if (scalar(@{$res->{$UNSUPPORTED_UNRESOLVED}}));
       printf "  | %-40s | %7d |\n", $UNTESTED_FAIL, scalar(@{$res->{$UNTESTED_FAIL}}) if (scalar(@{$res->{$UNTESTED_FAIL}}));
       printf "  | %-40s | %7d |\n", $UNTESTED_XPASS, scalar(@{$res->{$UNTESTED_XPASS}}) if (scalar(@{$res->{$UNTESTED_XPASS}}));
       printf "  | %-40s | %7d |\n", $UNTESTED_UNRESOLVED, scalar(@{$res->{$UNTESTED_UNRESOLVED}}) if (scalar(@{$res->{$UNTESTED_UNRESOLVED}}));
       printf "  | %-40s | %7d |\n", $UNRESOLVED_FAIL, scalar(@{$res->{$UNRESOLVED_FAIL}}) if (scalar(@{$res->{$UNRESOLVED_FAIL}}));
       printf "  | %-40s | %7d |\n", $UNRESOLVED_XPASS, scalar(@{$res->{$UNRESOLVED_XPASS}}) if (scalar(@{$res->{$UNRESOLVED_XPASS}}));
       printf "  | %-40s | %7d |\n", $UNRESOLVED_APPEARS, scalar(@{$res->{$UNRESOLVED_APPEARS}}) if (scalar(@{$res->{$UNRESOLVED_APPEARS}}));
       printf "  +------------------------------------------+---------+\n";
       printf "  | %-40s | %7d |\n", "TOTAL_REGRESSIONS", $rtotal;
       printf "  +------------------------------------------+---------+\n";
       printf "\n";

       if ($long)
       {
	      print_tclist($res, $PASS_XPASS, @{$res->{$PASS_XPASS}});
	      print_tclist($res, $PASS_FAIL, @{$res->{$PASS_FAIL}});
	      print_tclist($res, $PASS_UNRESOLVED, @{$res->{$PASS_UNRESOLVED}});
	      print_tclist($res, $PASSED_NOW_TIMEOUTS, @{$res->{$PASSED_NOW_TIMEOUTS}});
	      print_tclist($res, $ERROR_APPEARS, @{$res->{$ERROR_APPEARS}});
	      print_tclist($res, $XPASS_FAIL, @{$res->{$XPASS_FAIL}});
	      print_tclist($res, $XPASS_APPEARS, @{$res->{$XPASS_APPEARS}});
	      print_tclist($res, $XPASS_UNRESOLVED, @{$res->{$XPASS_UNRESOLVED}});
	      print_tclist($res, $FAIL_APPEARS, @{$res->{$FAIL_APPEARS}});
	      print_tclist($res, $FAIL_UNRESOLVED, @{$res->{$FAIL_UNRESOLVED}});
	      print_tclist($res, $XFAIL_FAIL, @{$res->{$XFAIL_FAIL}});
	      # When testcases disappear, make sure to pass $ref as 1st argument
	      print_tclist($ref, $XFAIL_DISAPPEARS, @{$res->{$XFAIL_DISAPPEARS}});
	      print_tclist($res, $XFAIL_UNRESOLVED, @{$res->{$XFAIL_UNRESOLVED}});
	      print_tclist($res, $UNSUPPORTED_FAIL, @{$res->{$UNSUPPORTED_FAIL}});
	      print_tclist($res, $UNSUPPORTED_XPASS, @{$res->{$UNSUPPORTED_XPASS}});
	      print_tclist($res, $UNSUPPORTED_UNRESOLVED, @{$res->{$UNSUPPORTED_UNRESOLVED}});
	      print_tclist($res, $UNTESTED_FAIL, @{$res->{$UNTESTED_FAIL}});
	      print_tclist($res, $UNTESTED_XPASS, @{$res->{$UNTESTED_XPASS}});
	      print_tclist($res, $UNTESTED_UNRESOLVED, @{$res->{$UNTESTED_UNRESOLVED}});
	      print_tclist($res, $UNRESOLVED_FAIL, @{$res->{$UNRESOLVED_FAIL}});
	      print_tclist($res, $UNRESOLVED_XPASS, @{$res->{$UNRESOLVED_XPASS}});
	      print_tclist($res, $UNRESOLVED_APPEARS, @{$res->{$UNRESOLVED_APPEARS}});
       }
       printf "$col_reset\n";
   }

   #### MINOR TO BE CHECKED ?
   if (not $quiet and not $short)
   {
       $total_better =
	   scalar(@{$res->{$PASS_DISAPPEARS}})+
	   scalar(@{$res->{$PASS_XFAIL}})+
	   scalar(@{$res->{$PASS_APPEARS}})+
	   scalar(@{$res->{$PASS_UNSUPPORTED}})+
	   scalar(@{$res->{$PASS_UNTESTED}})+
	   scalar(@{$res->{$XPASS_PASS}})+
	   scalar(@{$res->{$XPASS_XFAIL}})+
	   scalar(@{$res->{$XPASS_UNSUPPORTED}})+
	   scalar(@{$res->{$XPASS_UNTESTED}})+
	   scalar(@{$res->{$XPASS_DISAPPEARS}})+
	   scalar(@{$res->{$FAIL_PASS}})+
	   scalar(@{$res->{$FAIL_XPASS}})+
	   scalar(@{$res->{$FAIL_XFAIL}})+
	   scalar(@{$res->{$FAIL_UNSUPPORTED}})+
	   scalar(@{$res->{$FAIL_UNTESTED}})+
	   scalar(@{$res->{$FAIL_DISAPPEARS}})+
	   scalar(@{$res->{$XFAIL_PASS}})+
	   scalar(@{$res->{$XFAIL_XPASS}})+
	   scalar(@{$res->{$XFAIL_UNSUPPORTED}})+
	   scalar(@{$res->{$XFAIL_UNTESTED}})+
	   scalar(@{$res->{$XFAIL_APPEARS}})+
	   scalar(@{$res->{$UNSUPPORTED_PASS}})+
	   scalar(@{$res->{$UNSUPPORTED_XFAIL}})+
	   scalar(@{$res->{$UNSUPPORTED_UNTESTED}})+
	   scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}})+
	   scalar(@{$res->{$UNSUPPORTED_APPEARS}})+
	   scalar(@{$res->{$UNTESTED_PASS}})+
	   scalar(@{$res->{$UNTESTED_XFAIL}})+
	   scalar(@{$res->{$UNTESTED_UNSUPPORTED}})+
	   scalar(@{$res->{$UNTESTED_DISAPPEARS}})+
	   scalar(@{$res->{$UNTESTED_APPEARS}})+
	   scalar(@{$res->{$UNRESOLVED_PASS}})+
	   scalar(@{$res->{$UNRESOLVED_XFAIL}})+
	   scalar(@{$res->{$UNRESOLVED_UNSUPPORTED}})+
	   scalar(@{$res->{$UNRESOLVED_UNTESTED}})+
	   scalar(@{$res->{$UNRESOLVED_DISAPPEARS}})+
	   scalar(@{$res->{$ERROR_DISAPPEARS}})+
	   scalar(@{$res->{$UNHANDLED_CASES}});

       printf "$col_pink"."o  MINOR TO BE CHECKED:\n";
       printf "  +------------------------------------------+---------+\n";
       printf "  | %-40s | %7d |\n", $PASS_DISAPPEARS, scalar(@{$res->{$PASS_DISAPPEARS}}) if (scalar(@{$res->{$PASS_DISAPPEARS}}));
       printf "  | %-40s | %7d |\n", $PASS_XFAIL, scalar(@{$res->{$PASS_XFAIL}}) if (scalar(@{$res->{$PASS_XFAIL}}));
       printf "  | %-40s | %7d |\n", $PASS_APPEARS, scalar(@{$res->{$PASS_APPEARS}}) if (scalar(@{$res->{$PASS_APPEARS}}));
       printf "  | %-40s | %7d |\n", $PASS_UNSUPPORTED, scalar(@{$res->{$PASS_UNSUPPORTED}}) if (scalar(@{$res->{$PASS_UNSUPPORTED}}));
       printf "  | %-40s | %7d |\n", $PASS_UNTESTED, scalar(@{$res->{$PASS_UNTESTED}}) if (scalar(@{$res->{$PASS_UNTESTED}}));
       printf "  | %-40s | %7d |\n", $XPASS_PASS, scalar(@{$res->{$XPASS_PASS}}) if (scalar(@{$res->{$XPASS_PASS}}));
       printf "  | %-40s | %7d |\n", $XPASS_XFAIL, scalar(@{$res->{$XPASS_XFAIL}}) if (scalar(@{$res->{$XPASS_XFAIL}}));
       printf "  | %-40s | %7d |\n", $XPASS_UNSUPPORTED, scalar(@{$res->{$XPASS_UNSUPPORTED}}) if (scalar(@{$res->{$XPASS_UNSUPPORTED}}));
       printf "  | %-40s | %7d |\n", $XPASS_UNTESTED, scalar(@{$res->{$XPASS_UNTESTED}}) if (scalar(@{$res->{$XPASS_UNTESTED}}));
       printf "  | %-40s | %7d |\n", $XPASS_DISAPPEARS, scalar(@{$res->{$XPASS_DISAPPEARS}}) if (scalar(@{$res->{$XPASS_DISAPPEARS}}));
       printf "  | %-40s | %7d |\n", $FAIL_PASS, scalar(@{$res->{$FAIL_PASS}}) if (scalar(@{$res->{$FAIL_PASS}}));
       printf "  | %-40s | %7d |\n", $FAIL_XPASS, scalar(@{$res->{$FAIL_XPASS}}) if (scalar(@{$res->{$FAIL_XPASS}}));
       printf "  | %-40s | %7d |\n", $FAIL_XFAIL, scalar(@{$res->{$FAIL_XFAIL}}) if (scalar(@{$res->{$FAIL_XFAIL}}));
       printf "  | %-40s | %7d |\n", $FAIL_UNSUPPORTED, scalar(@{$res->{$FAIL_UNSUPPORTED}}) if (scalar(@{$res->{$FAIL_UNSUPPORTED}}));
       printf "  | %-40s | %7d |\n", $FAIL_UNTESTED, scalar(@{$res->{$FAIL_UNTESTED}}) if (scalar(@{$res->{$FAIL_UNTESTED}}));
       printf "  | %-40s | %7d |\n", $FAIL_DISAPPEARS, scalar(@{$res->{$FAIL_DISAPPEARS}}) if (scalar(@{$res->{$FAIL_DISAPPEARS}}));
       printf "  | %-40s | %7d |\n", $XFAIL_PASS, scalar(@{$res->{$XFAIL_PASS}}) if (scalar(@{$res->{$XFAIL_PASS}}));
       printf "  | %-40s | %7d |\n", $XFAIL_XPASS, scalar(@{$res->{$XFAIL_XPASS}}) if (scalar(@{$res->{$XFAIL_XPASS}}));
       printf "  | %-40s | %7d |\n", $XFAIL_UNSUPPORTED, scalar(@{$res->{$XFAIL_UNSUPPORTED}}) if (scalar(@{$res->{$XFAIL_UNSUPPORTED}}));
       printf "  | %-40s | %7d |\n", $XFAIL_UNTESTED, scalar(@{$res->{$XFAIL_UNTESTED}}) if (scalar(@{$res->{$XFAIL_UNTESTED}}));
       printf "  | %-40s | %7d |\n", $XFAIL_APPEARS, scalar(@{$res->{$XFAIL_APPEARS}}) if (scalar(@{$res->{$XFAIL_APPEARS}}));
       printf "  | %-40s | %7d |\n", $UNSUPPORTED_PASS, scalar(@{$res->{$UNSUPPORTED_PASS}}) if (scalar(@{$res->{$UNSUPPORTED_PASS}}));
       printf "  | %-40s | %7d |\n", $UNSUPPORTED_XFAIL, scalar(@{$res->{$UNSUPPORTED_XFAIL}}) if (scalar(@{$res->{$UNSUPPORTED_XFAIL}}));
       printf "  | %-40s | %7d |\n", $UNSUPPORTED_UNTESTED, scalar(@{$res->{$UNSUPPORTED_UNTESTED}}) if (scalar(@{$res->{$UNSUPPORTED_UNTESTED}}));
       printf "  | %-40s | %7d |\n", $UNSUPPORTED_DISAPPEARS, scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}}) if (scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}}));
       printf "  | %-40s | %7d |\n", $UNSUPPORTED_APPEARS, scalar(@{$res->{$UNSUPPORTED_APPEARS}}) if (scalar(@{$res->{$UNSUPPORTED_APPEARS}}));
       printf "  | %-40s | %7d |\n", $UNTESTED_PASS, scalar(@{$res->{$UNTESTED_PASS}}) if (scalar(@{$res->{$UNTESTED_PASS}}));
       printf "  | %-40s | %7d |\n", $UNTESTED_XFAIL, scalar(@{$res->{$UNTESTED_XFAIL}}) if (scalar(@{$res->{$UNTESTED_XFAIL}}));
       printf "  | %-40s | %7d |\n", $UNTESTED_UNSUPPORTED, scalar(@{$res->{$UNTESTED_UNSUPPORTED}}) if (scalar(@{$res->{$UNTESTED_UNSUPPORTED}}));
       printf "  | %-40s | %7d |\n", $UNTESTED_DISAPPEARS, scalar(@{$res->{$UNTESTED_DISAPPEARS}}) if (scalar(@{$res->{$UNTESTED_DISAPPEARS}}));
       printf "  | %-40s | %7d |\n", $UNTESTED_APPEARS, scalar(@{$res->{$UNTESTED_APPEARS}}) if (scalar(@{$res->{$UNTESTED_APPEARS}}));
       printf "  | %-40s | %7d |\n", $UNRESOLVED_PASS, scalar(@{$res->{$UNRESOLVED_PASS}}) if (scalar(@{$res->{$UNRESOLVED_PASS}}));
       printf "  | %-40s | %7d |\n", $UNRESOLVED_XFAIL, scalar(@{$res->{$UNRESOLVED_XFAIL}}) if (scalar(@{$res->{$UNRESOLVED_XFAIL}}));
       printf "  | %-40s | %7d |\n", $UNRESOLVED_UNSUPPORTED, scalar(@{$res->{$UNRESOLVED_UNSUPPORTED}}) if (scalar(@{$res->{$UNRESOLVED_UNSUPPORTED}}));
       printf "  | %-40s | %7d |\n", $UNRESOLVED_UNTESTED, scalar(@{$res->{$UNRESOLVED_UNTESTED}}) if (scalar(@{$res->{$UNRESOLVED_UNTESTED}}));
       printf "  | %-40s | %7d |\n", $UNRESOLVED_DISAPPEARS, scalar(@{$res->{$UNRESOLVED_DISAPPEARS}}) if (scalar(@{$res->{$UNRESOLVED_DISAPPEARS}}));
       printf "  | %-40s | %7d |\n", $ERROR_DISAPPEARS, scalar(@{$res->{$ERROR_DISAPPEARS}}) if (scalar(@{$res->{$ERROR_DISAPPEARS}}));
       printf "  | %-40s | %7d |\n", $UNHANDLED_CASES, scalar(@{$res->{$UNHANDLED_CASES}}) if (scalar(@{$res->{$UNHANDLED_CASES}}));
       printf "  | %-40s | %7d |\n", $UNSTABLE_CASES, scalar(@{$res->{$UNSTABLE_CASES}}) if (scalar(@{$res->{$UNSTABLE_CASES}}));
       printf "  +------------------------------------------+---------+\n";
       printf "  | %-40s | %7d |\n", "TOTAL_MINOR_TO_BE_CHECKED", $total_better + scalar(@{$res->{$UNSTABLE_CASES}});;
       printf "  +------------------------------------------+---------+\n";
       printf "\n";

       if ($long)
       {
	      # When testcases disappear, make sure to pass $ref as 1st argument
	      print_tclist($ref, $PASS_DISAPPEARS, @{$res->{$PASS_DISAPPEARS}});
	      print_tclist($res, $PASS_XFAIL, @{$res->{$PASS_XFAIL}});
	      print_tclist($res, $PASS_APPEARS, @{$res->{$PASS_APPEARS}});
	      print_tclist($res, $PASS_UNSUPPORTED, @{$res->{$PASS_UNSUPPORTED}});
	      print_tclist($res, $PASS_UNTESTED, @{$res->{$PASS_UNTESTED}});
	      print_tclist($res, $XPASS_PASS, @{$res->{$XPASS_PASS}});
	      print_tclist($res, $XPASS_XFAIL, @{$res->{$XPASS_XFAIL}});
	      print_tclist($res, $XPASS_UNSUPPORTED, @{$res->{$XPASS_UNSUPPORTED}});
	      print_tclist($res, $XPASS_UNTESTED, @{$res->{$XPASS_UNTESTED}});
	      # When testcases disappear, make sure to pass $ref as 1st argument
	      print_tclist($ref, $XPASS_DISAPPEARS, @{$res->{$XPASS_DISAPPEARS}});
	      print_tclist($res, $XFAIL_PASS, @{$res->{$XFAIL_PASS}});
	      print_tclist($res, $XFAIL_XPASS, @{$res->{$XFAIL_XPASS}});
	      print_tclist($res, $FAIL_PASS, @{$res->{$FAIL_PASS}});
	      print_tclist($res, $FAIL_XPASS, @{$res->{$FAIL_XPASS}});
	      print_tclist($res, $FAIL_XFAIL, @{$res->{$FAIL_XFAIL}});
	      print_tclist($res, $FAIL_UNSUPPORTED, @{$res->{$FAIL_UNSUPPORTED}});
	      print_tclist($res, $FAIL_UNTESTED, @{$res->{$FAIL_UNTESTED}});
	      # When testcases disappear, make sure to pass $ref as 1st argument
	      print_tclist($ref, $FAIL_DISAPPEARS, @{$res->{$FAIL_DISAPPEARS}});
	      print_tclist($res, $XFAIL_PASS, @{$res->{$XFAIL_PASS}});
	      print_tclist($res, $XFAIL_XPASS, @{$res->{$XFAIL_XPASS}});
	      print_tclist($res, $XFAIL_UNSUPPORTED, @{$res->{$XFAIL_UNSUPPORTED}});
	      print_tclist($res, $XFAIL_UNTESTED, @{$res->{$XFAIL_UNTESTED}});
	      print_tclist($res, $XFAIL_APPEARS, @{$res->{$XFAIL_APPEARS}});
	      print_tclist($res, $UNSUPPORTED_PASS, @{$res->{$UNSUPPORTED_PASS}});
	      print_tclist($res, $UNSUPPORTED_XFAIL, @{$res->{$UNSUPPORTED_XFAIL}});
	      print_tclist($res, $UNSUPPORTED_UNTESTED, @{$res->{$UNSUPPORTED_UNTESTED}});
	      # When testcases disappear, make sure to pass $ref as 1st argument
	      print_tclist($ref, $UNSUPPORTED_DISAPPEARS, @{$res->{$UNSUPPORTED_DISAPPEARS}});
	      print_tclist($res, $UNSUPPORTED_APPEARS, @{$res->{$UNSUPPORTED_APPEARS}});
	      print_tclist($res, $UNTESTED_PASS, @{$res->{$UNTESTED_PASS}});
	      print_tclist($res, $UNTESTED_XFAIL, @{$res->{$UNTESTED_XFAIL}});
	      print_tclist($res, $UNTESTED_UNSUPPORTED, @{$res->{$UNTESTED_UNSUPPORTED}});
	      # When testcases disappear, make sure to pass $ref as 1st argument
	      print_tclist($ref, $UNTESTED_DISAPPEARS, @{$res->{$UNTESTED_DISAPPEARS}});
	      print_tclist($res, $UNTESTED_APPEARS, @{$res->{$UNTESTED_APPEARS}});
	      print_tclist($res, $UNRESOLVED_PASS, @{$res->{$UNRESOLVED_PASS}});
	      print_tclist($res, $UNRESOLVED_XFAIL, @{$res->{$UNRESOLVED_XFAIL}});
	      print_tclist($res, $UNRESOLVED_UNSUPPORTED, @{$res->{$UNRESOLVED_UNSUPPORTED}});
	      print_tclist($res, $UNRESOLVED_UNTESTED, @{$res->{$UNRESOLVED_UNTESTED}});
	      # When testcases disappear, make sure to pass $ref as 1st argument
	      print_tclist($ref, $UNRESOLVED_DISAPPEARS, @{$res->{$UNRESOLVED_DISAPPEARS}});
	      print_tclist($ref, $ERROR_DISAPPEARS, @{$res->{$ERROR_DISAPPEARS}});
	      print_tclist($res, $UNHANDLED_CASES, @{$res->{$UNHANDLED_CASES}});
	      print_tclist($res, $UNSTABLE_CASES, @{$res->{$UNSTABLE_CASES}});
       }
       printf "$col_reset\n";
   }

   $return_value = 1 if ($total_better);

   $return_value = 2 if ($rtotal);

   # Error if there was no PASS (eg when sth went wrong and no .sum was generated)
   $return_value = 2 if (($res->{PASS} + $res->{XFAIL}) == 0);

   # Error if every test passed, or the ratio was too low. This
   # generally means the simulator is not configured well.
   $return_value = 2 if (($ref_ratio == 100) || ($ref_ratio < $ratio_thresh));
   $return_value = 2 if (($res_ratio == 100) || ($res_ratio < $ratio_thresh));

   # Error if no execution test passed.
   # It is possible that no execution test fails, though.
   $return_value = 2 if (($ignore_exec == 0)
			 && (($ref->{EXEC}->{PASS} == 0)
			     || ($res->{EXEC}->{PASS} == 0)));

   return $return_value;
}