Compute pass ratio as a sanity check.
Warn if it is below 0.95.
Change-Id: Id8c1ad8692020f77cfefbda9311ebdba19000317
diff --git a/compare_dg_tests.pl b/compare_dg_tests.pl
index c8b166c..932ec80 100755
--- a/compare_dg_tests.pl
+++ b/compare_dg_tests.pl
@@ -407,6 +407,23 @@
my $total_better = 0;
my $rtotal = 0;
my $quiet_reg = $quiet;
+ my $ref_ratio = 0;
+ my $res_ratio = 0;
+ my $ratio_thresh = 0.95; # Warn if pass ratio is below this threshold
+
+ # Compute the pass ratio as a sanity check
+ $ref_ratio = ($ref->{PASS} + $ref->{XFAIL}) /
+ ($ref->{PASS} + $ref->{XFAIL}
+ + $ref->{XPASS} + $ref->{FAIL}
+ + $ref->{UNRESOLVED}
+ + $ref->{UNSUPPORTED}
+ + $ref->{UNTESTED});
+ $res_ratio = ($res->{PASS} + $res->{XFAIL}) /
+ ($res->{PASS} + $res->{XFAIL}
+ + $res->{XPASS} + $res->{FAIL}
+ + $res->{UNRESOLVED}
+ + $res->{UNSUPPORTED}
+ + $res->{UNTESTED});
if (not $quiet)
{
@@ -430,6 +447,18 @@
printf " | %-40s | %7d | %7d |\n", "Untested [UNTESTED]", $ref->{UNTESTED}, $res->{UNTESTED};
printf " +------------------------------------------+---------+---------+\n";
printf "\n";
+
+ printf " REF PASS ratio: %.2f\n", $ref_ratio;
+ printf " RES PASS ratio: %.2f\n", $res_ratio;
+ if ($ref_ratio < $ratio_thresh)
+ {
+ printf " ***** WARNING: REF PASS ratio is abnormally low *****\n";
+ }
+ if ($res_ratio < $ratio_thresh)
+ {
+ printf " ***** WARNING: RES PASS ratio is abnormally low *****\n";
+ }
+
}
#### REGRESSIONS ?
@@ -585,8 +614,13 @@
$return_value = 2 if ($rtotal);
- # Error if there was no PASS (eg when sth went wrong and no .sum was generated
- $return_value = 2 if (($res->{PASS} + $res->{XPASS}) == 0);
+ # Error if there was no PASS (eg when sth went wrong and no .sum was generated)
+ $return_value = 2 if (($res->{PASS} + $res->{XFAIL}) == 0);
+
+ # Error if every test passed, or the ratio was too low. This
+ # generally means the simulator is not configured well.
+ $return_value = 2 if (($ref_ratio == 100) || ($ref_ratio < $ratio_thresh));
+ $return_value = 2 if (($res_ratio == 100) || ($res_ratio < $ratio_thresh));
return $return_value;
}