summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristophe Lyon <christophe.lyon@linaro.org>2017-05-22 12:04:00 +0000
committerChristophe Lyon <christophe.lyon@linaro.org>2017-05-30 14:23:47 +0000
commitca9d83087f0d16cc3ed942c625544c6bd787bbb3 (patch)
tree33fe71313f0deb5079b425a6ec2cb07516b6bbbd
parent14d1ec802b290b9ee5c886081c5c40e08e369f08 (diff)
Add -pass-thresh parameter.
To control the threshold for the "PASS ratio is abnormally low" error message. The use case is binutils testing, where cross tools have a high enough number of UNTESTED/UNSUPPORTED tests, leading to a pass rate of about 93%, which is below the current default of 95%, which we want to keep for GCC validations. Change-Id: Iad359f3efda19e4d70cf6e8776e8d541d582b42a
-rwxr-xr-xcompare_dg_tests.pl3
-rwxr-xr-xcompare_jobs.sh5
-rwxr-xr-xcompare_tests32
3 files changed, 34 insertions, 6 deletions
diff --git a/compare_dg_tests.pl b/compare_dg_tests.pl
index aec9573..6fb37b1 100755
--- a/compare_dg_tests.pl
+++ b/compare_dg_tests.pl
@@ -185,6 +185,7 @@ my ($ref_file_name, $res_file_name);
my $nounstable=0;
my $unstablefile=0;
my @unstable_markers=();
+my $ratio_thresh = 0.95; # Warn if pass ratio is below this threshold
GetOptions ("l" => \$long,
"s" => \$short,
@@ -193,6 +194,7 @@ GetOptions ("l" => \$long,
"dbg" => \$debug,
"testroot=s" => \$testroot,
"basename=s" => \$basename,
+ "pass-thresh=f" => \$ratio_thresh,
"no-unstable" => \$nounstable,
"unstable-tests=s" => \$unstablefile,
"unstable-marker=s" => \@unstable_markers);
@@ -538,7 +540,6 @@ sub print_compare_results_summary($$)
my $ref_total = 0;
my $res_ratio = 0;
my $res_total = 0;
- my $ratio_thresh = 0.95; # Warn if pass ratio is below this threshold
my $ignore_exec = 0;
my $ref_has_exec_tests = 0;
my $res_has_exec_tests = 0;
diff --git a/compare_jobs.sh b/compare_jobs.sh
index 1499376..76c6476 100755
--- a/compare_jobs.sh
+++ b/compare_jobs.sh
@@ -3,9 +3,11 @@
mydir="`dirname $0`"
status=0
+[ x"$1" = x"-pass-thresh" ] && pass_thresh=$2 && shift 2
+
if [ $# != 2 ]
then
- echo "Usage: $0 ref_logs new_logs"
+ echo "Usage: $0 [-pass-thresh pass-ratio-threshold] ref_logs new_logs"
exit 1
fi
@@ -129,6 +131,7 @@ do
then
printf "# Running on `hostname` in `pwd`\n" >> ${mylog}
${mydir}/compare_tests -target ${target} \
+ ${pass_thresh:+-pass-thresh=${pass_thresh}} \
${ref} ${build} >> ${mylog}
ret=$?
fi
diff --git a/compare_tests b/compare_tests
index d98f7fe..39f74a3 100755
--- a/compare_tests
+++ b/compare_tests
@@ -13,7 +13,7 @@ usage()
echo >&2
fi
cat >&2 <<EOUSAGE
-Usage: $0 [-target target-triplet] PREVIOUS CURRENT
+Usage: $0 [-target target-triplet] [-pass-thresh pass-ratio-threshold] PREVIOUS CURRENT
Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
@@ -23,6 +23,9 @@ Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of int
-target enables to provide the target name to use when parsing
the file containing the list of unstable tests.
+ -pass-thresh controls the level of warning when too few tests
+ pass.
+
Exit with the following values:
0 if there is nothing of interest
1 if there are improvements
@@ -54,8 +57,29 @@ sum1=/tmp/$tool-sum1.$$
sum2=/tmp/$tool-sum2.$$
tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
-[ "$1" = "-target" ] && target=$2 && shift 2
-[ "$1" = "-?" ] && usage
+while [ $# -gt 2 ]
+do
+ case "$1" in
+ "-target")
+ target=$2
+ shift 2
+ ;;
+ "-pass-thresh")
+ pass_thresh=$2
+ shift 2
+ ;;
+ "-?")
+ usage
+ ;;
+ *)
+ if [ $# -gt 2 ]; then
+ echo "ERROR: Too many arguments: $@"
+ usage
+ fi
+ ;;
+ esac
+done
+
[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
trap "rm -f $tmps" 0 1 2 3 5 9 13 15
@@ -113,7 +137,7 @@ if [ -d "$1" -a -d "$2" ] ; then
unstable_target="$unstable_target --unstable-marker $board"
fi
done
- ${CONFIG_SHELL-/usr/bin/perl} ${my_path}/compare_dg_tests.pl -l --unstable-test=${my_path}/unstable-tests.txt ${unstable_target} $sum1 $sum2
+ ${CONFIG_SHELL-/usr/bin/perl} ${my_path}/compare_dg_tests.pl ${pass_thresh:+-pass-thresh=${pass_thresh}} -l --unstable-test=${my_path}/unstable-tests.txt ${unstable_target} $sum1 $sum2
ret=$?
case $ret in
2)