diff options
author | Christophe Lyon <christophe.lyon@linaro.org> | 2017-05-22 12:04:00 +0000 |
---|---|---|
committer | Christophe Lyon <christophe.lyon@linaro.org> | 2017-05-30 14:23:47 +0000 |
commit | ca9d83087f0d16cc3ed942c625544c6bd787bbb3 (patch) | |
tree | 33fe71313f0deb5079b425a6ec2cb07516b6bbbd | |
parent | 14d1ec802b290b9ee5c886081c5c40e08e369f08 (diff) |
Add -pass-thresh parameter.
To control the threshold for the "PASS ratio is abnormally low" error
message.
The use case is binutils testing, where cross tools have a high enough
number of UNTESTED/UNSUPPORTED tests, leading to a pass rate of about
93%, which is below the current default of 95%, which we want to keep
for GCC validations.
Change-Id: Iad359f3efda19e4d70cf6e8776e8d541d582b42a
-rwxr-xr-x | compare_dg_tests.pl | 3 | ||||
-rwxr-xr-x | compare_jobs.sh | 5 | ||||
-rwxr-xr-x | compare_tests | 32 |
3 files changed, 34 insertions, 6 deletions
diff --git a/compare_dg_tests.pl b/compare_dg_tests.pl index aec9573..6fb37b1 100755 --- a/compare_dg_tests.pl +++ b/compare_dg_tests.pl @@ -185,6 +185,7 @@ my ($ref_file_name, $res_file_name); my $nounstable=0; my $unstablefile=0; my @unstable_markers=(); +my $ratio_thresh = 0.95; # Warn if pass ratio is below this threshold GetOptions ("l" => \$long, "s" => \$short, @@ -193,6 +194,7 @@ GetOptions ("l" => \$long, "dbg" => \$debug, "testroot=s" => \$testroot, "basename=s" => \$basename, + "pass-thresh=f" => \$ratio_thresh, "no-unstable" => \$nounstable, "unstable-tests=s" => \$unstablefile, "unstable-marker=s" => \@unstable_markers); @@ -538,7 +540,6 @@ sub print_compare_results_summary($$) my $ref_total = 0; my $res_ratio = 0; my $res_total = 0; - my $ratio_thresh = 0.95; # Warn if pass ratio is below this threshold my $ignore_exec = 0; my $ref_has_exec_tests = 0; my $res_has_exec_tests = 0; diff --git a/compare_jobs.sh b/compare_jobs.sh index 1499376..76c6476 100755 --- a/compare_jobs.sh +++ b/compare_jobs.sh @@ -3,9 +3,11 @@ mydir="`dirname $0`" status=0 +[ x"$1" = x"-pass-thresh" ] && pass_thresh=$2 && shift 2 + if [ $# != 2 ] then - echo "Usage: $0 ref_logs new_logs" + echo "Usage: $0 [-pass-thresh pass-ratio-threshold] ref_logs new_logs" exit 1 fi @@ -129,6 +131,7 @@ do then printf "# Running on `hostname` in `pwd`\n" >> ${mylog} ${mydir}/compare_tests -target ${target} \ + ${pass_thresh:+-pass-thresh=${pass_thresh}} \ ${ref} ${build} >> ${mylog} ret=$? fi diff --git a/compare_tests b/compare_tests index d98f7fe..39f74a3 100755 --- a/compare_tests +++ b/compare_tests @@ -13,7 +13,7 @@ usage() echo >&2 fi cat >&2 <<EOUSAGE -Usage: $0 [-target target-triplet] PREVIOUS CURRENT +Usage: $0 [-target target-triplet] [-pass-thresh pass-ratio-threshold] PREVIOUS CURRENT Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest. @@ -23,6 +23,9 @@ Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of int -target enables to provide the target name to use when parsing the file containing the list of unstable tests. + -pass-thresh controls the level of warning when too few tests + pass. + Exit with the following values: 0 if there is nothing of interest 1 if there are improvements @@ -54,8 +57,29 @@ sum1=/tmp/$tool-sum1.$$ sum2=/tmp/$tool-sum2.$$ tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2" -[ "$1" = "-target" ] && target=$2 && shift 2 -[ "$1" = "-?" ] && usage +while [ $# -gt 2 ] +do + case "$1" in + "-target") + target=$2 + shift 2 + ;; + "-pass-thresh") + pass_thresh=$2 + shift 2 + ;; + "-?") + usage + ;; + *) + if [ $# -gt 2 ]; then + echo "ERROR: Too many arguments: $@" + usage + fi + ;; + esac +done + [ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT" trap "rm -f $tmps" 0 1 2 3 5 9 13 15 @@ -113,7 +137,7 @@ if [ -d "$1" -a -d "$2" ] ; then unstable_target="$unstable_target --unstable-marker $board" fi done - ${CONFIG_SHELL-/usr/bin/perl} ${my_path}/compare_dg_tests.pl -l --unstable-test=${my_path}/unstable-tests.txt ${unstable_target} $sum1 $sum2 + ${CONFIG_SHELL-/usr/bin/perl} ${my_path}/compare_dg_tests.pl ${pass_thresh:+-pass-thresh=${pass_thresh}} -l --unstable-test=${my_path}/unstable-tests.txt ${unstable_target} $sum1 $sum2 ret=$? case $ret in 2) |