compare_tests: Fix usage and exit codes.
compare_jobs.sh: Handle the new return codes.
compare_test has now 5 exit codes:
0: no change
1: improvements
2: regressions
3: no logs in common, cannot compare
4: extra logs in either previous or current
Exit codes 2, 3 and 4 are reported as failures by compare_jobs.sh,
with different error messages in the report table.
Change-Id: I7313ed38cd4cbb5ab1c6795391a6977f3576562a
diff --git a/compare_tests b/compare_tests
index 74da3be..092760d 100755
--- a/compare_tests
+++ b/compare_tests
@@ -13,25 +13,23 @@
echo >&2
fi
cat >&2 <<EOUSAGE
-Usage: $0 [-strict] [-target target-triplet] PREVIOUS CURRENT
+Usage: $0 [-target target-triplet] PREVIOUS CURRENT
Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
- If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
-
- Unless -strict is given, these discrepancies are not counted as errors:
- missing/extra .sum files when comparing directories
- tests that failed in PREVIOUS but pass in CURRENT
- tests that were not in PREVIOUS but appear in CURRENT
- tests in PREVIOUS that are missing in CURRENT
+ PREVIOUS and CURRENT must be directories, this tool finds and
+ compares any *.sum files they contain.
-target enables to provide the target name to use when parsing
the file containing the list of unstable tests.
Exit with the following values:
0 if there is nothing of interest
- 1 if there are errors when comparing single test case files
- N for the number of errors found when comparing directories
+ 1 if there are improvements
+ 2 if the are regressions or new errors
+ 3 if there were build errors (no common logs)
+ 4 if there are extra .sum files in either PREVIOUS or
+ CURRENT
EOUSAGE
exit 2
}
@@ -56,7 +54,6 @@
sum2=/tmp/$tool-sum2.$$
tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
-[ "$1" = "-strict" ] && strict=$1 && shift
[ "$1" = "-target" ] && target=$2 && shift 2
[ "$1" = "-?" ] && usage
[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
@@ -79,19 +76,19 @@
echo "# Extra sum files in Dir1=$1"
sed -e "s|^|< $1/|" $lst5
echo
- [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+ exit_status=4
fi
comm -13 $lst3 $lst4 >$lst5
if [ -s $lst5 ] ; then
echo "# Extra sum files in Dir2=$2"
sed -e "s|^|> $2/|" $lst5
echo
- [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+ exit_status=4
fi
comm -12 $lst3 $lst4 | sort -u >$lst5
if [ ! -s $lst5 ] ; then
echo "# No common sum files"
- exit_status=`expr $exit_status + 1`
+ exit_status=3
exit $exit_status
fi
cmnsums=`cat $lst5 | wc -l`
@@ -118,11 +115,11 @@
ret=$?
case $ret in
2)
- exit_status=`expr $exit_status + 2`
+ [ $exit_status -eq 0 ] && exit_status=2
echo "# Regressions found"
;;
1)
- exit_status=`expr $exit_status + 1`
+ [ $exit_status -eq 0 ] && exit_status=1
echo "# Improvements found"
;;
esac
@@ -133,5 +130,5 @@
fi
exit $exit_status
elif [ -d "$1" -o -d "$2" ] ; then
- usage "Must specify either two directories or two files"
+ usage "Must specify two directories"
fi