compare_tests

#!/bin/sh
# This script automatically test the given tool with the tool's test cases,
# reporting anything of interest.

# Written by Mike Stump <mrs@cygnus.com>
# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
# Modified by Yvan Roux <yvan.roux@linaro.org> and
# Christophe Lyon <christophe.lyon@linaro.org>
usage()
{
	if [ -n "$1" ] ; then
		echo "$0: Error: $1" >&2
		echo >&2
	fi
	cat >&2 <<EOUSAGE
Usage: $0 [-target target-triplet] [-pass-thresh pass-ratio-threshold] [-compr compression-type] PREVIOUS CURRENT

Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.

	PREVIOUS and CURRENT must be directories, this tool finds and
	compares any *.sum files they contain.

        -target enables to provide the target name to use when parsing
         the file containing the list of unstable tests.

        -pass-thresh controls the level of warning when too few tests
         pass.

        -compr controls the compression type used when searching for
         the .sum files. Defaults to xz, use 'none' for uncompressed
         .sum files.

	Exit with the following values:
		0 if there is nothing of interest
		1 if there are improvements
		2 if the are regressions or new errors
		3 if there were build errors (no common logs)
		4 if there are extra .sum files in either PREVIOUS or
		  CURRENT
EOUSAGE
	exit 2
}

export LC_ALL=C

me="`which $0`"
my_path="`dirname ${me}`"

tool=gxx

tmp1=/tmp/$tool-testing.$$a
tmp2=/tmp/$tool-testing.$$b
now_s=/tmp/$tool-testing.$$d
before_s=/tmp/$tool-testing.$$e
lst1=/tmp/$tool-lst1.$$
lst2=/tmp/$tool-lst2.$$
lst3=/tmp/$tool-lst3.$$
lst4=/tmp/$tool-lst4.$$
lst5=/tmp/$tool-lst5.$$
sum1=/tmp/$tool-sum1.$$
sum2=/tmp/$tool-sum2.$$
tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
compr="xz"
compr_suffix=".xz"

while [ $# -gt 2 ]
do
    case "$1" in
	"-target")
	    target=$2
	    shift 2
	    ;;
	"-pass-thresh")
	    pass_thresh=$2
	    shift 2
	    ;;
	"-compr")
	    compr=$2
	    compr_suffix=".$2"
	    shift 2
	    if [ x"$compr" = xnone ]; then
		compr=""
		compr_suffix=""
	    fi
	    ;;
	"-flaky-tests")
	    flaky_tests=$2
	    shift 2
	    ;;
	"-?")
	    usage
	    ;;
	*)
	    if [ $# -gt 2 ]; then
		echo "ERROR: Too many arguments: $@"
		usage
	    fi
	    ;;
    esac
done

[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"

trap "rm -f $tmps" 0 1 2 3 5 9 13 15
exit_status=0

if [ -d "$1" -a -d "$2" ] ; then
	find "$1/" \( -name "*.sum${compr_suffix}" \)>$lst1
	find "$2/" \( -name "*.sum${compr_suffix}" \)>$lst2
	echo "# Comparing directories"
	echo "# REFERENCE: $1"
	echo "# CURRENT:   $2"
	echo
	# remove leading directory components to compare
	sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
	sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
	comm -23 $lst3 $lst4 >$lst5
	if [ -s $lst5 ] ; then
		echo "# Extra sum files in Dir1=$1"
		sed -e "s|^|< $1/|" $lst5
		echo
		exit_status=4
	fi
	comm -13 $lst3 $lst4 >$lst5
	if [ -s $lst5 ] ; then
		echo "# Extra sum files in Dir2=$2"
		sed -e "s|^|> $2/|" $lst5
		echo
		exit_status=4
	fi
	comm -12 $lst3 $lst4 | sort -u >$lst5
	if [ ! -s $lst5 ] ; then
		echo "# No common sum files"
		exit_status=3
		exit $exit_status
	fi
	cmnsums=`cat $lst5 | wc -l`
	echo "# Comparing $cmnsums common sum files:"
	cat $lst5
	( for fname in `cat $lst5`; do
	      ${compr}cat $1/$fname \
		  | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/' \
		  | sed -e "s|^\([A-Z]*: \).*/home/.*/testsuite/|\1|"
	  done ) >$sum1
	( for fname in `cat $lst5`; do
	      ${compr}cat $2/$fname \
		  | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/' \
		  | sed -e "s|^\([A-Z]*: \).*/home/.*/testsuite/|\1|"
	  done ) >$sum2
	if [ "x${target}" != "x" ] ; then
	    unstable_target="--unstable-marker ${target}"
	fi
	for sum in $sum1 $sum2; do
	    board="$(grep "Running target " $sum | head -n 1 | sed -e "s/Running target //")"
	    if [ x"$board" != x"" ]; then
		unstable_target="$unstable_target --unstable-marker $board"
	    fi
	done
	${CONFIG_SHELL-/usr/bin/perl} ${my_path}/compare_dg_tests.pl \
				      -l \
				      ${pass_thresh:+-pass-thresh=${pass_thresh}} \
				      ${flaky_tests:+-flaky-tests=${flaky_tests}} \
				      --unstable-test=${my_path}/unstable-tests.txt \
				      ${unstable_target} \
				      --hwdep-tests=${my_path}/host-hw-dependent-tests.txt \
				      $sum1 $sum2
	ret=$?
	case $ret in
	    2)
		[ $exit_status -eq 0 ] && exit_status=2
		echo "# Regressions found"
		;;
	    1)
		[ $exit_status -eq 0 ] && exit_status=1
		echo "# Improvements found"
		;;
	esac
	if [ $ret -eq 2 ]; then
		echo "# Regressions in $cmnsums common sum files found"
	else
		echo "# No regression found in $cmnsums common sum files"
	fi
	exit $exit_status
elif [ -d "$1" -o -d "$2" ] ; then
	usage "Must specify two directories"
fi