blob: 39f74a323e930e477fe1d6e9d22f834ebbfdf76a [file] [log] [blame]
Christophe Lyonbff39612015-11-18 16:29:11 +01001#!/bin/sh
2# This script automatically test the given tool with the tool's test cases,
3# reporting anything of interest.
4
5# Written by Mike Stump <mrs@cygnus.com>
6# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
7# Modified by Yvan Roux <yvan.roux@linaro.org> and
8# Christophe Lyon <christophe.lyon@linaro.org>
9usage()
10{
11 if [ -n "$1" ] ; then
12 echo "$0: Error: $1" >&2
13 echo >&2
14 fi
15 cat >&2 <<EOUSAGE
Christophe Lyonca9d8302017-05-22 12:04:00 +000016Usage: $0 [-target target-triplet] [-pass-thresh pass-ratio-threshold] PREVIOUS CURRENT
Christophe Lyonbff39612015-11-18 16:29:11 +010017
18Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
19
Christophe Lyonf02cb6b2016-01-12 14:32:24 +010020 PREVIOUS and CURRENT must be directories, this tool finds and
21 compares any *.sum files they contain.
Christophe Lyonbff39612015-11-18 16:29:11 +010022
23 -target enables to provide the target name to use when parsing
24 the file containing the list of unstable tests.
25
Christophe Lyonca9d8302017-05-22 12:04:00 +000026 -pass-thresh controls the level of warning when too few tests
27 pass.
28
Christophe Lyonbff39612015-11-18 16:29:11 +010029 Exit with the following values:
30 0 if there is nothing of interest
Christophe Lyonf02cb6b2016-01-12 14:32:24 +010031 1 if there are improvements
32 2 if the are regressions or new errors
33 3 if there were build errors (no common logs)
34 4 if there are extra .sum files in either PREVIOUS or
35 CURRENT
Christophe Lyonbff39612015-11-18 16:29:11 +010036EOUSAGE
37 exit 2
38}
39
40export LC_ALL=C
41
42me="`which $0`"
43my_path="`dirname ${me}`"
44
45tool=gxx
46
47tmp1=/tmp/$tool-testing.$$a
48tmp2=/tmp/$tool-testing.$$b
49now_s=/tmp/$tool-testing.$$d
50before_s=/tmp/$tool-testing.$$e
51lst1=/tmp/$tool-lst1.$$
52lst2=/tmp/$tool-lst2.$$
53lst3=/tmp/$tool-lst3.$$
54lst4=/tmp/$tool-lst4.$$
55lst5=/tmp/$tool-lst5.$$
56sum1=/tmp/$tool-sum1.$$
57sum2=/tmp/$tool-sum2.$$
58tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
59
Christophe Lyonca9d8302017-05-22 12:04:00 +000060while [ $# -gt 2 ]
61do
62 case "$1" in
63 "-target")
64 target=$2
65 shift 2
66 ;;
67 "-pass-thresh")
68 pass_thresh=$2
69 shift 2
70 ;;
71 "-?")
72 usage
73 ;;
74 *)
75 if [ $# -gt 2 ]; then
76 echo "ERROR: Too many arguments: $@"
77 usage
78 fi
79 ;;
80 esac
81done
82
Christophe Lyonbff39612015-11-18 16:29:11 +010083[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
84
85trap "rm -f $tmps" 0 1 2 3 5 9 13 15
86exit_status=0
87
88if [ -d "$1" -a -d "$2" ] ; then
Yvan Roux8e7c7062016-01-25 17:36:51 +010089 find "$1/" \( -name '*.sum.xz' \)>$lst1
90 find "$2/" \( -name '*.sum.xz' \)>$lst2
Christophe Lyonbff39612015-11-18 16:29:11 +010091 echo "# Comparing directories"
92 echo "# REFERENCE: $1"
93 echo "# CURRENT: $2"
94 echo
95 # remove leading directory components to compare
96 sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
97 sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
98 comm -23 $lst3 $lst4 >$lst5
99 if [ -s $lst5 ] ; then
100 echo "# Extra sum files in Dir1=$1"
101 sed -e "s|^|< $1/|" $lst5
102 echo
Christophe Lyonf02cb6b2016-01-12 14:32:24 +0100103 exit_status=4
Christophe Lyonbff39612015-11-18 16:29:11 +0100104 fi
105 comm -13 $lst3 $lst4 >$lst5
106 if [ -s $lst5 ] ; then
107 echo "# Extra sum files in Dir2=$2"
108 sed -e "s|^|> $2/|" $lst5
109 echo
Christophe Lyonf02cb6b2016-01-12 14:32:24 +0100110 exit_status=4
Christophe Lyonbff39612015-11-18 16:29:11 +0100111 fi
112 comm -12 $lst3 $lst4 | sort -u >$lst5
113 if [ ! -s $lst5 ] ; then
114 echo "# No common sum files"
Christophe Lyonf02cb6b2016-01-12 14:32:24 +0100115 exit_status=3
Christophe Lyonbff39612015-11-18 16:29:11 +0100116 exit $exit_status
117 fi
118 cmnsums=`cat $lst5 | wc -l`
119 echo "# Comparing $cmnsums common sum files:"
120 cat $lst5
121 ( for fname in `cat $lst5`; do
122 bname=`basename $fname .sum.xz`
Christophe Lyona140aa82015-12-17 22:53:03 +0100123 xzcat $1/$fname | sed -e "s/^\([A-Z]*: \)/\1 $bname:/" | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/' \
124 | sed -e "s|^\([A-Z]*: \).*/home/.*/testsuite/|\1|"
Christophe Lyonbff39612015-11-18 16:29:11 +0100125 done ) >$sum1
126 ( for fname in `cat $lst5`; do
127 bname=`basename $fname .sum.xz`
Christophe Lyona140aa82015-12-17 22:53:03 +0100128 xzcat $2/$fname | sed -e "s/^\([A-Z]*: \)/\1 $bname:/" | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/' \
129 | sed -e "s|^\([A-Z]*: \).*/home/.*/testsuite/|\1|"
Christophe Lyonbff39612015-11-18 16:29:11 +0100130 done ) >$sum2
131 if [ "x${target}" != "x" ] ; then
132 unstable_target="--unstable-marker ${target}"
133 fi
134 for sum in $sum1 $sum2; do
135 board="$(grep "Running target " $sum | head -n 1 | sed -e "s/Running target //")"
136 if [ x"$board" != x"" ]; then
137 unstable_target="$unstable_target --unstable-marker $board"
138 fi
139 done
Christophe Lyonca9d8302017-05-22 12:04:00 +0000140 ${CONFIG_SHELL-/usr/bin/perl} ${my_path}/compare_dg_tests.pl ${pass_thresh:+-pass-thresh=${pass_thresh}} -l --unstable-test=${my_path}/unstable-tests.txt ${unstable_target} $sum1 $sum2
Christophe Lyonbff39612015-11-18 16:29:11 +0100141 ret=$?
Christophe Lyona8755e82015-11-20 00:40:34 +0100142 case $ret in
143 2)
Christophe Lyonf02cb6b2016-01-12 14:32:24 +0100144 [ $exit_status -eq 0 ] && exit_status=2
Christophe Lyonbff39612015-11-18 16:29:11 +0100145 echo "# Regressions found"
Christophe Lyona8755e82015-11-20 00:40:34 +0100146 ;;
147 1)
Christophe Lyonf02cb6b2016-01-12 14:32:24 +0100148 [ $exit_status -eq 0 ] && exit_status=1
Christophe Lyonbff39612015-11-18 16:29:11 +0100149 echo "# Improvements found"
Christophe Lyona8755e82015-11-20 00:40:34 +0100150 ;;
151 esac
152 if [ $ret -eq 2 ]; then
Christophe Lyonbff39612015-11-18 16:29:11 +0100153 echo "# Regressions in $cmnsums common sum files found"
154 else
155 echo "# No regression found in $cmnsums common sum files"
156 fi
157 exit $exit_status
158elif [ -d "$1" -o -d "$2" ] ; then
Christophe Lyonf02cb6b2016-01-12 14:32:24 +0100159 usage "Must specify two directories"
Christophe Lyonbff39612015-11-18 16:29:11 +0100160fi