blob: 39f74a323e930e477fe1d6e9d22f834ebbfdf76a (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
#!/bin/sh
# This script automatically test the given tool with the tool's test cases,
# reporting anything of interest.
# Written by Mike Stump <mrs@cygnus.com>
# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
# Modified by Yvan Roux <yvan.roux@linaro.org> and
# Christophe Lyon <christophe.lyon@linaro.org>
usage()
{
if [ -n "$1" ] ; then
echo "$0: Error: $1" >&2
echo >&2
fi
cat >&2 <<EOUSAGE
Usage: $0 [-target target-triplet] [-pass-thresh pass-ratio-threshold] PREVIOUS CURRENT
Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
PREVIOUS and CURRENT must be directories, this tool finds and
compares any *.sum files they contain.
-target enables to provide the target name to use when parsing
the file containing the list of unstable tests.
-pass-thresh controls the level of warning when too few tests
pass.
Exit with the following values:
0 if there is nothing of interest
1 if there are improvements
2 if the are regressions or new errors
3 if there were build errors (no common logs)
4 if there are extra .sum files in either PREVIOUS or
CURRENT
EOUSAGE
exit 2
}
export LC_ALL=C
me="`which $0`"
my_path="`dirname ${me}`"
tool=gxx
tmp1=/tmp/$tool-testing.$$a
tmp2=/tmp/$tool-testing.$$b
now_s=/tmp/$tool-testing.$$d
before_s=/tmp/$tool-testing.$$e
lst1=/tmp/$tool-lst1.$$
lst2=/tmp/$tool-lst2.$$
lst3=/tmp/$tool-lst3.$$
lst4=/tmp/$tool-lst4.$$
lst5=/tmp/$tool-lst5.$$
sum1=/tmp/$tool-sum1.$$
sum2=/tmp/$tool-sum2.$$
tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
while [ $# -gt 2 ]
do
case "$1" in
"-target")
target=$2
shift 2
;;
"-pass-thresh")
pass_thresh=$2
shift 2
;;
"-?")
usage
;;
*)
if [ $# -gt 2 ]; then
echo "ERROR: Too many arguments: $@"
usage
fi
;;
esac
done
[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
trap "rm -f $tmps" 0 1 2 3 5 9 13 15
exit_status=0
if [ -d "$1" -a -d "$2" ] ; then
find "$1/" \( -name '*.sum.xz' \)>$lst1
find "$2/" \( -name '*.sum.xz' \)>$lst2
echo "# Comparing directories"
echo "# REFERENCE: $1"
echo "# CURRENT: $2"
echo
# remove leading directory components to compare
sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
comm -23 $lst3 $lst4 >$lst5
if [ -s $lst5 ] ; then
echo "# Extra sum files in Dir1=$1"
sed -e "s|^|< $1/|" $lst5
echo
exit_status=4
fi
comm -13 $lst3 $lst4 >$lst5
if [ -s $lst5 ] ; then
echo "# Extra sum files in Dir2=$2"
sed -e "s|^|> $2/|" $lst5
echo
exit_status=4
fi
comm -12 $lst3 $lst4 | sort -u >$lst5
if [ ! -s $lst5 ] ; then
echo "# No common sum files"
exit_status=3
exit $exit_status
fi
cmnsums=`cat $lst5 | wc -l`
echo "# Comparing $cmnsums common sum files:"
cat $lst5
( for fname in `cat $lst5`; do
bname=`basename $fname .sum.xz`
xzcat $1/$fname | sed -e "s/^\([A-Z]*: \)/\1 $bname:/" | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/' \
| sed -e "s|^\([A-Z]*: \).*/home/.*/testsuite/|\1|"
done ) >$sum1
( for fname in `cat $lst5`; do
bname=`basename $fname .sum.xz`
xzcat $2/$fname | sed -e "s/^\([A-Z]*: \)/\1 $bname:/" | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/' \
| sed -e "s|^\([A-Z]*: \).*/home/.*/testsuite/|\1|"
done ) >$sum2
if [ "x${target}" != "x" ] ; then
unstable_target="--unstable-marker ${target}"
fi
for sum in $sum1 $sum2; do
board="$(grep "Running target " $sum | head -n 1 | sed -e "s/Running target //")"
if [ x"$board" != x"" ]; then
unstable_target="$unstable_target --unstable-marker $board"
fi
done
${CONFIG_SHELL-/usr/bin/perl} ${my_path}/compare_dg_tests.pl ${pass_thresh:+-pass-thresh=${pass_thresh}} -l --unstable-test=${my_path}/unstable-tests.txt ${unstable_target} $sum1 $sum2
ret=$?
case $ret in
2)
[ $exit_status -eq 0 ] && exit_status=2
echo "# Regressions found"
;;
1)
[ $exit_status -eq 0 ] && exit_status=1
echo "# Improvements found"
;;
esac
if [ $ret -eq 2 ]; then
echo "# Regressions in $cmnsums common sum files found"
else
echo "# No regression found in $cmnsums common sum files"
fi
exit $exit_status
elif [ -d "$1" -o -d "$2" ] ; then
usage "Must specify two directories"
fi
|
