blob: 824ba6586f615504835740e3347c4a40b1a85726 [file] [log] [blame]
Maxim Kuvyrkov63ad5352021-07-04 07:38:22 +00001#!/usr/bin/python3
Maxim Kuvyrkov59877482021-07-07 11:22:26 +00002
3# Script to compare testsuite failures against a list of known-to-fail
4# tests.
5#
Maxim Kuvyrkov59877482021-07-07 11:22:26 +00006# Contributed by Diego Novillo <dnovillo@google.com>
7#
8# Copyright (C) 2011-2013 Free Software Foundation, Inc.
9#
10# This file is part of GCC.
11#
12# GCC is free software; you can redistribute it and/or modify
13# it under the terms of the GNU General Public License as published by
14# the Free Software Foundation; either version 3, or (at your option)
15# any later version.
16#
17# GCC is distributed in the hope that it will be useful,
18# but WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20# GNU General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License
23# along with GCC; see the file COPYING. If not, write to
24# the Free Software Foundation, 51 Franklin Street, Fifth Floor,
25# Boston, MA 02110-1301, USA.
26
27"""This script provides a coarser XFAILing mechanism that requires no
28detailed DejaGNU markings. This is useful in a variety of scenarios:
29
30- Development branches with many known failures waiting to be fixed.
31- Release branches with known failures that are not considered
32 important for the particular release criteria used in that branch.
33
34The script must be executed from the toplevel build directory. When
35executed it will:
36
371- Determine the target built: TARGET
382- Determine the source directory: SRCDIR
393- Look for a failure manifest file in
40 <SRCDIR>/<MANIFEST_SUBDIR>/<MANIFEST_NAME>.xfail
414- Collect all the <tool>.sum files from the build tree.
425- Produce a report stating:
43 a- Failures expected in the manifest but not present in the build.
44 b- Failures in the build not expected in the manifest.
456- If all the build failures are expected in the manifest, it exits
46 with exit code 0. Otherwise, it exits with error code 1.
47
48Manifest files contain expected DejaGNU results that are otherwise
49treated as failures.
50They may also contain additional text:
51
52# This is a comment. - self explanatory
53@include file - the file is a path relative to the includer
54@remove result text - result text is removed from the expected set
55"""
56
57import datetime
58import optparse
59import os
60import re
61import sys
62
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +000063# Formats of .sum file sections
64_TOOL_LINE_FORMAT = '\t\t=== %s tests ===\n'
Christophe Lyona7d8c4c2023-04-14 12:01:23 +000065_EXP_LINE_FORMAT = '\nRunning %s:%s ...\n'
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +000066_SUMMARY_LINE_FORMAT = '\n\t\t=== %s Summary ===\n'
67
68# ... and their compiled regexs.
69_TOOL_LINE_REX = re.compile('^\t\t=== (.*) tests ===\n')
Christophe Lyona7d8c4c2023-04-14 12:01:23 +000070# Match .exp file name, optionally prefixed by a "tool:" name and a
71# path ending with "testsuite/"
72_EXP_LINE_REX = re.compile('^Running (?:.*:)?(?:.*/testsuite/)?(.*\.exp) \.\.\.\n')
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +000073_SUMMARY_LINE_REX = re.compile('^\t\t=== (.*) Summary ===\n')
74
Maxim Kuvyrkov59877482021-07-07 11:22:26 +000075# Subdirectory of srcdir in which to find the manifest file.
76_MANIFEST_SUBDIR = 'contrib/testsuite-management'
77
78# Pattern for naming manifest files.
79# The first argument should be the toplevel GCC(/GNU tool) source directory.
80# The second argument is the manifest subdir.
81# The third argument is the manifest target, which defaults to the target
82# triplet used during the build.
83_MANIFEST_PATH_PATTERN = '%s/%s/%s.xfail'
84
85# The options passed to the program.
86_OPTIONS = None
87
88def Error(msg):
Maxim Kuvyrkov63ad5352021-07-04 07:38:22 +000089 print('error: %s' % msg, file=sys.stderr)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +000090 sys.exit(1)
91
92
93class TestResult(object):
94 """Describes a single DejaGNU test result as emitted in .sum files.
95
96 We are only interested in representing unsuccessful tests. So, only
97 a subset of all the tests are loaded.
98
99 The summary line used to build the test result should have this format:
100
101 attrlist | XPASS: gcc.dg/unroll_1.c (test for excess errors)
102 ^^^^^^^^ ^^^^^ ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^
103 optional state name description
104 attributes
105
106 Attributes:
107 attrlist: A comma separated list of attributes.
108 Valid values:
109 flaky Indicates that this test may not always fail. These
110 tests are reported, but their presence does not affect
111 the results.
112
113 expire=YYYYMMDD After this date, this test will produce an error
114 whether it is in the manifest or not.
115
116 state: One of UNRESOLVED, XPASS or FAIL.
117 name: File name for the test.
118 description: String describing the test (flags used, dejagnu message, etc)
119 ordinal: Monotonically increasing integer.
120 It is used to keep results for one .exp file sorted
121 by the order the tests were run.
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000122 tool: Top-level testsuite name (aka "tool" in DejaGnu parlance) of the test.
123 exp: Name of .exp testsuite file.
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000124 """
125
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000126 def __init__(self, summary_line, ordinal, tool, exp):
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000127 try:
128 (self.attrs, summary_line) = SplitAttributesFromSummaryLine(summary_line)
129 try:
130 (self.state,
131 self.name,
Maxim Kuvyrkov56734ff2021-08-30 14:24:25 +0000132 self.description) = _VALID_TEST_RESULTS_REX.match(summary_line).groups()
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000133 except:
Thiago Jung Bauermann7b82a592023-04-22 14:12:06 +0000134 print('Failed to parse summary line: "%s"' % summary_line,
135 file=sys.stderr)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000136 raise
137 self.ordinal = ordinal
Maxim Kuvyrkovf09ab0e2021-08-30 14:19:04 +0000138 if tool == None or exp == None:
139 # .sum file seem to be broken. There was no "tool" and/or "exp"
140 # lines preceding this result.
Thiago Jung Bauermann7b82a592023-04-22 14:12:06 +0000141 print(f'.sum file seems to be broken: tool="{tool}", exp="{exp}", summary_line="{summary_line}"',
142 file=sys.stderr)
Maxim Kuvyrkovf09ab0e2021-08-30 14:19:04 +0000143 raise
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000144 self.tool = tool
145 self.exp = exp
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000146 except ValueError:
147 Error('Cannot parse summary line "%s"' % summary_line)
148
149 if self.state not in _VALID_TEST_RESULTS:
150 Error('Invalid test result %s in "%s" (parsed as "%s")' % (
151 self.state, summary_line, self))
152
153 def __lt__(self, other):
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000154 if (self.tool != other.tool):
155 return self.tool < other.tool
156 if (self.exp != other.exp):
157 return self.exp < other.exp
158 if (self.name != other.name):
159 return self.name < other.name
160 return self.ordinal < other.ordinal
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000161
162 def __hash__(self):
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000163 return (hash(self.state) ^ hash(self.tool) ^ hash(self.exp)
164 ^ hash(self.name) ^ hash(self.description))
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000165
166 def __eq__(self, other):
167 return (self.state == other.state and
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000168 self.tool == other.tool and
169 self.exp == other.exp and
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000170 self.name == other.name and
171 self.description == other.description)
172
173 def __ne__(self, other):
174 return not (self == other)
175
176 def __str__(self):
177 attrs = ''
178 if self.attrs:
179 attrs = '%s | ' % self.attrs
180 return '%s%s: %s %s' % (attrs, self.state, self.name, self.description)
181
182 def ExpirationDate(self):
183 # Return a datetime.date object with the expiration date for this
184 # test result. Return None, if no expiration has been set.
185 if re.search(r'expire=', self.attrs):
186 expiration = re.search(r'expire=(\d\d\d\d)(\d\d)(\d\d)', self.attrs)
187 if not expiration:
188 Error('Invalid expire= format in "%s". Must be of the form '
189 '"expire=YYYYMMDD"' % self)
190 return datetime.date(int(expiration.group(1)),
191 int(expiration.group(2)),
192 int(expiration.group(3)))
193 return None
194
195 def HasExpired(self):
196 # Return True if the expiration date of this result has passed.
197 expiration_date = self.ExpirationDate()
198 if expiration_date:
199 now = datetime.date.today()
200 return now > expiration_date
201
202
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000203class ResultSet(set):
204 """Describes a set of DejaGNU test results.
205 This set can be read in from .sum files or emitted as a manifest.
206
207 Attributes:
208 current_tool: Name of the current top-level DejaGnu testsuite.
209 current_exp: Name of the current .exp testsuite file.
210 """
211
212 def __init__(self):
213 super().__init__()
214 self.ResetToolExp()
215
216 def ResetToolExp(self):
217 self.current_tool = None
218 self.current_exp = None
219
220 def MakeTestResult(self, summary_line, ordinal=-1):
221 return TestResult(summary_line, ordinal,
222 self.current_tool, self.current_exp)
223
224 def Print(self, outfile=sys.stdout):
225 current_tool = None
226 current_exp = None
227
228 for result in sorted(self):
229 if current_tool != result.tool:
230 current_tool = result.tool
231 outfile.write(_TOOL_LINE_FORMAT % current_tool)
232 if current_exp != result.exp:
233 current_exp = result.exp
Christophe Lyona7d8c4c2023-04-14 12:01:23 +0000234 outfile.write(_EXP_LINE_FORMAT % (current_tool, current_exp))
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000235 outfile.write('%s\n' % result)
236
237 outfile.write(_SUMMARY_LINE_FORMAT % 'Results')
238
239
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000240def GetMakefileValue(makefile_name, value_name):
241 if os.path.exists(makefile_name):
242 makefile = open(makefile_name)
243 for line in makefile:
244 if line.startswith(value_name):
245 (_, value) = line.split('=', 1)
246 value = value.strip()
247 makefile.close()
248 return value
249 makefile.close()
250 return None
251
252
253def ValidBuildDirectory(builddir):
254 if (not os.path.exists(builddir) or
255 not os.path.exists('%s/Makefile' % builddir)):
256 return False
257 return True
258
259
260def IsComment(line):
261 """Return True if line is a comment."""
262 return line.startswith('#')
263
264
265def SplitAttributesFromSummaryLine(line):
266 """Splits off attributes from a summary line, if present."""
267 if '|' in line and not _VALID_TEST_RESULTS_REX.match(line):
268 (attrs, line) = line.split('|', 1)
269 attrs = attrs.strip()
270 else:
271 attrs = ''
272 line = line.strip()
273 return (attrs, line)
274
275
276def IsInterestingResult(line):
277 """Return True if line is one of the summary lines we care about."""
278 (_, line) = SplitAttributesFromSummaryLine(line)
279 return bool(_VALID_TEST_RESULTS_REX.match(line))
280
281
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000282def IsToolLine(line):
283 """Return True if line mentions the tool (in DejaGnu terms) for the following tests."""
284 return bool(_TOOL_LINE_REX.match(line))
285
286
287def IsExpLine(line):
288 """Return True if line mentions the .exp file for the following tests."""
289 return bool(_EXP_LINE_REX.match(line))
290
291
292def IsSummaryLine(line):
293 """Return True if line starts .sum footer."""
294 return bool(_SUMMARY_LINE_REX.match(line))
295
296
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000297def IsInclude(line):
298 """Return True if line is an include of another file."""
299 return line.startswith("@include ")
300
301
302def GetIncludeFile(line, includer):
303 """Extract the name of the include file from line."""
304 includer_dir = os.path.dirname(includer)
305 include_file = line[len("@include "):]
306 return os.path.join(includer_dir, include_file.strip())
307
308
309def IsNegativeResult(line):
310 """Return True if line should be removed from the expected results."""
311 return line.startswith("@remove ")
312
313
314def GetNegativeResult(line):
315 """Extract the name of the negative result from line."""
316 line = line[len("@remove "):]
317 return line.strip()
318
319
320def ParseManifestWorker(result_set, manifest_path):
321 """Read manifest_path, adding the contents to result_set."""
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000322 if _OPTIONS.verbosity >= 5:
Maxim Kuvyrkov63ad5352021-07-04 07:38:22 +0000323 print('Parsing manifest file %s.' % manifest_path)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000324 manifest_file = open(manifest_path)
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000325 for orig_line in manifest_file:
326 line = orig_line.strip()
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000327 if line == "":
328 pass
329 elif IsComment(line):
330 pass
331 elif IsNegativeResult(line):
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000332 result_set.remove(result_set.MakeTestResult(GetNegativeResult(line)))
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000333 elif IsInclude(line):
334 ParseManifestWorker(result_set, GetIncludeFile(line, manifest_path))
335 elif IsInterestingResult(line):
Maxim Kuvyrkova6b29dd2023-04-12 14:35:39 +0000336 result = result_set.MakeTestResult(line)
337 if result.HasExpired():
338 # Ignore expired manifest entries.
339 if _OPTIONS.verbosity >= 3:
340 print('WARNING: Expected failure "%s" has expired.' % line.strip())
341 continue
342 result_set.add(result)
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000343 elif IsExpLine(orig_line):
344 result_set.current_exp = _EXP_LINE_REX.match(orig_line).groups()[0]
345 elif IsToolLine(orig_line):
346 result_set.current_tool = _TOOL_LINE_REX.match(orig_line).groups()[0]
347 elif IsSummaryLine(orig_line):
348 result_set.ResetToolExp()
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000349 else:
350 Error('Unrecognized line in manifest file: %s' % line)
351 manifest_file.close()
352
353
354def ParseManifest(manifest_path):
355 """Create a set of TestResult instances from the given manifest file."""
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000356 result_set = ResultSet()
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000357 ParseManifestWorker(result_set, manifest_path)
358 return result_set
359
360
361def ParseSummary(sum_fname):
362 """Create a set of TestResult instances from the given summary file."""
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000363 result_set = ResultSet()
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000364 # ordinal is used when sorting the results so that tests within each
365 # .exp file are kept sorted.
366 ordinal=0
367 sum_file = open(sum_fname)
368 for line in sum_file:
369 if IsInterestingResult(line):
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000370 result = result_set.MakeTestResult(line, ordinal)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000371 ordinal += 1
372 if result.HasExpired():
Maxim Kuvyrkova6b29dd2023-04-12 14:35:39 +0000373 # ??? What is the use-case for this? How "expiry" annotations are
374 # ??? supposed to be added to .sum results?
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000375 # Tests that have expired are not added to the set of expected
376 # results. If they are still present in the set of actual results,
377 # they will cause an error to be reported.
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000378 if _OPTIONS.verbosity >= 3:
379 print('WARNING: Expected failure "%s" has expired.' % line.strip())
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000380 continue
381 result_set.add(result)
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000382 elif IsExpLine(line):
383 result_set.current_exp = _EXP_LINE_REX.match(line).groups()[0]
384 elif IsToolLine(line):
385 result_set.current_tool = _TOOL_LINE_REX.match(line).groups()[0]
Maxim Kuvyrkovd8951a22021-07-08 08:20:28 +0000386 result_set.current_exp = None
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000387 elif IsSummaryLine(line):
388 result_set.ResetToolExp()
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000389 sum_file.close()
390 return result_set
391
392
393def GetManifest(manifest_path):
394 """Build a set of expected failures from the manifest file.
395
396 Each entry in the manifest file should have the format understood
397 by the TestResult constructor.
398
399 If no manifest file exists for this target, it returns an empty set.
400 """
401 if os.path.exists(manifest_path):
402 return ParseManifest(manifest_path)
403 else:
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000404 return ResultSet()
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000405
406
407def CollectSumFiles(builddir):
408 sum_files = []
409 for root, dirs, files in os.walk(builddir):
410 for ignored in ('.svn', '.git'):
411 if ignored in dirs:
412 dirs.remove(ignored)
413 for fname in files:
414 if fname.endswith('.sum'):
415 sum_files.append(os.path.join(root, fname))
416 return sum_files
417
418
Maxim Kuvyrkov8ef7c852021-07-08 08:21:18 +0000419def GetResults(sum_files, build_results = None):
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000420 """Collect all the test results from the given .sum files."""
Maxim Kuvyrkov8ef7c852021-07-08 08:21:18 +0000421 if build_results == None:
422 build_results = ResultSet()
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000423 for sum_fname in sum_files:
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000424 if _OPTIONS.verbosity >= 3:
425 print('\t%s' % sum_fname)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000426 build_results |= ParseSummary(sum_fname)
427 return build_results
428
429
430def CompareResults(manifest, actual):
431 """Compare sets of results and return two lists:
432 - List of results present in ACTUAL but missing from MANIFEST.
433 - List of results present in MANIFEST but missing from ACTUAL.
434 """
435 # Collect all the actual results not present in the manifest.
436 # Results in this set will be reported as errors.
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000437 actual_vs_manifest = ResultSet()
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000438 for actual_result in actual:
439 if actual_result not in manifest:
440 actual_vs_manifest.add(actual_result)
441
442 # Collect all the tests in the manifest that were not found
443 # in the actual results.
444 # Results in this set will be reported as warnings (since
445 # they are expected failures that are not failing anymore).
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000446 manifest_vs_actual = ResultSet()
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000447 for expected_result in manifest:
448 # Ignore tests marked flaky.
449 if 'flaky' in expected_result.attrs:
450 continue
451 if expected_result not in actual:
452 manifest_vs_actual.add(expected_result)
453
454 return actual_vs_manifest, manifest_vs_actual
455
456
Maxim Kuvyrkov918bc262021-07-08 08:27:39 +0000457def GetManifestPath(user_provided_must_exist):
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000458 """Return the full path to the manifest file."""
459 manifest_path = _OPTIONS.manifest
460 if manifest_path:
461 if user_provided_must_exist and not os.path.exists(manifest_path):
462 Error('Manifest does not exist: %s' % manifest_path)
463 return manifest_path
464 else:
Maxim Kuvyrkov918bc262021-07-08 08:27:39 +0000465 (srcdir, target) = GetBuildData()
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000466 if not srcdir:
467 Error('Could not determine the location of GCC\'s source tree. '
468 'The Makefile does not contain a definition for "srcdir".')
469 if not target:
470 Error('Could not determine the target triplet for this build. '
471 'The Makefile does not contain a definition for "target_alias".')
472 return _MANIFEST_PATH_PATTERN % (srcdir, _MANIFEST_SUBDIR, target)
473
474
475def GetBuildData():
476 if not ValidBuildDirectory(_OPTIONS.build_dir):
477 # If we have been given a set of results to use, we may
478 # not be inside a valid GCC build directory. In that case,
479 # the user must provide both a manifest file and a set
480 # of results to check against it.
481 if not _OPTIONS.results or not _OPTIONS.manifest:
482 Error('%s is not a valid GCC top level build directory. '
483 'You must use --manifest and --results to do the validation.' %
484 _OPTIONS.build_dir)
485 else:
486 return None, None
487 srcdir = GetMakefileValue('%s/Makefile' % _OPTIONS.build_dir, 'srcdir =')
488 target = GetMakefileValue('%s/Makefile' % _OPTIONS.build_dir, 'target_alias=')
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000489 if _OPTIONS.verbosity >= 3:
490 print('Source directory: %s' % srcdir)
491 print('Build target: %s' % target)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000492 return srcdir, target
493
494
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000495def PrintSummary(summary):
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000496 summary.Print()
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000497
498def GetSumFiles(results, build_dir):
499 if not results:
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000500 if _OPTIONS.verbosity >= 3:
501 print('Getting actual results from build directory %s' % build_dir)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000502 sum_files = CollectSumFiles(build_dir)
503 else:
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000504 if _OPTIONS.verbosity >= 3:
505 print('Getting actual results from user-provided results')
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000506 sum_files = results.split()
507 return sum_files
508
509
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000510def PerformComparison(expected, actual):
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000511 actual_vs_expected, expected_vs_actual = CompareResults(expected, actual)
512
513 tests_ok = True
514 if len(actual_vs_expected) > 0:
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000515 if _OPTIONS.verbosity >= 3:
516 print('\n\nUnexpected results in this build (new failures)')
517 if _OPTIONS.verbosity >= 1:
518 PrintSummary(actual_vs_expected)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000519 tests_ok = False
520
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000521 if _OPTIONS.verbosity >= 2 and len(expected_vs_actual) > 0:
522 print('\n\nExpected results not present in this build (fixed tests)'
523 '\n\nNOTE: This is not a failure. It just means that these '
524 'tests were expected\nto fail, but either they worked in '
525 'this configuration or they were not\npresent at all.\n')
526 PrintSummary(expected_vs_actual)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000527
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000528 if tests_ok and _OPTIONS.verbosity >= 3:
Maxim Kuvyrkov63ad5352021-07-04 07:38:22 +0000529 print('\nSUCCESS: No unexpected failures.')
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000530
531 return tests_ok
532
533
534def CheckExpectedResults():
Maxim Kuvyrkov918bc262021-07-08 08:27:39 +0000535 manifest_path = GetManifestPath(True)
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000536 if _OPTIONS.verbosity >= 3:
537 print('Manifest: %s' % manifest_path)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000538 manifest = GetManifest(manifest_path)
539 sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir)
540 actual = GetResults(sum_files)
541
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000542 if _OPTIONS.verbosity >= 5:
543 print('\n\nTests expected to fail')
544 PrintSummary(manifest)
545 print('\n\nActual test results')
546 PrintSummary(actual)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000547
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000548 return PerformComparison(manifest, actual)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000549
550
551def ProduceManifest():
Maxim Kuvyrkov918bc262021-07-08 08:27:39 +0000552 manifest_path = GetManifestPath(False)
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000553 if _OPTIONS.verbosity >= 3:
554 print('Manifest: %s' % manifest_path)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000555 if os.path.exists(manifest_path) and not _OPTIONS.force:
556 Error('Manifest file %s already exists.\nUse --force to overwrite.' %
557 manifest_path)
558
559 sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir)
560 actual = GetResults(sum_files)
561 manifest_file = open(manifest_path, 'w')
Maxim Kuvyrkov51e3fa12021-07-04 10:58:53 +0000562 actual.Print(manifest_file)
563 actual.Print()
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000564 manifest_file.close()
565
566 return True
567
568
569def CompareBuilds():
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000570 sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir)
571 actual = GetResults(sum_files)
572
Maxim Kuvyrkov8ef7c852021-07-08 08:21:18 +0000573 clean = ResultSet()
574
575 if _OPTIONS.manifest:
Maxim Kuvyrkov918bc262021-07-08 08:27:39 +0000576 manifest_path = GetManifestPath(True)
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000577 if _OPTIONS.verbosity >= 3:
578 print('Manifest: %s' % manifest_path)
Maxim Kuvyrkov8ef7c852021-07-08 08:21:18 +0000579 clean = GetManifest(manifest_path)
580
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000581 clean_sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.clean_build)
Maxim Kuvyrkov8ef7c852021-07-08 08:21:18 +0000582 clean = GetResults(clean_sum_files, clean)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000583
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000584 return PerformComparison(clean, actual)
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000585
586
587def Main(argv):
588 parser = optparse.OptionParser(usage=__doc__)
589
590 # Keep the following list sorted by option name.
591 parser.add_option('--build_dir', action='store', type='string',
592 dest='build_dir', default='.',
593 help='Build directory to check (default = .)')
594 parser.add_option('--clean_build', action='store', type='string',
595 dest='clean_build', default=None,
596 help='Compare test results from this build against '
597 'those of another (clean) build. Use this option '
598 'when comparing the test results of your patch versus '
599 'the test results of a clean build without your patch. '
600 'You must provide the path to the top directory of your '
601 'clean build.')
602 parser.add_option('--force', action='store_true', dest='force',
603 default=False, help='When used with --produce_manifest, '
604 'it will overwrite an existing manifest file '
605 '(default = False)')
Maxim Kuvyrkova6df4d22021-07-13 09:27:35 +0000606 parser.add_option('--ignore_ERRORs', action='store_true',
607 dest='ignore_ERRORs', default=False,
608 help='Ignore DejaGnu "ERROR: foo" results '
609 '(default = False)')
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000610 parser.add_option('--manifest', action='store', type='string',
611 dest='manifest', default=None,
612 help='Name of the manifest file to use (default = '
613 'taken from '
614 'contrib/testsuite-managment/<target_alias>.xfail)')
615 parser.add_option('--produce_manifest', action='store_true',
616 dest='produce_manifest', default=False,
617 help='Produce the manifest for the current '
618 'build (default = False)')
619 parser.add_option('--results', action='store', type='string',
620 dest='results', default=None, help='Space-separated list '
621 'of .sum files with the testing results to check. The '
622 'only content needed from these files are the lines '
623 'starting with FAIL, XPASS or UNRESOLVED (default = '
624 '.sum files collected from the build directory).')
625 parser.add_option('--verbosity', action='store', dest='verbosity',
Maxim Kuvyrkov40205382021-07-12 15:41:47 +0000626 type='int', default=3, help='Verbosity level '
627 '(default = 3). Level 0: only error output, this is '
628 'useful in scripting when only the exit code is used. '
629 'Level 1: output unexpected failures. '
630 'Level 2: output unexpected passes. '
631 'Level 3: output helpful information. '
632 'Level 5: output debug information.')
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000633 global _OPTIONS
634 (_OPTIONS, _) = parser.parse_args(argv[1:])
635
Maxim Kuvyrkova6df4d22021-07-13 09:27:35 +0000636 # Handled test results.
637 global _VALID_TEST_RESULTS
638 global _VALID_TEST_RESULTS_REX
639 _VALID_TEST_RESULTS = [ 'FAIL', 'UNRESOLVED', 'XPASS' ]
640 if not _OPTIONS.ignore_ERRORs:
641 _VALID_TEST_RESULTS.append('ERROR')
Maxim Kuvyrkov56734ff2021-08-30 14:24:25 +0000642
643 valid_results_regex = "|".join(_VALID_TEST_RESULTS)
644 valid_results_regex = '(%s):\s*(\S+)\s*(.*)' % valid_results_regex
645 _VALID_TEST_RESULTS_REX = re.compile(valid_results_regex)
Maxim Kuvyrkova6df4d22021-07-13 09:27:35 +0000646
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000647 if _OPTIONS.produce_manifest:
648 retval = ProduceManifest()
649 elif _OPTIONS.clean_build:
650 retval = CompareBuilds()
651 else:
652 retval = CheckExpectedResults()
653
654 if retval:
655 return 0
656 else:
Maxim Kuvyrkov972bb812021-08-30 14:18:09 +0000657 return 2
Maxim Kuvyrkov59877482021-07-07 11:22:26 +0000658
659
660if __name__ == '__main__':
661 retval = Main(sys.argv)
662 sys.exit(retval)