[WIP] Summary handling
Change-Id: Idc1862969059f04a1e3932c90abff8dbf01a7a55
diff --git a/contrib/testsuite-management/validate_failures.py b/contrib/testsuite-management/validate_failures.py
index 05c6308..742ab71 100755
--- a/contrib/testsuite-management/validate_failures.py
+++ b/contrib/testsuite-management/validate_failures.py
@@ -61,8 +61,9 @@
import sys
# Handled test results.
-_VALID_TEST_RESULTS = [ 'FAIL', 'UNRESOLVED', 'XPASS', 'ERROR' ]
-_VALID_TEST_RESULTS_REX = re.compile("%s" % "|".join(_VALID_TEST_RESULTS))
+_INTERESTING_TEST_RESULTS = [ 'FAIL', 'UNRESOLVED', 'XPASS', 'ERROR' ]
+_INTERESTING_TEST_RESULTS_REX = re.compile("%s" %
+ "|".join(_INTERESTING_TEST_RESULTS))
# Formats of .sum file sections
_TOOL_LINE_FORMAT = '\t\t=== %s tests ===\n'
@@ -74,6 +75,15 @@
_EXP_LINE_REX = re.compile('^Running .*(?:/testsuite/)?(.*\.exp) \.\.\.\n')
_SUMMARY_LINE_REX = re.compile('^\t\t=== (.*) Summary ===\n')
+_STATE_ID_TO_PRETTY_STATE = {
+ 'expected passes' : 'PASS',
+ 'unexpected failures' : 'FAIL',
+ 'unexpected successes' : 'XPASS',
+ 'expected failures' : 'XFAIL',
+ 'unresolved testcases' : 'UNRESOLVED',
+ 'unsupported tests' : 'UNSUPPORTED'
+}
+
# Subdirectory of srcdir in which to find the manifest file.
_MANIFEST_SUBDIR = 'contrib/testsuite-management'
@@ -142,7 +152,7 @@
except ValueError:
Error('Cannot parse summary line "%s"' % summary_line)
- if self.state not in _VALID_TEST_RESULTS:
+ if self.state not in _INTERESTING_TEST_RESULTS:
Error('Invalid test result %s in "%s" (parsed as "%s")' % (
self.state, summary_line, self))
@@ -196,6 +206,41 @@
return now > expiration_date
+class ResultStats(dict):
+ def __init__(self, tool_line):
+ super().__init__()
+
+ def Count(self, line):
+ state = re.match(r'([A-Z]+):\s*(\S+)\s*(.*)', line).groups()[0]
+ if not state in self:
+ self[state] = 0
+ self[state] += 1
+
+ def IdOfPrettyState(pretty_state):
+ for i in _STATE_ID_TO
+ if pretty_state in _PRETTY_STATE_TO_ID:
+ return _PRETTY_STATE_TO_ID[pretty_state]
+ Error('Cannot parse pretty state %s' % pretty_state)
+
+ def PrettyState(state):
+ if state in _STATE_ID_TO_PRETTY_STATE:
+ return _STATE_ID_TO_PRETTY_STATE[state]
+ return state
+
+ def Parse(sum_line, infile):
+ stats = ResultStats(sum_line)
+ for orig_line in infile:
+ line = orig_line.strip()
+ if line == '':
+ pass
+ elif isStatLine(orig_line):
+ (state_desc, count) = re.match(r'', orig_line).groups()
+ state = desc2state(state_desc)
+ self[state] = count
+ else
+ return stats
+
+
class ResultSet(set):
"""Describes a set of DejaGNU test results.
This set can be read in from .sum files or emitted as a manifest.
@@ -208,6 +253,7 @@
def __init__(self):
super().__init__()
self.ResetToolExp()
+ self.stats = ResultStats()
def ResetToolExp(self):
self.current_tool = None
@@ -260,7 +306,7 @@
def SplitAttributesFromSummaryLine(line):
"""Splits off attributes from a summary line, if present."""
- if '|' in line and not _VALID_TEST_RESULTS_REX.match(line):
+ if '|' in line and not _INTERESTING_TEST_RESULTS_REX.match(line):
(attrs, line) = line.split('|', 1)
attrs = attrs.strip()
else:
@@ -272,7 +318,12 @@
def IsInterestingResult(line):
"""Return True if line is one of the summary lines we care about."""
(_, line) = SplitAttributesFromSummaryLine(line)
- return bool(_VALID_TEST_RESULTS_REX.match(line))
+ return bool(_INTERESTING_TEST_RESULTS_REX.match(line))
+
+
+def IsBoringResult(line):
+ """Return True if line is a result that should be included in stats."""
+ return bool(_BORING_TEST_RESULTS_REX.match(line))
def IsToolLine(line):
@@ -330,12 +381,16 @@
ParseManifestWorker(result_set, GetIncludeFile(line, manifest_path))
elif IsInterestingResult(line):
result_set.add(result_set.MakeTestResult(line))
+ elif IsBoringResult(line):
+ pass
elif IsExpLine(orig_line):
result_set.current_exp = _EXP_LINE_REX.match(orig_line).groups()[0]
elif IsToolLine(orig_line):
result_set.current_tool = _TOOL_LINE_REX.match(orig_line).groups()[0]
elif IsSummaryLine(orig_line):
result_set.ResetToolExp()
+ manifest_stats = ParseResultStats(line, manifest_file)
+ result_set.stats[result_set.current_tool] += manifest_stats
else:
Error('Unrecognized line in manifest file: %s' % line)
manifest_file.close()
@@ -354,9 +409,11 @@
# ordinal is used when sorting the results so that tests within each
# .exp file are kept sorted.
ordinal=0
+ current_tool_stats = None
sum_file = open(sum_fname)
for line in sum_file:
if IsInterestingResult(line):
+ current_tool_stats.Count(line)
result = result_set.MakeTestResult(line, ordinal)
ordinal += 1
if result.HasExpired():
@@ -366,12 +423,20 @@
print('WARNING: Expected failure "%s" has expired.' % line.strip())
continue
result_set.add(result)
+ elif IsBoringResult(line):
+ current_tool_stats.Count(line)
elif IsExpLine(line):
result_set.current_exp = _EXP_LINE_REX.match(line).groups()[0]
elif IsToolLine(line):
result_set.current_tool = _TOOL_LINE_REX.match(line).groups()[0]
+ current_tool_stats = ResultStats(results_set.current_tool)
elif IsSummaryLine(line):
result_set.ResetToolExp()
+ sum_stats = ResultStats.Parse(line, sum_file)
+ if (current_tool_stats != sum_stats):
+ Error('Calculated and parsed statistics do not match')
+ results_set.stats[results_set.current_tool] += current_tool_stats
+
sum_file.close()
return result_set