[contrib] validate_failures.py: Support json results
... from as produced by llvm-lit on llvm-test-suite.
This is a very light-weight implementation to start experimenting
with LLVM test results.
Change-Id: I147af69e4fad95498fc6cc72552c340e374580f6
diff --git a/contrib/testsuite-management/validate_failures.py b/contrib/testsuite-management/validate_failures.py
index aa450c1..7e163df 100755
--- a/contrib/testsuite-management/validate_failures.py
+++ b/contrib/testsuite-management/validate_failures.py
@@ -59,8 +59,11 @@
import os
import re
import sys
+import json
-_INTERESTING_RESULTS = [ 'FAIL', 'UNRESOLVED', 'XPASS', 'ERROR' ]
+# Results that we want keep an eye on.
+# Note, 'NOEXE' state is used by llvm-test-suite.
+_INTERESTING_RESULTS = [ 'FAIL', 'UNRESOLVED', 'XPASS', 'ERROR', 'NOEXE' ]
# <STATE>: <NAME> <DESCRIPTION"
_INTERESTING_RESULTS_REX = re.compile('(%s):\s*(\S+)\s*(.*)'
% "|".join(_INTERESTING_RESULTS))
@@ -410,14 +413,39 @@
return result_set
+def convertJSONResult(json_result):
+ """Convert JSON result into a dejagnu-like line."""
+ # We are just starting to process JSON data (from LLVM's testsuite),
+ # and it seems prudent to convert JSON into dejagnu-like lines, so that we
+ # can re-use dejagnu parsing logic, rather than duplicating it for JSON.
+
+ name = json_result['name']
+ tool_exp = name.split(' :: ')
+ tool_exp[1] = os.path.dirname(tool_exp[1])
+ line = json_result['code'] + ': ' + json_result['name']
+ tool_exp.append(line)
+ return tool_exp
+
def ParseSummary(sum_fname):
"""Create a set of TestResult instances from the given summary file."""
result_set = ResultSet()
# ordinal is used when sorting the results so that tests within each
# .exp file are kept sorted.
ordinal=0
+
sum_file = open(sum_fname, encoding='latin-1', mode='r')
- for line in sum_file:
+ file_results = sum_file
+ json_input = sum_fname.endswith('.json')
+
+ if json_input:
+ file_results = json.load(sum_file)
+ file_results = file_results['tests']
+
+ for file_result in file_results:
+ if json_input:
+ (result_set.current_tool, result_set.current_exp, line)\
+ = convertJSONResult(file_result)
+
if IsInterestingResult(result_set, line):
result = result_set.MakeTestResult(line, ordinal)
ordinal += 1
@@ -434,6 +462,8 @@
result_set.total += 1
elif IsValidResult(result_set, line):
result_set.total += 1
+ elif json_input:
+ Error('Unrecognized json result: %s' % line)
elif IsExpLine(line):
result_set.current_exp = _EXP_LINE_REX.match(line).groups()[0]
if _OPTIONS.srcpath_regex and _OPTIONS.srcpath_regex != '':
@@ -446,6 +476,10 @@
result_set.current_exp = None
elif IsSummaryLine(line):
result_set.ResetToolExp()
+
+ if json_input:
+ result_set.ResetToolExp()
+
sum_file.close()
return result_set