[contrib] validate_failures.py: Discard NOEXE to FAIL transitions
When comparing expected with actual results, discard NOEXE to FAIL
transitions, as these are not regressions.
Change-Id: Ia9b4774d085538aa4c3db7c2293fd9cb68e0f143
diff --git a/contrib/testsuite-management/validate_failures.py b/contrib/testsuite-management/validate_failures.py
index 2efd70f..cd7609f 100755
--- a/contrib/testsuite-management/validate_failures.py
+++ b/contrib/testsuite-management/validate_failures.py
@@ -191,6 +191,12 @@
def __ne__(self, other):
return not (self == other)
+ def IsSameTest(self, other):
+ return (self.tool == other.tool and
+ self.exp == other.exp and
+ self.name == other.name and
+ self.description == other.description)
+
def __str__(self):
attrs = ''
if self.attrs:
@@ -658,13 +664,36 @@
return len(flaky_list)
+def DiscardNoexeToFail(expected, actual):
+ """ Discard NOEXE to FAIL transitions, as these are not regressions. """
+ discard_expected_list = []
+
+ for expected_result in expected:
+ if expected_result.state == 'NOEXE':
+ discard_actual = None
+ for actual_result in actual:
+ if actual_result.state == 'FAIL' and \
+ expected_result.IsSameTest(actual_result):
+ discard_actual = actual_result
+ discard_expected_list.append(expected_result)
+ break
+ if discard_actual:
+ actual.discard(discard_actual)
+
+ for discard_expected in discard_expected_list:
+ expected.discard(discard_expected)
+
def PerformComparison(expected, actual):
stats = ResultsStats()
stats.total = actual.total
- # We need to ignore flaky tests in comparison, so remove them now from
- # both expected and actual sets.
+ # We need to ignore flaky tests and NOEXE to FAIL transitions in comparison,
+ # so remove them now from both expected and actual sets.
stats.flaky = DiscardFlaky(expected, actual)
+ if _OPTIONS.inverse_match:
+ DiscardNoexeToFail(actual, expected)
+ else:
+ DiscardNoexeToFail(expected, actual)
stats.fails = len(actual)
actual_vs_expected, expected_vs_actual = CompareResults(expected, actual)