aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTom Gall <tom.gall@linaro.org>2012-11-21 15:17:47 -0600
committerTom Gall <tom.gall@linaro.org>2012-11-21 15:17:47 -0600
commit9e5843ee8346e4e7ac5803c3990dc9a21bfae2aa (patch)
tree3b9cfc42e734feaff76dd1b7b5f494396de91cba
parentb817d1ec3c80c0bf23db5a09c0011fa1509792a9 (diff)
Fix up results reporting between the python side in glmark2test.pyglmark2
and from the glmark2 side. Based on scene results return a proper piglit marked dictionary
-rw-r--r--framework/glmark2test.py138
-rw-r--r--tests/glmark2/src/scene-simpletriangle.cpp2
-rw-r--r--tests/glmark2/src/scene.h1
-rw-r--r--tests/glmark2/src/test-collection.cpp21
4 files changed, 151 insertions, 11 deletions
diff --git a/framework/glmark2test.py b/framework/glmark2test.py
index ed8031f9..98a7d6e8 100644
--- a/framework/glmark2test.py
+++ b/framework/glmark2test.py
@@ -1,5 +1,5 @@
#
-# Permission is hereby granted, free of charge, to any person
+# Permission is hereby granted, free f charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
@@ -25,6 +25,7 @@ import subprocess
from core import checkDir, testBinDir, Test, TestResult
from exectest import ExecTest
+from exectest import PlainExecTest
#############################################################################
##### Glmark2Test: Execute a sub-test of Glean
@@ -35,7 +36,7 @@ def glmark2Executable():
def glmark2ResultDir(r_dir):
return os.path.join(r_dir, 'glmark2')
-class Glmark2Test(ExecTest):
+class Glmark2Test(PlainExecTest):
globalParams = []
def __init__(self, name, resdir):
@@ -49,12 +50,131 @@ class Glmark2Test(ExecTest):
self.name = name
def run(self, valgrind):
- self.command += Glmark2Test.globalParams
- return ExecTest.run(self, valgrind)
+ fullenv = os.environ.copy()
+ for e in self.env:
+ fullenv[e] = str(self.env[e])
+
+ self.command += Glmark2Test.globalParams
+
+ if self.command is not None:
+ command = self.command
+ i = 0
+ while True:
+ if self.skip_test:
+ out = "PIGLIT: {'result': 'skip'}\n"
+ err = ""
+ returncode = None
+ else:
+ (out, err, returncode) = \
+ self.get_command_result(command, fullenv)
+
+ # https://bugzilla.gnome.org/show_bug.cgi?id=680214 is
+ # affecting many developers. If we catch it
+ # happening, try just re-running the test.
+ if out.find("Got spurious window resize") >= 0:
+ i = i + 1
+ if i >= 5:
+ break
+ else:
+ break
+
+ # proc.communicate() returns 8-bit strings, but we need # unicode strings. In Python 2.x, this is because we
+ # will eventually be serializing the strings as JSON,
+ # and the JSON library expects unicode. In Python 3.x,
+ # this is because all string operations require
+ # unicode. So translate the strings into unicode, # assuming they are using UTF-8 encoding.
+ #
+ # If the subprocess output wasn't properly UTF-8
+ # encoded, we don't want to raise an exception, so
+ # translate the strings using 'replace' mode, which
+ # replaces erroneous charcters with the Unicode
+ # "replacement character" (a white question mark inside
+ # a black diamond).
+ out = out.decode('utf-8', 'replace')
+ err = err.decode('utf-8', 'replace')
+ results = TestResult()
+
+ if self.skip_test:
+ results['result'] = 'skip'
+ else:
+ out = self.interpretResult(out, results)
+ crash_codes = [
+ # Unix: terminated by a signal
+ -5, # SIGTRAP
+ -6, # SIGABRT
+ -8, # SIGFPE (Floating point exception)
+ -10, # SIGUSR1
+ -11, # SIGSEGV (Segmentation fault)
+ # Windows:
+ # EXCEPTION_ACCESS_VIOLATION (0xc0000005):
+ -1073741819,
+ # EXCEPTION_INT_DIVIDE_BY_ZERO (0xc0000094):
+ -1073741676
+ ]
+
+ if returncode in crash_codes:
+ results['result'] = 'crash'
+ elif returncode != 0:
+ results['note'] = 'Returncode was {0}'.format(returncode)
+ env = ''
+ for key in self.env:
+ env = env + key + '="' + self.env[key] + '" '
+ if env:
+ results['environment'] = env
+
+ results['info'] = "Returncode: {0}\n\nErrors:\n{1}\n\nOutput:\n{2}".format(returncode, err, out)
+ results['returncode'] = returncode
+ results['command'] = ' '.join(self.command)
+
+ self.handleErr(results, err)
- def interpretResult(self, out, results):
- if out.find('FAIL') >= 0:
- results['result'] = 'fail'
else:
- results['result'] = 'pass'
- return out
+ results = TestResult()
+ results['result'] = 'skip'
+
+ return results
+
+ def get_command_result(self, command, fullenv):
+ try:
+ proc = subprocess.Popen(
+ command,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=fullenv,
+ universal_newlines=True
+ )
+ out, err = proc.communicate()
+ returncode = proc.returncode
+ except OSError as e:
+ # Different sets of tests get built under
+ # different build configurations. If
+ # a developer chooses to not build a test,
+ # Piglit should not report that test as having
+ # failed.
+ if e.strerror == "No such file or directory":
+ out = "PIGLIT: {'result': 'skip'}\n" \
+ + "Test executable not found.\n"
+ err = ""
+ returncode = None
+ else:
+ raise e
+ return out, err, returncode
+
+ def handleErr(self, results,err):
+ errors = filter(lambda s: len(s) > 0, map(lambda s: s.strip(), err.split('\n')))
+
+ ignored = [s for s in errors if self.isIgnored(s)]
+ errors = [s for s in errors if s not in ignored]
+
+ if len(errors) > 0:
+ results['errors'] = errors
+
+ if len(ignored) > 0:
+ results['errors_ignored'] = ignored
+
+ def isIgnored(self,s):
+ if s == "libEGL warning: unsupported platform Windows":
+ return True
+ else:
+ return False
+
diff --git a/tests/glmark2/src/scene-simpletriangle.cpp b/tests/glmark2/src/scene-simpletriangle.cpp
index a148d70a..c8785fc4 100644
--- a/tests/glmark2/src/scene-simpletriangle.cpp
+++ b/tests/glmark2/src/scene-simpletriangle.cpp
@@ -38,7 +38,7 @@ SceneSimpleTriangle::SceneSimpleTriangle(Canvas &pCanvas) :
Scene(pCanvas, "simpletriangle"), radius_(0.0),
orientModel_(false), orientationAngle_(0.0)
{
-
+testResult=Scene::ValidationUnknown;
duration_=30; // set for 30 seconds
}
diff --git a/tests/glmark2/src/scene.h b/tests/glmark2/src/scene.h
index e543bf57..3930928d 100644
--- a/tests/glmark2/src/scene.h
+++ b/tests/glmark2/src/scene.h
@@ -293,6 +293,7 @@ public:
~SceneSimpleTriangle();
protected:
+ Scene::ValidationResult testResult;
Program program_;
Mesh mesh_;
GLuint texture_;
diff --git a/tests/glmark2/src/test-collection.cpp b/tests/glmark2/src/test-collection.cpp
index bf960903..31775f30 100644
--- a/tests/glmark2/src/test-collection.cpp
+++ b/tests/glmark2/src/test-collection.cpp
@@ -25,6 +25,7 @@
#include "options.h"
#include "log.h"
#include "util.h"
+#include "scene.h"
TestCollection::~TestCollection()
{
@@ -120,5 +121,23 @@ TestCollection::benchmarks_contain_normal_scenes()
void
TestCollection::report_results()
{
-
+ Scene::ValidationResult r;
+
+ fflush(stderr);
+
+ for (std::vector<Benchmark *>::const_iterator iter = tests_.begin();
+ iter != tests_.end();
+ iter++) {
+ const Benchmark *bench=*iter;
+ switch(bench->scene().validate()) {
+ case Scene::ValidationSuccess:
+ printf("PIGLIT: {'result': 'pass' }\n");
+ break;
+ case Scene::ValidationFailure:
+ case Scene::ValidationUnknown:
+ printf("PIGLIT: {'result': 'fail' }\n");
+ break;
+ }
+ }
+ fflush(stdout);
}