automated: android: enable CTS/VTS atomic result reporting

This patch allows to either record the CTS/VTS results as 'atomic' or as
'aggregated'. Atomic results show each test case while aggregated only
present number of passed/failed for each module. Examples
1. aggregated
arm64-v8a.CtsBionicTestCases_executed pass 1853
arm64-v8a.CtsBionicTestCases_passed pass 1850
arm64-v8a.CtsBionicTestCases_failed fail 3

2. atomic
arm64-v8a.CtsBionicTestCases/DlExtRelroSharingTest.ChildWritesGoodData
pass
arm64-v8a.CtsBionicTestCases/DlExtRelroSharingTest.ChildWritesNoRelro
pass
arm64-v8a.CtsBionicTestCases/DlExtRelroSharingTest.RelroFileEmpty pass
arm64-v8a.CtsBionicTestCases/DlExtRelroSharingTest.VerifyMemorySaving
pass
...

Change-Id: I6729235607cfa51b04dd7a66e4a8e49859c40876
Signed-off-by: Milosz Wasilewski <milosz.wasilewski@linaro.org>

3 files changed, 54 insertions, 28 deletions

diff --git a/automated/android/tradefed/tradefed-runner.py b/automated/android/tradefed/tradefed-runner.py
index cc7a281e..c6e78e74 100755
--- a/automated/android/tradefed/tradefed-runner.py
+++ b/automated/android/tradefed/tradefed-runner.py
@@ -17,7 +17,16 @@ sys.path.insert(0, '../../lib/')
 import py_test_lib  # nopep8
 
 
-def result_parser(xml_file):
+OUTPUT = '%s/output' % os.getcwd()
+RESULT_FILE = '%s/result.txt' % OUTPUT
+TRADEFED_STDOUT = '%s/tradefed-stdout.txt' % OUTPUT
+TRADEFED_LOGCAT = '%s/tradefed-logcat.txt' % OUTPUT
+TEST_PARAMS = ''
+AGGREGATED = 'aggregated'
+ATOMIC = 'atomic'
+
+
+def result_parser(xml_file, result_format):
     etree_file = open(xml_file, 'rb')
     etree_content = etree_file.read()
     rx = re.compile("&#([0-9]+);|&#x([0-9a-fA-F]+);")
@@ -58,35 +67,48 @@ def result_parser(xml_file):
         else:
             module_name = elem.attrib['name']
 
-        tests_executed = len(elem.findall('.//Test'))
-        tests_passed = len(elem.findall('.//Test[@result="pass"]'))
-        tests_failed = len(elem.findall('.//Test[@result="fail"]'))
+        if result_format == AGGREGATED:
+            tests_executed = len(elem.findall('.//Test'))
+            tests_passed = len(elem.findall('.//Test[@result="pass"]'))
+            tests_failed = len(elem.findall('.//Test[@result="fail"]'))
+
+            result = '%s_executed pass %s' % (module_name, str(tests_executed))
+            py_test_lib.add_result(RESULT_FILE, result)
+
+            result = '%s_passed pass %s' % (module_name, str(tests_passed))
+            py_test_lib.add_result(RESULT_FILE, result)
+
+            failed_result = 'pass'
+            if tests_failed > 0:
+                failed_result = 'fail'
+            result = '%s_failed %s %s' % (module_name, failed_result,
+                                          str(tests_failed))
+            py_test_lib.add_result(RESULT_FILE, result)
+
+        if result_format == ATOMIC:
+            test_cases = elem.findall('.//TestCase')
+            for test_case in test_cases:
+                tests = test_case.findall('.//Test')
+                for atomic_test in tests:
+                    atomic_test_result = atomic_test.get("result")
+                    atomic_test_name = "%s/%s.%s" % (module_name,
+                                                     test_case.get("name"),
+                                                     atomic_test.get("name"))
+                    py_test_lib.add_result(
+                        RESULT_FILE, "%s %s" % (atomic_test_name,
+                                                atomic_test_result))
 
-        result = '%s_executed pass %s' % (module_name, str(tests_executed))
-        py_test_lib.add_result(RESULT_FILE, result)
-
-        result = '%s_passed pass %s' % (module_name, str(tests_passed))
-        py_test_lib.add_result(RESULT_FILE, result)
-
-        failed_result = 'pass'
-        if tests_failed > 0:
-            failed_result = 'fail'
-        result = '%s_failed %s %s' % (module_name, failed_result,
-                                      str(tests_failed))
-        py_test_lib.add_result(RESULT_FILE, result)
-
-
-OUTPUT = '%s/output' % os.getcwd()
-RESULT_FILE = '%s/result.txt' % OUTPUT
-TRADEFED_STDOUT = '%s/tradefed-stdout.txt' % OUTPUT
-TRADEFED_LOGCAT = '%s/tradefed-logcat.txt' % OUTPUT
-TEST_PARAMS = ''
 
 parser = argparse.ArgumentParser()
 parser.add_argument('-t', dest='TEST_PARAMS', required=True,
                     help="tradefed shell test parameters")
 parser.add_argument('-p', dest='TEST_PATH', required=True,
                     help="path to tradefed package top directory")
+parser.add_argument('-r', dest='RESULTS_FORMAT', required=False,
+                    default=AGGREGATED, choices=[AGGREGATED, ATOMIC],
+                    help="The format of the saved results. 'aggregated' means number of \
+                    passed and failed tests are recorded for each module. 'atomic' means \
+                    each test result is recorded separately")
 args = parser.parse_args()
 # TEST_PARAMS = args.TEST_PARAMS
 
@@ -187,4 +209,4 @@ if os.path.exists(result_dir) and os.path.isdir(result_dir):
     for root, dirs, files in os.walk(result_dir):
         for name in files:
             if name == test_result:
-                result_parser(xml_file=os.path.join(root, name))
+                result_parser(os.path.join(root, name), args.RESULTS_FORMAT)
diff --git a/automated/android/tradefed/tradefed.sh b/automated/android/tradefed/tradefed.sh
index 9aaf3b76..868e81b0 100755
--- a/automated/android/tradefed/tradefed.sh
+++ b/automated/android/tradefed/tradefed.sh
@@ -10,21 +10,23 @@ TIMEOUT="300"
 TEST_URL="http://testdata.validation.linaro.org/cts/android-cts-7.1_r1.zip"
 TEST_PARAMS="run cts -m CtsBionicTestCases --abi arm64-v8a --disable-reboot --skip-preconditions --skip-device-info"
 TEST_PATH="android-cts"
+RESULT_FORMAT="aggregated"
 RESULT_FILE="$(pwd)/output/result.txt"
 export RESULT_FILE
 
 usage() {
-    echo "Usage: $0 [-o timeout] [-n serialno] [-c cts_url] [-t test_params] [-p test_path]" 1>&2
+    echo "Usage: $0 [-o timeout] [-n serialno] [-c cts_url] [-t test_params] [-p test_path] [-r <aggregated|atomic>]" 1>&2
     exit 1
 }
 
-while getopts ':o:n:c:t:p:' opt; do
+while getopts ':o:n:c:t:p:r:' opt; do
     case "${opt}" in
         o) TIMEOUT="${OPTARG}" ;;
         n) ANDROID_SERIAL="${OPTARG}" ;;
         c) TEST_URL="${OPTARG}" ;;
         t) TEST_PARAMS="${OPTARG}" ;;
         p) TEST_PATH="${OPTARG}" ;;
+        r) RESULT_FORMAT="${OPTARG}" ;;
         *) usage ;;
     esac
 done
@@ -56,4 +58,4 @@ fi
 
 # Run tradefed test.
 info_msg "About to run tradefed shell on device ${ANDROID_SERIAL}"
-./tradefed-runner.py -t "${TEST_PARAMS}" -p "${TEST_PATH}"
+./tradefed-runner.py -t "${TEST_PARAMS}" -p "${TEST_PATH}" -r "${RESULT_FORMAT}"
diff --git a/automated/android/tradefed/tradefed.yaml b/automated/android/tradefed/tradefed.yaml
index 86a8ad50..142f21fc 100644
--- a/automated/android/tradefed/tradefed.yaml
+++ b/automated/android/tradefed/tradefed.yaml
@@ -24,6 +24,8 @@ params:
     # set to the name of the top directory in TEST_URL archive
     # This should be 'android-cts' for CTS and android-vts for VTS
     TEST_PATH: "android-cts"
+    # Specify result format: aggregated or atomic
+    RESULTS_FORMAT: "aggregated"
     # Specify url and token for file uploading.
     URL: "https://archive.validation.linaro.org/artifacts/team/qa/"
     TOKEN: ""
@@ -34,7 +36,7 @@ run:
         - ./setup.sh
         - useradd testuser
         - mkdir /home/testuser; chown testuser.testuser /home/testuser
-        - sudo -u testuser ./tradefed.sh  -o "${TIMEOUT}" -c "${TEST_URL}" -t "${TEST_PARAMS}" -p "${TEST_PATH}"
+        - sudo -u testuser ./tradefed.sh  -o "${TIMEOUT}" -c "${TEST_URL}" -t "${TEST_PARAMS}" -p "${TEST_PATH}" -r "${RESULTS_FORMAT}"
         # Upload test log and result files to artifactorial.
         - cp -r ./${TEST_PATH}/results ./output/ || true
         - cp -r ./${TEST_PATH}/logs ./output/ || true