rt-tests: Enable JSON test report
All rt-tests learned to output all results in JSON format. This makes
it possible the replace the really awkward log parser (it has to
handle subtle differences how a rt-tests logs on the console) and
just rely on a generic JSON parser implementation.
Signed-off-by: Daniel Wagner <dwagner@suse.de>
diff --git a/automated/lib/parse_rt_tests_results.py b/automated/lib/parse_rt_tests_results.py
index 3f24870..7ce9980 100755
--- a/automated/lib/parse_rt_tests_results.py
+++ b/automated/lib/parse_rt_tests_results.py
@@ -24,106 +24,46 @@
# SOFTWARE.
#
-import os
-import re
import sys
+import json
-def print_res(res, key):
- print("t{}-{}-latency pass {} us".format(res["t"], key, res[key]))
+def print_res(t, res, key):
+ print("t{}-{}-latency pass {} us".format(t, key, res[key]))
-def get_block(filename):
- # Fetch a text block from the file iterating backwards. Each block
- # starts with an escape sequence which starts with '\x1b'.
- with open(filename, "rb") as f:
- try:
- f.seek(0, os.SEEK_END)
- while True:
- pe = f.tell()
+def parse_threads(rawdata):
+ num_threads = int(rawdata["num_threads"])
+ for thread_id in range(num_threads):
+ tid = str(thread_id)
+ if "receiver" in rawdata["thread"][tid]:
+ data = rawdata["thread"][tid]["receiver"]
+ else:
+ data = rawdata["thread"][tid]
- f.seek(-2, os.SEEK_CUR)
- while f.read(1) != b"\x1b":
- f.seek(-2, os.SEEK_CUR)
- pa = f.tell()
-
- blk = f.read(pe - pa)
-
- # Remove escape sequence at the start of the block
- # The control sequence ends in 'A'
- i = blk.find("A") + 1
- yield blk[i:]
-
- # Jump back to next block
- f.seek(pa - 1, os.SEEK_SET)
- except IOError:
- # No escape sequence found
- f.seek(0, os.SEEK_SET)
- yield f.read()
+ for k in ["min", "avg", "max"]:
+ print_res(tid, data, k)
-def get_lastlines(filename):
- for b in get_block(filename):
- # Ignore empty blocks
- if len(b.strip("\n")) == 0:
- continue
+def parse_json(testname, filename):
+ with open(filename) as file:
+ rawdata = json.load(file)
- return b.split("\n")
+ if "num_threads" in rawdata:
+ # most rt-tests have generic per thread results
+ parse_threads(rawdata)
+ elif "inversions" in rawdata:
+ # pi_stress
+ print("inversion {}\n".format(rawdata("inversions")))
-
-def parse_cyclictest(filename):
- fields = ["t", "min", "avg", "max"]
-
- r = re.compile("[ :\n]+")
- for line in get_lastlines(filename):
- if not line.startswith("T:"):
- continue
-
- data = [x.lower() for x in r.split(line)]
- res = {}
- it = iter(data)
- for e in it:
- if e in fields:
- res[e] = next(it)
-
- print_res(res, "min")
- print_res(res, "avg")
- print_res(res, "max")
-
-
-def parse_pmqtest(filename):
- fields = ["min", "avg", "max"]
-
- rl = re.compile("[ ,:\n]+")
- rt = re.compile("[ ,#]+")
- for line in get_lastlines(filename):
- data = [x.lower() for x in rl.split(line)]
- res = {}
- it = iter(data)
- for e in it:
- if e in fields:
- res[e] = next(it)
-
- if not res:
- continue
-
- # The id is constructed from the '#FROM -> #TO' output, e.g.
- # #1 -> #0, Min 1, Cur 3, Avg 4, Max 119
- data = rt.split(line)
- res["t"] = "{}-{}".format(data[1], data[3])
-
- print_res(res, "min")
- print_res(res, "avg")
- print_res(res, "max")
+ if int(rawdata["return_code"]) == 0:
+ print("{} pass".format(testname))
+ else:
+ print("{} fail".format(testname))
def main():
- tool = sys.argv[1]
- logfile = sys.argv[2]
- if tool in ["cyclictest", "signaltest", "cyclicdeadline"]:
- parse_cyclictest(logfile)
- elif tool in ["pmqtest", "ptsematest", "sigwaittest", "svsematest"]:
- parse_pmqtest(logfile)
+ parse_json(sys.argv[1], sys.argv[2])
if __name__ == "__main__":
diff --git a/automated/linux/cyclicdeadline/cyclicdeadline.sh b/automated/linux/cyclicdeadline/cyclicdeadline.sh
index 4a689c6..20ab72e 100755
--- a/automated/linux/cyclicdeadline/cyclicdeadline.sh
+++ b/automated/linux/cyclicdeadline/cyclicdeadline.sh
@@ -7,7 +7,7 @@
. ../../lib/sh-test-lib
OUTPUT="$(pwd)/output"
-LOGFILE="${OUTPUT}/cyclicdeadline.txt"
+LOGFILE="${OUTPUT}/cyclicdeadline.json"
RESULT_FILE="${OUTPUT}/result.txt"
INTERVAL="1000"
@@ -45,7 +45,7 @@
background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
"${binary}" -q -i "${INTERVAL}" -s "${STEP}" -t "${THREADS}" \
- -D "${DURATION}" | tee "${LOGFILE}"
+ -D "${DURATION}" --json="${LOGFILE}"
background_process_stop bgcmd
diff --git a/automated/linux/cyclictest/cyclictest.sh b/automated/linux/cyclictest/cyclictest.sh
index 31ffb25..ab1484a 100755
--- a/automated/linux/cyclictest/cyclictest.sh
+++ b/automated/linux/cyclictest/cyclictest.sh
@@ -7,7 +7,7 @@
. ../../lib/sh-test-lib
OUTPUT="$(pwd)/output"
-LOGFILE="${OUTPUT}/cyclictest.txt"
+LOGFILE="${OUTPUT}/cyclictest.json"
RESULT_FILE="${OUTPUT}/result.txt"
PRIORITY="98"
@@ -47,7 +47,7 @@
background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
"${binary}" -q -p "${PRIORITY}" -i "${INTERVAL}" -t "${THREADS}" -a "${AFFINITY}" \
- -D "${DURATION}" -m | tee "${LOGFILE}"
+ -D "${DURATION}" -m --json="${LOGFILE}"
background_process_stop bgcmd
diff --git a/automated/linux/pi-stress/pi-stress.sh b/automated/linux/pi-stress/pi-stress.sh
index 637fc96..9a71eb3 100755
--- a/automated/linux/pi-stress/pi-stress.sh
+++ b/automated/linux/pi-stress/pi-stress.sh
@@ -7,7 +7,7 @@
. ../../lib/sh-test-lib
OUTPUT="$(pwd)/output"
-LOGFILE="${OUTPUT}/pi-stress.txt"
+LOGFILE="${OUTPUT}/pi-stress.json"
RESULT_FILE="${OUTPUT}/result.txt"
export RESULT_FILE
@@ -56,17 +56,10 @@
# pi_stress will send SIGTERM when test fails. The single will terminate the
# test script. Catch and ignore it with trap.
trap '' TERM
-"${binary}" -q --duration "${DURATION}" "${MLOCKALL}" "${RR}" | tee "${LOGFILE}"
+"${binary}" -q --duration "${DURATION}" "${MLOCKALL}" "${RR}" --json="${LOGFILE}"
background_process_stop bgcmd
-# shellcheck disable=SC2181
-if [ "$?" -ne "0" ]; then
- report_fail "pi-stress"
-elif grep -q -e "^ERROR:" -e "is deadlocked!" "${LOGFILE}"; then
- report_fail "pi-stress"
-elif ! grep -q -e "Current Inversions:" "${LOGFILE}"; then
- report_fail "pi-stress"
-else
- report_pass "pi-stress"
-fi
+# Parse test log.
+../../lib/parse_rt_tests_results.py pi-stress "${LOGFILE}" \
+ | tee -a "${RESULT_FILE}"
diff --git a/automated/linux/pmqtest/pmqtest.sh b/automated/linux/pmqtest/pmqtest.sh
index c18f133..57ecfba 100755
--- a/automated/linux/pmqtest/pmqtest.sh
+++ b/automated/linux/pmqtest/pmqtest.sh
@@ -6,7 +6,7 @@
TEST_DIR=$(dirname "$(realpath "$0")")
OUTPUT="${TEST_DIR}/output"
-LOGFILE="${OUTPUT}/pmqtest.log"
+LOGFILE="${OUTPUT}/pmqtest.json"
RESULT_FILE="${OUTPUT}/result.txt"
DURATION="5m"
BACKGROUND_CMD=""
@@ -38,7 +38,7 @@
background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
-"${binary}" -q -S -p 98 -D "${DURATION}" | tee "${LOGFILE}"
+"${binary}" -q -S -p 98 -D "${DURATION}" --json="${LOGFILE}"
background_process_stop bgcmd
diff --git a/automated/linux/ptsematest/ptsematest.sh b/automated/linux/ptsematest/ptsematest.sh
index d3996b3..e98a473 100755
--- a/automated/linux/ptsematest/ptsematest.sh
+++ b/automated/linux/ptsematest/ptsematest.sh
@@ -5,7 +5,7 @@
TEST_DIR=$(dirname "$(realpath "$0")")
OUTPUT="${TEST_DIR}/output"
-LOGFILE="${OUTPUT}/ptsematest.log"
+LOGFILE="${OUTPUT}/ptsematest.json"
RESULT_FILE="${OUTPUT}/result.txt"
DURATION="5m"
BACKGROUND_CMD=""
@@ -37,7 +37,7 @@
background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
-"${binary}" -q -S -p 98 -D "${DURATION}" | tee "${LOGFILE}"
+"${binary}" -q -S -p 98 -D "${DURATION}" --json="${LOGFILE}"
background_process_stop bgcmd
diff --git a/automated/linux/rt-migrate-test/rt-migrate-test.sh b/automated/linux/rt-migrate-test/rt-migrate-test.sh
index 5bd54e4..281578d 100755
--- a/automated/linux/rt-migrate-test/rt-migrate-test.sh
+++ b/automated/linux/rt-migrate-test/rt-migrate-test.sh
@@ -5,7 +5,7 @@
. ../../lib/sh-test-lib
OUTPUT="$(pwd)/output"
-LOGFILE="${OUTPUT}/rt-migrate-test.txt"
+LOGFILE="${OUTPUT}/rt-migrate-test.json"
RESULT_FILE="${OUTPUT}/result.txt"
PRIORITY="51"
DURATION="1m"
@@ -37,18 +37,10 @@
background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
-"${binary}" -q -p "${PRIORITY}" -D "${DURATION}" -c | tee "${LOGFILE}"
+"${binary}" -q -p "${PRIORITY}" -D "${DURATION}" -c --json="${LOGFILE}"
background_process_stop bgcmd
# Parse test log.
-task_num=$(grep "Task" "${LOGFILE}" | tail -1 | awk '{print $2}')
-r=$(sed -n 's/Passed!/pass/p; s/Failed!/fail/p' "${LOGFILE}" | awk '{$1=$1;print}')
-for t in $(seq 0 "${task_num}"); do
- # Get the priority of the task.
- p=$(grep "Task $t" "${LOGFILE}" | awk '{print substr($4,1,length($4)-1)}')
- sed -n "/Task $t/,/Avg/p" "${LOGFILE}" \
- | grep -v "Task" \
- | awk -v t="$t" -v p="$p" -v r="$r" '{printf("t%s-p%s-%s %s %s %s\n",t,p,tolower(substr($1, 1, length($1)-1)),r,$2,$3)}' \
- | tee -a "${RESULT_FILE}"
-done
+../../lib/parse_rt_tests_results.py rt-migrate-test "${LOGFILE}" \
+ | tee -a "${RESULT_FILE}"
diff --git a/automated/linux/signaltest/signaltest.sh b/automated/linux/signaltest/signaltest.sh
index 5b53453..b40c1b9 100755
--- a/automated/linux/signaltest/signaltest.sh
+++ b/automated/linux/signaltest/signaltest.sh
@@ -5,7 +5,7 @@
. ../../lib/sh-test-lib
OUTPUT="$(pwd)/output"
-LOGFILE="${OUTPUT}/signaltest.txt"
+LOGFILE="${OUTPUT}/signaltest.json"
RESULT_FILE="${OUTPUT}/result.txt"
PRIORITY="98"
@@ -40,8 +40,7 @@
background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
-"${binary}" -q -D "${DURATION}" -m -p "${PRIORITY}" -t "${THREADS}" \
- | tee "${LOGFILE}"
+"${binary}" -q -D "${DURATION}" -m -p "${PRIORITY}" -t "${THREADS}" --json="${LOGFILE}"
background_process_stop bgcmd
diff --git a/automated/linux/sigwaittest/sigwaittest.sh b/automated/linux/sigwaittest/sigwaittest.sh
index 7b3d066..4e98519 100755
--- a/automated/linux/sigwaittest/sigwaittest.sh
+++ b/automated/linux/sigwaittest/sigwaittest.sh
@@ -7,7 +7,7 @@
TEST_DIR=$(dirname "$(realpath "$0")")
OUTPUT="${TEST_DIR}/output"
-LOGFILE="${OUTPUT}/sigwaittest.log"
+LOGFILE="${OUTPUT}/sigwaittest.json"
RESULT_FILE="${OUTPUT}/result.txt"
DURATION="5m"
BACKGROUND_CMD=""
@@ -39,7 +39,7 @@
background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
-"${binary}" -q -t -a -p 98 -D "${DURATION}" | tee "${LOGFILE}"
+"${binary}" -q -t -a -p 98 -D "${DURATION}" --json="${LOGFILE}"
background_process_stop bgcmd
diff --git a/automated/linux/svsematest/svsematest.sh b/automated/linux/svsematest/svsematest.sh
index c78df17..d5a8e76 100755
--- a/automated/linux/svsematest/svsematest.sh
+++ b/automated/linux/svsematest/svsematest.sh
@@ -7,7 +7,7 @@
TEST_DIR=$(dirname "$(realpath "$0")")
OUTPUT="${TEST_DIR}/output"
-LOGFILE="${OUTPUT}/svsematest.log"
+LOGFILE="${OUTPUT}/svsematest.json"
RESULT_FILE="${OUTPUT}/result.txt"
DURATION="5m"
BACKGROUND_CMD=""
@@ -39,7 +39,7 @@
background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
-"${binary}" -q -t -a -p 98 -D "${DURATION}" | tee "${LOGFILE}"
+"${binary}" -q -t -a -p 98 -D "${DURATION}" --json="${LOGFILE}"
background_process_stop bgcmd