automated: add support for workload automation test suite

Change-Id: Ie07d58c902e4c905c938b1aad8a7c126cdcf7ebf
Signed-off-by: Chase Qi <chase.qi@linaro.org>

4 files changed, 294 insertions, 0 deletions

diff --git a/automated/android/workload-automation/workload-automation.sh b/automated/android/workload-automation/workload-automation.sh
new file mode 100755
index 0000000..327b4db
--- /dev/null
+++ b/automated/android/workload-automation/workload-automation.sh
@@ -0,0 +1,112 @@
+#!/bin/sh -ex
+# shellcheck disable=SC1090
+
+TEST_DIR=$(dirname "$(realpath "$0")")
+OUTPUT="${TEST_DIR}/output"
+RESULT_FILE="${OUTPUT}/result.txt"
+export RESULT_FILE
+SKIP_INSTALL="false"
+ANDROID_SERIAL=""
+BOOT_TIMEOUT="300"
+
+WA_TAG="master"
+WA_TEMPLATES_REPO="https://git.linaro.org/qa/wa2-lava.git"
+TEMPLATES_BRANCH="wa-templates"
+CONFIG="config/generic-android.py"
+AGENDA="agenda/generic-linpack.yaml"
+BUILD_TOOLS_URL="http://testdata.validation.linaro.org/apks/workload-automation/build-tools.tar.gz"
+WA_HOME_URL="http://testdata.validation.linaro.org/apks/workload-automation/workload_automation_home.tar.gz"
+
+usage() {
+    echo "Usage: $0 [-s <true|false>] [-S <android_serial>] [-t <boot_timeout>] [-T <wa_tag>] [-r <wa_templates_repo>] [-g <templates_branch>] [-c <config>] [-a <agenda>] [-b <build_tools_url>] [-w <wa_home_url>]" 1>&2
+    exit 1
+}
+
+while getopts ":s:S:t:T:r:g:c:a:b:w:" opt; do
+    case "${opt}" in
+        s) SKIP_INSTALL="${OPTARG}" ;;
+        S) ANDROID_SERIAL="${OPTARG}" ;;
+        t) BOOT_TIMEOUT="${OPTARG}" ;;
+        T) WA_TAG="${OPTARG}" ;;
+        r) WA_TEMPLATES_REPO="${OPTARG}" ;;
+        g) TEMPLATES_BRANCH="${OPTARG}" ;;
+        c) CONFIG="${OPTARG}" ;;
+        a) AGENDA="${OPTARG}" ;;
+        b) BUILD_TOOLS_URL="${OPTARG}" ;;
+        w) WA_HOME_URL="${OPTARG}" ;;
+        *) usage ;;
+    esac
+done
+
+. "${TEST_DIR}/../../lib/sh-test-lib"
+. "${TEST_DIR}/../../lib/android-test-lib"
+
+cd "${TEST_DIR}"
+create_out_dir "${OUTPUT}"
+
+if [ "${SKIP_INSTALL}" = "true" ] || [ "${SKIP_INSTALL}" = "True" ]; then
+    info_msg "WA installation skipped"
+else
+    PKGS="git wget zip tar xz-utils python python-yaml python-lxml python-setuptools python-numpy python-colorama python-pip sqlite3 libstdc++6:i386 libgcc1:i386 zlib1g:i386 libncurses5:i386 aapt android-tools-adb time sysstat python-jinja2 curl"
+    ! check_root && error_msg "Please run this test as root."
+    dpkg --add-architecture i386
+    install_deps "${PKGS}"
+    pip install --upgrade pip && hash -r
+    pip install --upgrade setuptools
+    pip install pexpect pyserial pyyaml docutils python-dateutil
+    info_msg "Installing workload-automation..."
+    rm -rf workload-automation
+    git clone https://github.com/ARM-software/workload-automation
+    (
+    cd workload-automation
+    git checkout -b test-branch "${WA_TAG}"
+    )
+    pip2 install ./workload-automation
+    export PATH=$PATH:/usr/local/bin
+    which wa
+
+    info_msg "Installing SDK build-tools..."
+    (
+        cd /usr/
+        # Copy build-tools.tar.gz to /usr for local run.
+        test -f build-tools.tar.gz || wget -S --progress=dot:giga "${BUILD_TOOLS_URL}"
+        tar -xf build-tools.tar.gz
+    )
+
+    info_msg "Installing workloads bbench and APKs..."
+    (
+        cd /root/
+        # Copy workload_automation_home.tar.gz to /root for local run.
+        test -f workload_automation_home.tar.gz || wget -S --progress=dot:giga "${WA_HOME_URL}"
+        tar -xf workload_automation_home.tar.gz
+    )
+fi
+
+initialize_adb
+adb_root
+wait_boot_completed "${BOOT_TIMEOUT}"
+disable_suspend
+
+rm -rf wa-templates
+git clone "${WA_TEMPLATES_REPO}" wa-templates
+(
+    cd wa-templates
+    git checkout "${TEMPLATES_BRANCH}"
+    cp "${CONFIG}" ../config.py
+    cp "${AGENDA}" ../agenda.yaml
+)
+
+sed -i "s/adb_name=.*/adb_name=\'${ANDROID_SERIAL}\',/" ./config.py
+# Ensure that csv is enabled in result processors.
+if ! awk '/result_processors = [[]/,/[]]/' ./config.py | grep -q 'csv'; then
+    sed -i "s/result_processors = [[]/result_processors = [\n    'csv',/" ./config.py
+fi
+
+info_msg "device-${ANDROID_SERIAL}: About to run WA with ${AGENDA}..."
+wa run ./agenda.yaml -v -f -d "${OUTPUT}/wa" -c ./config.py || report_fail "wa-test-run"
+
+# Generate result.txt for sending results to LAVA.
+# Use id-iteration_metric as test case name.
+awk -F',' 'NR>1 {gsub(/[ _]/,"-",$4); printf("%s-itr%s_%s pass %s %s\n",$1,$3,$4,$5,$6)}' "${OUTPUT}/wa/results.csv" \
+    | sed 's/\r//g' \
+    | tee -a "${RESULT_FILE}"
diff --git a/automated/android/workload-automation/workload-automation.yaml b/automated/android/workload-automation/workload-automation.yaml
new file mode 100644
index 0000000..ccecb07
--- /dev/null
+++ b/automated/android/workload-automation/workload-automation.yaml
@@ -0,0 +1,48 @@
+metadata:
+    name: workload-automation
+    format: "Lava-Test-Shell Test Definition 1.0"
+    description: "Workload Automation on Android"
+    maintainer:
+        - milosz.wasilewski@linaro.org
+        - chase.qi@linaro.org
+    os:
+        - android
+    devices:
+        - juno
+        - hi6220-hikey
+        - apq8016-sbc
+        - x15
+    scope:
+        - performance
+
+params:
+    SKIP_INSTALL: "false"
+    # Timeout for wait_boot_completed in seconds.
+    BOOT_TIMEOUT: "300"
+    # Specify adb device SN if more then one device connected.
+    ANDROID_SERIAL: ""
+    # Params for WA test run.
+    WA_TAG: "master"
+    BUILD_TOOLS_URL: "http://testdata.validation.linaro.org/apks/workload-automation/build-tools.tar.gz"
+    WA_HOME_URL: "http://testdata.validation.linaro.org/apks/workload-automation/workload_automation_home.tar.gz"
+    WA_TEMPLATES_REPO: "https://git.linaro.org/qa/wa2-lava.git"
+    TEMPLATES_BRANCH: "wa-templates"
+    CONFIG: "config/generic-android.py"
+    AGENDA: "agenda/android-linpack.yaml"
+    # Specify url and token for publishing artifacts.
+    # For safety reasons, please set 'ARTIFACTORIAL_TOKEN' variable in job definition with
+    # 'secrets' dictionary, and set job visibility to personal or group.
+    # Refer to https://validation.linaro.org/static/docs/v2/publishing-artifacts.html
+    ARTIFACTORIAL_URL: "https://archive.validation.linaro.org/artifacts/team/qa/"
+    ARTIFACTORIAL_TOKEN: ""
+
+run:
+    steps:
+        - cd ./automated/android/workload-automation
+        # Test run.
+        - ./workload-automation.sh -s "${SKIP_INSTALL}" -t "${BOOT_TIMEOUT}" -S "${ANDROID_SERIAL}" -T "${WA_TAG}" -r "${WA_TEMPLATES_REPO}" -g "${TEMPLATES_BRANCH}" -c "${CONFIG}" -a "${AGENDA}" -b "${BUILD_TOOLS_URL}" -w "${WA_HOME_URL}"
+        # Upload test output to artifactorial.
+        - tar caf "wa-output.tar.xz" "./output"
+        - ../../utils/upload-to-artifactorial.sh -a "wa-output.tar.xz" -u "${ARTIFACTORIAL_URL}" -t "${ARTIFACTORIAL_TOKEN}"
+        # Send test result to LAVA.
+        - ../../utils/send-to-lava.sh "./output/result.txt"
diff --git a/automated/linux/workload-automation/workload-automation.sh b/automated/linux/workload-automation/workload-automation.sh
new file mode 100755
index 0000000..a0a1d35
--- /dev/null
+++ b/automated/linux/workload-automation/workload-automation.sh
@@ -0,0 +1,89 @@
+#!/bin/sh -ex
+# shellcheck disable=SC1090
+
+TEST_DIR=$(dirname "$(realpath "$0")")
+OUTPUT="${TEST_DIR}/output"
+RESULT_FILE="${OUTPUT}/result.txt"
+export RESULT_FILE
+SKIP_INSTALL="false"
+
+WA_TAG="master"
+WA_TEMPLATES_REPO="https://git.linaro.org/qa/wa2-lava.git"
+TEMPLATES_BRANCH="wa-templates"
+CONFIG="config/generic-linux-localhost.py"
+AGENDA="agenda/linux-dhrystone.yaml"
+
+usage() {
+    echo "Usage: $0 [-s <true|false>] [-t <wa_tag>] [-r <wa_templates_repo>] [-T <templates_branch>] [-c <config>] [-a <agenda>]" 1>&2
+    exit 1
+}
+
+while getopts ":s:t:r:T:c:a:" opt; do
+    case "${opt}" in
+        s) SKIP_INSTALL="${OPTARG}" ;;
+        t) WA_TAG="${OPTARG}" ;;
+        r) WA_TEMPLATES_REPO="${OPTARG}" ;;
+        T) TEMPLATES_BRANCH="${OPTARG}" ;;
+        c) CONFIG="${OPTARG}" ;;
+        a) AGENDA="${OPTARG}" ;;
+        *) usage ;;
+    esac
+done
+
+. "${TEST_DIR}/../../lib/sh-test-lib"
+
+! check_root && error_msg "Please run this test as root."
+cd "${TEST_DIR}"
+create_out_dir "${OUTPUT}"
+
+if [ "${SKIP_INSTALL}" = "true" ] || [ "${SKIP_INSTALL}" = "True" ]; then
+    info_msg "WA installation skipped"
+else
+    PKGS="git wget zip tar xz-utils python python-yaml python-lxml python-setuptools python-numpy python-colorama python-pip sqlite3 time sysstat openssh-client openssh-server sshpass python-jinja2 curl"
+    install_deps "${PKGS}"
+    pip install --upgrade pip && hash -r
+    pip install --upgrade setuptools
+    pip install pexpect pyserial pyyaml docutils python-dateutil
+    info_msg "Installing workload-automation..."
+    rm -rf workload-automation
+    git clone https://github.com/ARM-software/workload-automation
+    (
+    cd workload-automation
+    git checkout -b test-branch "${WA_TAG}"
+    )
+    pip2 install ./workload-automation
+    export PATH=$PATH:/usr/local/bin
+    which wa
+    mkdir -p ~/.workload_automation
+    wa --version
+fi
+
+rm -rf wa-templates
+git clone "${WA_TEMPLATES_REPO}" wa-templates
+(
+    cd wa-templates
+    git checkout "${TEMPLATES_BRANCH}"
+    cp "${CONFIG}" ../config.py
+    cp "${AGENDA}" ../agenda.yaml
+)
+
+# Setup root SSH login with password for test run via loopback.
+sed -i 's/^PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config
+sed -i 's/^# *PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config
+grep "PermitRootLogin yes" /etc/ssh/sshd_config
+echo "root:linaro123" | chpasswd
+/etc/init.d/ssh restart && sleep 3
+
+# Ensure that csv is enabled in result processors.
+if ! awk '/result_processors = [[]/,/[]]/' ./config.py | grep -q 'csv'; then
+    sed -i "s/result_processors = [[]/result_processors = [\n    'csv',/" ./config.py
+fi
+
+info_msg "About to run WA with ${AGENDA}..."
+wa run ./agenda.yaml -v -f -d "${OUTPUT}/wa" -c ./config.py || report_fail "wa-test-run"
+
+# Save results from results.csv to result.txt.
+# Use id-iteration_metric as test case name.
+awk -F',' 'NR>1 {gsub(/[ _]/,"-",$4); printf("%s-itr%s_%s pass %s %s\n",$1,$3,$4,$5,$6)}' "${OUTPUT}/wa/results.csv" \
+    | sed 's/\r//g' \
+    | tee -a "${RESULT_FILE}"
diff --git a/automated/linux/workload-automation/workload-automation.yaml b/automated/linux/workload-automation/workload-automation.yaml
new file mode 100644
index 0000000..4aa1da3
--- /dev/null
+++ b/automated/linux/workload-automation/workload-automation.yaml
@@ -0,0 +1,45 @@
+metadata:
+    name: workload-automation-linux
+    format: "Lava-Test-Shell Test Definition 1.0"
+    description: "Workload Automation on Linux"
+    maintainer:
+        - milosz.wasilewski@linaro.org
+        - chase.qi@linaro.org
+    os:
+        - debian
+        - ubuntu
+        - centos
+        - fedora
+    devices:
+        - juno
+        - hi6220-hikey
+        - apq8016-sbc
+        - x15
+    scope:
+        - performance
+
+params:
+    SKIP_INSTALL: "false"
+    # Params for WA test run.
+    WA_TAG: "master"
+    WA_TEMPLATES_REPO: "https://git.linaro.org/qa/wa2-lava.git"
+    TEMPLATES_BRANCH: "wa-templates"
+    CONFIG: "config/generic-linux-localhost.py"
+    AGENDA: "agenda/linux-dhrystone.yaml"
+    # Specify url and token for publishing artifacts.
+    # For safety reasons, please set 'ARTIFACTORIAL_TOKEN' variable in job definition with
+    # 'secrets' dictionary, and set job visibility to personal or group.
+    # Refer to https://validation.linaro.org/static/docs/v2/publishing-artifacts.html
+    ARTIFACTORIAL_URL: "https://archive.validation.linaro.org/artifacts/team/qa/"
+    ARTIFACTORIAL_TOKEN: ""
+
+run:
+    steps:
+        - cd ./automated/linux/workload-automation
+        # Test run.
+        - ./workload-automation.sh -s "${SKIP_INSTALL}" -t "${WA_TAG}" -r "${WA_TEMPLATES_REPO}" -T "${TEMPLATES_BRANCH}" -c "${CONFIG}" -a "${AGENDA}"
+        # Upload test output to artifactorial.
+        - tar caf "wa-output.tar.xz" "./output"
+        - ../../utils/upload-to-artifactorial.sh -a "wa-output.tar.xz" -u "${ARTIFACTORIAL_URL}" -t "${ARTIFACTORIAL_TOKEN}"
+        # Send test result to LAVA.
+        - ../../utils/send-to-lava.sh "./output/result.txt"