lava-submit.py

#!/usr/bin/python

import os
import sys
import json
import xmlrpclib
import re

tests_nano = [
    'device-tree',
    'gatortests',
    'perf',
    'pwrmgmt',
    ]

tests_alip = [
    'bootchart',
    ]

tests_desktop = [
    'e2eaudiotest',
    'bluetooth-enablement',
    'wifi-enablement',
    'leb-basic-graphics',
    ]

tests_openembedded = [
    ]

# Mapping device type - bundle stream
DEVICE_STREAM = {
    'beaglexm': 'beaglexm',
    'origen': 'leb-origen',
    'panda': 'leb-panda-4430',
    'panda-es': 'leb-panda-es',
    'rtsm_foundation-armv8': 'vexpress64',
    'rtsm_ve-a15x1-a7x1': 'vexpress',
    'rtsm_ve-a15x4-a7x4': 'vexpress',
    'rtsm_ve-armv8': 'vexpress',
    'snowball_sd': 'leb-snowball',
    'vexpress-a5': 'vexpress',
    'vexpress-a9': 'vexpress',
    'vexpress-tc2': 'vexpress',
}


def obfuscate_credentials(s):
    return re.sub(r"([^ ]:).+?(@)", r"\1xxx\2", s)


def main():
    """Script entry point: return some JSON based on calling args.
    We should be called from Jenkins and expect the following to be defined:
    $HWPACK_BUILD_NUMBER $HWPACK_JOB_NAME HWPACK_FILE_NAME $DEVICE_TYPE
    """

    # CI base URL
    ci_base_url = "https://ci.linaro.org/jenkins/job/"
    # Snapshots base URL
    snapshots_url = "http://snapshots.linaro.org"

    # Name of the hardware pack project
    hwpack_job_name = os.environ.get("HWPACK_JOB_NAME")
    # The hardware pack build number
    hwpack_build_number = os.environ.get("HWPACK_BUILD_NUMBER")
    # Hardware pack file name
    hwpack_file_name = os.environ.get("HWPACK_FILE_NAME", "Undefined")
    if hwpack_file_name == "Undefined":
        sys.exit("Hardware pack is not defined.")
    # Device type
    device_type = os.environ.get("DEVICE_TYPE", "Undefined")
    if device_type == "Undefined":
        sys.exit("Device type is not defined.")

    # Distribution, architecture and hardware pack type
    ret_split = hwpack_job_name.split("-", 2)
    (distribution, architecture, hwpack_type) =\
    ret_split[0], ret_split[1], ret_split[2]
    if hwpack_type.startswith('pre-built-images-'):
        hwpack_type = hwpack_type[(len("pre-built-images-")):]
    if hwpack_type.find('='):
        hwpack_type = hwpack_type[(len("pre-built-images/hwpack=")):]
    # Rootfs type, default is developer
    rootfs_type = os.getenv("ROOTFS_TYPE", "developer")

    # Bundle stream name
    bundle_stream_name = os.environ.get("BUNDLE_STREAM_NAME", \
        "/private/team/linaro/pre-built-%s/" % DEVICE_STREAM[device_type])
    # LAVA user
    lava_user = os.environ.get("LAVA_USER")
    if lava_user == None:
        f = open('/var/run/lava/lava-user')
        lava_user = f.read().strip()
        f.close()
    # LAVA token
    lava_token = os.environ.get("LAVA_TOKEN")
    if lava_token == None:
        f = open('/var/run/lava/lava-token')
        lava_token = f.read().strip()
        f.close()
    # LAVA server URL
    lava_server = os.environ.get("LAVA_SERVER", \
        "validation.linaro.org/lava-server/RPC2/")
    # LAVA server base URL
    lava_server_root = lava_server.rstrip("/")
    if lava_server_root.endswith("/RPC2"):
        lava_server_root = lava_server_root[:-len("/RPC2")]

    image_url = "%s/%s/pre-built/%s/%s/%s" %\
                (snapshots_url, \
                 distribution, \
                 hwpack_type, \
                 hwpack_build_number, \
                 hwpack_file_name)

    git_repo = "git://git.linaro.org/qa/test-definitions.git"

    # tests set specific to an image
    tests = tests_nano

    # if ubuntu-desktop, cover more test cases (LEB)
    if rootfs_type == 'ubuntu-desktop':
        tests += tests_desktop

    # if alip, specific tests like bootchart (which should be first)
    if rootfs_type == 'alip':
        tests = tests_alip + tests

    # removing bluetooth and wifi for devices that don't support it
    if device_type in ['beaglexm', 'vexpress-a9', 'mx53loco']:
        try:
            tests.remove('bluetooth-enablement')
            tests.remove('wifi-enablement')
        except ValueError:
            pass

    # vexpress doesn't support PM, so disable pwrmgmt
    if device_type in ['vexpress-a9']:
        try:
            tests.remove('pwrmgmt')
        except ValueError:
            pass

    if distribution == 'openembedded':
        tests = tests_openembedded
        actions = [{
            "command": "deploy_linaro_image",
            "parameters": {
                "image": "%s" % image_url,
            },
            "metadata": {
                "hwpack.type": "%s" % hwpack_type,
                "hwpack.build": "%s" % hwpack_build_number,
                "rootfs.type": "%s" % rootfs_type,
                "distribution": "%s" % distribution,
            }
        }]
    else: # Distribution is Ubuntu
        actions = [{
            "command": "deploy_linaro_image",
            "parameters": {
                "image": "%s" % image_url,
            },
            "metadata": {
                "ubuntu.name": "%s" % hwpack_type,
                "ubuntu.build": "%s" % hwpack_build_number,
                "rootfs.type": "%s" % rootfs_type,
                "ubuntu.distribution": "%s" % distribution,
            }
        }]

    if len(tests) > 0:
        test_list = []
        for test in tests:
            test_list.append({"git-repo": git_repo,
                              "testdef": 'ubuntu/{0}.yaml'.format(test)})
        actions.append({
            "command": "lava_test_shell",
            "parameters": {
                    "testdef_repos": test_list
            }
        })

    actions.append({
        "command": "submit_results",
        "parameters": {
            "stream": bundle_stream_name,
            "server": "%s%s" % ("http://", lava_server)
        }
    })

    config = json.dumps({"timeout": 18000,
                         "actions": actions,
                         "job_name": "%s%s/%s/" % (ci_base_url,\
                                                   hwpack_job_name,\
                                                   hwpack_build_number),
                         "device_type": device_type,
                        }, indent=2)

    print config

    skip_lava = os.environ.get("SKIP_LAVA")
    if skip_lava == None:
        try:
            server_url = \
                "https://{lava_user:>s}:{lava_token:>s}@{lava_server:>s}"
            server = \
                xmlrpclib.ServerProxy(server_url.format(\
                    lava_user=lava_user, \
                    lava_token=lava_token, \
                    lava_server=lava_server))
            lava_job_id = server.scheduler.submit_job(config)
        except xmlrpclib.ProtocolError, e:
            print "Error making a LAVA request:", obfuscate_credentials(str(e))
            sys.exit(1)

        print "LAVA Job Id: %s, URL: http://%s/scheduler/job/%s" % \
              (lava_job_id, lava_server_root, lava_job_id)
        json.dump({'lava_url': "http://" + lava_server_root,
                   'job_id': lava_job_id},
                   open('lava-job-info', 'w'))

        lava_job_info_orig = hwpack_file_name.replace(".img.gz", ".html")
        top_dir = os.environ.get("WORKSPACE", ".")
        for root, dirs, files in os.walk(top_dir):
            for file in files:
                if file == lava_job_info_orig:
                    lava_job_info_orig  = os.path.join(root, file)

                    with open(lava_job_info_orig) as f:
                        buffer = f.read()

                    buffer = buffer.replace("var lavaJobId = 0", \
                                            "var lavaJobId = " + str(lava_job_id), 1)

                    lava_job_info = lava_job_info_orig.replace(".html",\
                        "-" + device_type + ".html")
                    with open(lava_job_info, 'w') as f:
                        f.write(buffer)
                    os.remove(lava_job_info_orig)
    else:
        print "LAVA job submission skipped."

if __name__ == "__main__":
    main()