Add support for pipeline functional tests

Ensure restricted devices are not seen as online.
Add a check for going offline as offline.
Gather pipeline devices separately.

Change-Id: Ic2a7c036b468476cf8f5f0bb6b30d9bbf37b1523

3 files changed, 106 insertions, 8 deletions

diff --git a/lava-functional-test.yaml b/lava-functional-test.yaml
index 9cd755c..f672513 100644
--- a/lava-functional-test.yaml
+++ b/lava-functional-test.yaml
@@ -14,9 +14,11 @@ install:
     deps:
         - python
         - git
+        - python-yaml
 
 run:
     steps:
         - "git clone http://git.linaro.org/git/lava-team/lava-ci.git"
         - "cd lava-ci"
-        - "python lava-job-runner.py $USERNAME $TOKEN $SERVER --stream $BUNDLE_STREAM --repo $JOB_REPO --poll lava-functional-tests.json"
+        - "git clone http://git.linaro.org/git/lava-team/refactoring.git"
+        - "python ./lava-job-runner.py $USERNAME $TOKEN $SERVER --stream $BUNDLE_STREAM --repo $JOB_REPO --poll lava-functional-tests.json"
diff --git a/lava-job-runner.py b/lava-job-runner.py
index 53cbac4..48e41c1 100644
--- a/lava-job-runner.py
+++ b/lava-job-runner.py
@@ -6,11 +6,13 @@
 import subprocess
 import fnmatch
 import time
+import yaml
 import re
 import argparse
 from utils import *
 
 job_map = {}
+yaml_map = {}
 
 
 def poll_jobs(connection):
@@ -23,6 +25,11 @@ def poll_jobs(connection):
             # Push
             submitted_jobs[job_map[job]] = job
 
+    for job in yaml_map:
+        if yaml_map[job] is not None:
+            # Push
+            submitted_jobs[yaml_map[job]] = job
+
     while run:
         if not submitted_jobs:
             run = False
@@ -62,7 +69,7 @@ def poll_jobs(connection):
 def submit_jobs(connection, server, bundle_stream):
     online_devices, offline_devices = gather_devices(connection)
     online_device_types, offline_device_types = gather_device_types(connection)
-    print "Submitting Jobs to Server..."
+    print "Submitting current dispatcher jobs to Server..."
     for job in job_map:
         try:
             with open(job, 'rb') as stream:
@@ -132,6 +139,49 @@ def submit_jobs(connection, server, bundle_stream):
             print e
             continue
 
+def submit_yaml_jobs(connection, server, bundle_stream=None):
+    online_devices, offline_devices = gather_pipeline_devices(connection)
+    online_device_types, offline_device_types = gather_device_types(connection)
+    print "Submitting pipeline dispatcher jobs to Server..."
+    for job in yaml_map:
+        try:
+            with open(job, 'rb') as stream:
+                job_data = stream.read()
+            try:
+                job_info = yaml.load(job_data)
+            except yaml.YAMLError:
+                print "Skipping invalid yaml file %s" % job
+                continue
+            if 'device_type' not in job_info:
+                print "Skipping job %s" % job
+                continue
+            if job_info['device_type'] in offline_device_types:
+                print "All devices of type: %s are OFFLINE, skipping..." % job_info['device_type']
+                print os.path.basename(job) + ' : skip'
+            elif job_info['device_type'] in online_device_types:
+                devices = [item for item in online_devices if job_info['device_type'] in item]
+                pipeline_data = connection.system.user_can_view_devices(devices)
+                if job_info['device_type'] not in pipeline_data:
+                    continue
+                for device_data in pipeline_data[job_info['device_type']]:
+                    for key, value in device_data.items():
+                        if value['is_pipeline']:
+                            submit = True
+                            break
+                    if submit:
+                        break
+                if submit:
+                    yaml_map[job] = connection.scheduler.submit_job(job_data)
+            else:
+                print "No device-type available on server, skipping..."
+                print os.path.basename(job) + ' : skip'
+        except (xmlrpclib.ProtocolError, xmlrpclib.Fault,
+                IOError, ValueError, yaml.YAMLError) as e:
+            print "YAML VALIDATION ERROR!"
+            print job
+            print e
+            continue
+
 
 def load_jobs():
     top = os.getcwd()
@@ -140,6 +190,15 @@ def load_jobs():
             job_map[os.path.join(root, filename)] = None
 
 
+def load_pipeline_jobs():
+    top = os.path.join(os.getcwd(), 'refactoring')
+    for root, _, filenames in os.walk(top):
+        for filename in fnmatch.filter(filenames, '*.yaml'):
+            if 'device' in root:
+                continue
+            yaml_map[os.path.join(root, filename)] = None
+
+
 def retrieve_jobs(jobs):
     cmd = 'git clone %s' % jobs
     try:
@@ -160,14 +219,43 @@ def gather_devices(connection):
     print "Gathering Devices..."
     all_devices = connection.scheduler.all_devices()
     for device in all_devices:
-        if device[2] == 'offline':
-            offline_devices[device[0]] = 1
-        else:
-            online_devices[device[0]] = 1
+        permission_data = connection.system.user_can_view_devices([device[0]])
+        for device_data in permission_data[device[1]]:
+            for key, value in device_data.items():
+                if value['visible']:
+                    submit = True
+                    break
+            if submit:
+                if device[2] in ['going offline', 'offline']:
+                    offline_devices[device[0]] = 1
+                else:
+                    online_devices[device[0]] = 1
     print "Gathered Devices Successfully!"
     return online_devices, offline_devices
 
 
+def gather_pipeline_devices(connection):
+    online_devices = {}
+    offline_devices = {}
+    print "Gathering Pipeline Devices..."
+    all_devices = connection.scheduler.all_devices()
+    for device in all_devices:
+        permission_data = connection.system.user_can_view_devices([device[0]])
+        for device_data in permission_data[device[1]]:
+            submit = False
+            for key, value in device_data.items():
+                if value['visible'] and value['is_pipeline']:
+                    submit = True
+                    break
+            if submit:
+                if device[2] in ['going offline', 'offline']:
+                    offline_devices[device[0]] = 1
+                else:
+                    online_devices[device[0]] = 1
+    print "Gathered Pipeline Devices Successfully!"
+    return online_devices, offline_devices
+
+
 def gather_device_types(connection):
     online_device_types = {}
     offline_device_types = {}
@@ -195,7 +283,9 @@ def main(args):
     if args.repo:
         retrieve_jobs(args.repo)
     load_jobs()
+    load_pipeline_jobs()
     start_time = time.time()
+    submit_yaml_jobs(connection, args.server, None)
     if args.stream:
         submit_jobs(connection, args.server, args.stream)
     else:
@@ -222,4 +312,4 @@ if __name__ == '__main__':
     parser.add_argument("--repo", help="git repo for LAVA jobs")
     parser.add_argument("--poll", help="poll the submitted LAVA jobs, dumps info into specified json")
     args = parser.parse_args()
-    main(args)
\ No newline at end of file
+    main(args)
diff --git a/utils.py b/utils.py
index 6802e04..f90531d 100644
--- a/utils.py
+++ b/utils.py
@@ -4,6 +4,7 @@ import shutil
 import urlparse
 import xmlrpclib
 import json
+import yaml
 
 
 def write_file(file, name, directory):
@@ -21,6 +22,11 @@ def load_json(json_file):
         return json.load(f)
 
 
+def load_yaml(yaml_file):
+    with open(yaml_file, 'r') as data:
+        return yaml.load(data)
+
+
 def mkdir(directory):
     if not ensure_dir(directory):
         shutil.rmtree(directory)
@@ -63,4 +69,4 @@ def connect(url):
         print "Unable to connect to %s" % url
         print e
         print "connect-to-server : fail"
-        exit(1)
\ No newline at end of file
+        exit(1)