Added support for agenda file

2 files changed, 163 insertions, 53 deletions

diff --git a/agenda-example.yaml b/agenda-example.yaml
new file mode 100644
index 0000000..fcba369
--- /dev/null
+++ b/agenda-example.yaml
@@ -0,0 +1,29 @@
+# support remote and local repo
+repository: https://git.linaro.org/qa/test-definitions.git
+# repository: ./test-definitions
+definitions:
+  - path: ubuntu/smoke-tests-basic.yaml
+    timeout: 180
+  - path: ubuntu/blogbench.yaml
+    timeout: 1800
+    parameters:
+        ITERATION: 3
+  - path: ubuntu/pi-stress-test.yaml
+    timeout: 1800
+    params:
+        DURATION: 30
+  - path: ubuntu/pi-stress-test.yaml
+    timeout: 1800
+    params:
+        DURATION: 30
+        GROUP: 1
+    skip_install: True
+  - path: ubuntu/ltp.yaml
+    timeout: 3600
+    params:
+       TST_CMDFILES: math
+  - path: ubuntu/ltp.yaml
+    timeout: 3600
+    params:
+       TST_CMDFILES: timers
+    skip_install: True
diff --git a/lava-local-test.py b/lava-local-test.py
index 5fc4eaa..4a2b8f2 100755
--- a/lava-local-test.py
+++ b/lava-local-test.py
@@ -15,6 +15,27 @@ import argparse
 from uuid import uuid4
 
 
+class Agenda(object):
+    """
+    Analysis and convert agenda file.
+    """
+
+    def __init__(self):
+        self.agenda = agenda
+        with open(self.agenda, 'r') as f:
+            self.agenda = yaml.safe_load(f)
+
+    def validate(self):
+        key_list = ['repository', 'definitions']
+        for item in key_list:
+            if item not in self.agenda:
+                print('%s field is missing from agenda file' % item)
+                sys.exit(1)
+
+    def agenda_dict(self):
+        return self.agenda
+
+
 class TestDefinition(object):
     """
     Analysis and convert test definition.
@@ -23,10 +44,11 @@ class TestDefinition(object):
     def __init__(self):
         self.test_def = test_path + '/' + test_def
         self.test_path = test_path
+        self.test_parameters = test_parameters
         # Read the YAML to create a testdef dict
         with open(self.test_def, 'r') as f:
             self.testdef = yaml.safe_load(f)
-        self.parameters = self.parameters()
+        self.parameters = self.handle_parameters()
         self.skip_install = skip_install
 
     def definition(self):
@@ -97,7 +119,7 @@ class TestDefinition(object):
                     f.write('%s\n' % cmd)
             f.write('echo "<LAVA_SIGNAL_ENDRUN $TESTRUN_ID $UUID>"\n')
 
-    def parameters(self):
+    def handle_parameters(self):
         ret_val = ['###default parameters from test definition###\n']
 
         if 'params' in self.testdef:
@@ -115,6 +137,15 @@ class TestDefinition(object):
             return None
 
         ret_val.append('######\n')
+
+        ret_val.append('###test parameters from agenda file###\n')
+        if self.test_parameters:
+            for param_name, param_value in list(self.test_parameters.items()):
+                if param_name is 'yaml_line':
+                    continue
+                ret_val.append('%s=\'%s\'\n' % (param_name, param_value))
+        ret_val.append('######\n')
+
         return ret_val
 
     def return_pattern(self):
@@ -168,7 +199,6 @@ class TestRunner(object):
         self.lava_path = LAVA_PATH
         self.test_uuid = test_uuid
         self.test_timeout = test_timeout
-        print('\nAbout to run %s' % self.test_uuid)
         self.child = pexpect.spawn('%s/bin/lava-test-runner %s' % (self.lava_path, self.lava_path))
 
     def check_output(self):
@@ -193,6 +223,7 @@ class TestRunner(object):
 
 class ResultPaser(object):
     def __init__(self):
+        self.lava_path = LAVA_PATH
         self.result_path = result_path
         # Fix result path with timestamp added by lava-test-runner.
         self.result_path = glob.glob('%s-[0-9]*' % self.result_path)[0]
@@ -212,7 +243,7 @@ class ResultPaser(object):
         self.dict_to_json()
         self.dict_to_csv()
         print('\nResult files saved to: %s' % self.result_path)
-        print('---Printing results.csv---')
+        print('--- Printing results.csv ---')
         with open('%s/results.csv' % self.result_path) as f:
             print(f.read())
 
@@ -266,6 +297,12 @@ class ResultPaser(object):
             for metric in self.results['metrics']:
                 writer.writerow(metric)
 
+        with open('%s/results.csv' % self.lava_path, 'a') as f:
+            writer = csv.DictWriter(f, fieldnames=fieldnames)
+
+            for metric in self.results['metrics']:
+                writer.writerow(metric)
+
 # Parse arguments.
 parser = argparse.ArgumentParser()
 parser.add_argument('-o', '--output', default='/result', dest='LAVA_PATH',
@@ -273,13 +310,18 @@ parser.add_argument('-o', '--output', default='/result', dest='LAVA_PATH',
                     specify a directory to store test and result files.
                     Default: /result
                     ''')
+parser.add_argument('-a', '--agenda', default=None, dest='agenda',
+                    help='''
+                    specify an agenda file which has tests and related
+                    params listed in yaml format.
+                    ''')
 parser.add_argument('-r', '--repo', dest='repo',
                     default='https://git.linaro.org/qa/test-definitions.git',
                     help='''
                     specify url or local path of test definitions repo.
                     Default: https://git.linaro.org/qa/test-definitions.git
                     ''')
-parser.add_argument('-d', '--test', required=True, dest='test_def',
+parser.add_argument('-d', '--test', default=None, dest='test_def',
                     help='''
                     base on test definition repo location, specify relative path
                     to the test definition to run.
@@ -295,10 +337,31 @@ args = parser.parse_args()
 
 # Obtain values from arguments.
 LAVA_PATH = args.LAVA_PATH
+agenda = args.agenda
 repo = args.repo
 test_def = args.test_def
 test_timeout = args.test_timeout
-skip_install = args.skip_install
+
+# Generate test list.
+if test_def:
+    test_list = [{'path': test_def}]
+elif agenda:
+    test_agenda = Agenda()
+    test_agenda.validate()
+    agenda_dict = test_agenda.agenda_dict()
+    repo = agenda_dict['repository']
+    test_list = agenda_dict['definitions']
+    for item in test_list:
+        if 'path' not in item:
+            print('Relative path is needed for each test')
+            sys.exit(1)
+else:
+    print('Plese specify either agenda file or test_def argument.')
+    sys.exit(1)
+
+print('--- Test list ---')
+for test in test_list:
+    print(test)
 
 # If repo points to remote url, clone the latest code.
 # If repo points to a local repo and it exists, use it.
@@ -306,53 +369,71 @@ if repo.startswith(('http', 'git', 'ssh')):
     repo_name = os.path.splitext(repo.split('/')[-1])[0]
     if os.path.exists(repo_name):
         shutil.rmtree(repo_name)
+    print('\nCloning %s' % repo)
     subprocess.call(['git', 'clone', repo])
     repo_path = os.path.realpath(repo_name)
-elif os.path.exists(repo):
-    repo_path = os.path.realpath(repo)
-elif not os.path.exists(repo):
-    print('%s NOT exists, exiting...' % repo)
-    sys.exit(1)
-
-test_def_path = os.path.join(repo_path, test_def)
-if not os.path.exists(test_def_path):
-    print(' %s NOT found, exiting...' % test_def_path)
-    sys.exit(1)
 else:
-    print('Test definition: %s' % test_def_path)
-
-# Fixup related variables.
-uuid = str(uuid4())
-test_name = os.path.splitext(test_def.split('/')[-1])[0]
-print('Test name: %s' % test_name)
-test_uuid = test_name + '_' + uuid
-print('Test UUID: %s' % test_uuid)
-bin_path = os.path.join(LAVA_PATH, 'bin')
-print('Binary path: %s' % bin_path)
-test_path = os.path.join(LAVA_PATH, 'tests', test_uuid)
-print('Test path: %s' % test_path)
-result_path = os.path.join(LAVA_PATH, 'results', test_uuid)
-print('Result path: %s' % result_path)
-
-# Create a hierarchy of directories and generate files needed.
-setup = TestSetup()
-setup.copy_test_repo()
-setup.create_test_runner_conf()
-setup.copy_bin_files()
-setup.create_uuid_file()
-
-# Convert test definition to the files needed by lava-test-runner.
-test_def = TestDefinition()
-test_def.definition()
-test_def.metadata()
-test_def.install()
-test_def.run()
-pattern = test_def.return_pattern()
-
-# Test run.
-test_run = TestRunner()
-test_run.check_output()
-
-# Parse test output, save results in json and csv format.
-result_parser = ResultPaser()
-result_parser.run()
+    if os.path.exists(repo):
+        repo_path = os.path.realpath(repo)
+    else:
+        print('%s NOT exists, exiting...' % repo)
+        sys.exit(1)
+
+# Run tests
+for test in test_list:
+    test_def = test['path']
+    test_def_path = os.path.join(repo_path, test_def)
+    if not os.path.exists(test_def_path):
+        print(' %s NOT found, exiting...' % test_def_path)
+        sys.exit(1)
+    else:
+        print('\nAbout to run: %s' % test_def_path)
+
+    if 'timeout' in test:
+        test_timeout = test['timeout']
+    skip_install = args.skip_install
+    if 'skip_install' in test:
+        skip_install = test['skip_install']
+    test_parameters = None
+    if 'parameters' in test:
+        test_parameters = test['parameters']
+    if 'params' in test:
+        test_parameters = test['params']
+    if test_parameters:
+        print('Test parameters from agenda file: %s' % test_parameters)
+
+    # Fixup related variables.
+    uuid = str(uuid4())
+    test_name = os.path.splitext(test_def.split('/')[-1])[0]
+    print('Test name: %s' % test_name)
+    test_uuid = test_name + '_' + uuid
+    print('Test UUID: %s' % test_uuid)
+    bin_path = os.path.join(LAVA_PATH, 'bin')
+    print('Binary path: %s' % bin_path)
+    test_path = os.path.join(LAVA_PATH, 'tests', test_uuid)
+    print('Test path: %s' % test_path)
+    result_path = os.path.join(LAVA_PATH, 'results', test_uuid)
+    print('Result path: %s' % result_path)
+
+    # Create a hierarchy of directories and generate files needed.
+    setup = TestSetup()
+    setup.copy_test_repo()
+    setup.create_test_runner_conf()
+    setup.copy_bin_files()
+    setup.create_uuid_file()
+
+    # Convert test definition to the files needed by lava-test-runner.
+    test_def = TestDefinition()
+    test_def.definition()
+    test_def.metadata()
+    test_def.install()
+    test_def.run()
+    pattern = test_def.return_pattern()
+
+    # Test run.
+    test_run = TestRunner()
+    test_run.check_output()
+
+    # Parse test output, save results in json and csv format.
+    result_parser = ResultPaser()
+    result_parser.run()