Add yaml parsing and improve local testing script

- Files are parsed and written as yaml, preserving whitespace and
  comments

When testing:
- Dump files to filesystem (tmp/)
- Loop through each device type
- Execute against local sources, rather than cloning
  remote repo when testing
- Reuse submit_for_testing.sh for local tests, too

This allows developers to see directly how template changes affect the
yaml job definitions.

This change introduces a requirement for python library ruamel.yaml.

Change-Id: Iefa751f5f29a2690e4e9905e12082f56ea6c9189
Signed-off-by: Dan Rue <dan.rue@linaro.org>

4 files changed, 91 insertions, 78 deletions

diff --git a/openembedded-lkft/.gitignore b/openembedded-lkft/.gitignore
new file mode 100644
index 0000000000..3fec32c842
--- /dev/null
+++ b/openembedded-lkft/.gitignore
@@ -0,0 +1 @@
+tmp/
diff --git a/openembedded-lkft/submit_for_testing.py b/openembedded-lkft/submit_for_testing.py
index e96fce5033..b88744a033 100644
--- a/openembedded-lkft/submit_for_testing.py
+++ b/openembedded-lkft/submit_for_testing.py
@@ -2,10 +2,12 @@ import argparse
 import os
 import requests
 import sys
+import StringIO
 from copy import deepcopy
 from string import Template
 from jinja2 import Environment, FileSystemLoader, StrictUndefined
-from jinja2.exceptions import TemplateNotFound
+from ruamel.yaml import YAML
+
 
 try:
     from urllib.parse import urlsplit
@@ -20,6 +22,20 @@ testplan_device_path = 'devices/'
 # Snapshots base URL
 snapshots_url = 'https://snapshots.linaro.org/openembedded/lkft'
 
+def parse_template(yaml_string):
+    '''
+        Round trip lava_job through ruamel to test parsing and
+        improve formatting. Comments are preserved.
+
+        In: yaml-formatted string
+        Out: validated yaml-formatted string
+    '''
+    yaml = YAML()
+    # ruamel does not provide a mechanism to dump to string, so use StringIO
+    # to catch it
+    output = StringIO.StringIO()
+    yaml.dump(yaml.load(yaml_string), output)
+    return output.getvalue()
 
 def _load_template(template_name, template_path, device_type):
     template = ''
@@ -139,9 +155,8 @@ def main():
                         nargs="+",
                         default=[])
     parser.add_argument("--dry-run",
-                        help="""Only prepare and print templates.
-                        Don't submit to actual servers.
-                        This option disables --quiet""",
+                        help="""Prepare and write templates to tmp/.
+                        Don't submit to actual servers.""",
                         action='store_true',
                         dest="dryrun")
     parser.add_argument("--quiet",
@@ -151,11 +166,12 @@ def main():
 
     args, _ = parser.parse_known_args()
 
+    output_path = "tmp"
     if args.dryrun:
-        # disable quiet when dryrun is enabled
-        args.quiet = False
+        if not os.path.exists(output_path):
+            os.mkdir(output_path)
     if args.qa_token is None:
-        print "QA_REPORTS_TOKEN is missing"
+        print("QA_REPORTS_TOKEN is missing")
         sys.exit(1)
 
     qa_server_base = args.qa_server
@@ -182,6 +198,7 @@ def main():
     template_base_post, _ = _load_template(args.template_base_post,
                                            args.template_path,
                                            args.device_type)
+    lava_jobs = []
     for test in args.template_names:
         test_template, template_file_name = _load_template(test,
                                                            args.template_path,
@@ -194,15 +211,17 @@ def main():
         template = Template(test_template)
         print("using template: %s" % template_file_name)
         lava_job = template.substitute(os.environ)
+        lava_job = parse_template(lava_job)
+        lava_jobs.append(lava_job)
+
         if not args.quiet:
             print(lava_job)
-        if not args.dryrun:
-            _submit_to_squad(lava_job,
-                lava_url_base,
-                qa_server_api,
-                qa_server_base,
-                args.qa_token,
-                args.quiet)
+        if args.dry_run:
+            testpath = os.path.join(output_path, args.device_type, test)
+            if not os.path.exists(os.path.dirname(testpath)):
+                os.makedirs(os.path.dirname(testpath))
+            with open(os.path.join(testpath), 'w') as f:
+                f.write(lava_job)
 
     THIS_DIR = os.path.abspath(args.testplan_path)
     # prevent creating templates when variables are missing
@@ -210,17 +229,22 @@ def main():
     context = deepcopy(os.environ)
     context.update({"device_type": os.path.join(args.testplan_device_path, args.device_type)})
     for test in args.test_plan:
-        lava_job = None
-        try:
-            lava_job = j2_env.get_template(test).render(context)
-        except TemplateNotFound as e:
-            print("Test plan or device_type not found")
-            print(e)
-
-        if lava_job is None:
-            continue
+        ''' Prepare lava jobs '''
+        lava_job = j2_env.get_template(test).render(context)
+        lava_job = parse_template(lava_job)
+        lava_jobs.append(lava_job)
+
         if not args.quiet:
             print(lava_job)
+        if args.dryrun:
+            testpath = os.path.join(output_path, args.device_type, test)
+            if not os.path.exists(os.path.dirname(testpath)):
+                os.makedirs(os.path.dirname(testpath))
+            with open(os.path.join(testpath), 'w') as f:
+                f.write(lava_job)
+
+    for lava_job in lava_jobs:
+        ''' Submit lava jobs '''
         if not args.dryrun:
             _submit_to_squad(lava_job,
                 lava_url_base,
@@ -229,5 +253,6 @@ def main():
                 args.qa_token,
                 args.quiet)
 
+
 if __name__ == "__main__":
     main()
diff --git a/openembedded-lkft/submit_for_testing.sh b/openembedded-lkft/submit_for_testing.sh
index a66f17396a..751949d685 100755
--- a/openembedded-lkft/submit_for_testing.sh
+++ b/openembedded-lkft/submit_for_testing.sh
@@ -6,30 +6,40 @@ set -ex
 [ -z "${LAVA_JOB_PRIORITY}" ] && export LAVA_JOB_PRIORITY="low"
 [ -z "${SKIP_LAVA}" ] || unset DEVICE_TYPE
 
+if [ -n "${DRY_RUN}" ]; then
+    export DRY_RUN="--dry-run --template-path lava-job-definitions --testplan-path lava-job-definitions/ --quiet"
+    export BASE_PATH=.
+else
+    export DRY_RUN=""
+    export BASE_PATH=configs/openembedded-lkft/
+fi
+
 if [ -z "${DEVICE_TYPE}" ]; then
     echo "DEVICE_TYPE not set. Exiting"
     exit 0
 fi
 
-case "${QA_SERVER_PROJECT}" in
-  linux-mainline-*)
-    source /srv/oe/build/lkftmetadata/packages/*/${KERNEL_RECIPE}/metadata
-    export KSELFTESTS_URL=${LINUX_GENERIC_MAINLINE_URL}
-    export KSELFTESTS_VERSION=${LINUX_GENERIC_MAINLINE_VERSION}
-    export KSELFTESTS_REVISION=${KERNEL_COMMIT}
-    ;;
-  linux-next-*)
-    source /srv/oe/build/lkftmetadata/packages/*/${KERNEL_RECIPE}/metadata
-    export KSELFTESTS_URL=${LINUX_GENERIC_NEXT_URL}
-    export KSELFTESTS_VERSION=${LINUX_GENERIC_NEXT_VERSION}
-    export KSELFTESTS_REVISION=${KERNEL_COMMIT}
-    ;;
-  *)
-    export KSELFTESTS_URL=${KSELFTESTS_MAINLINE_URL}
-    export KSELFTESTS_VERSION=${KSELFTESTS_MAINLINE_VERSION}
-    export KSELFTESTS_REVISION=${KSELFTESTS_MAINLINE_VERSION}
-    ;;
-esac
+if [ -z "${DRY_RUN}" ]; then
+    case "${QA_SERVER_PROJECT}" in
+      linux-mainline-*)
+        source /srv/oe/build/lkftmetadata/packages/*/${KERNEL_RECIPE}/metadata
+        export KSELFTESTS_URL=${LINUX_GENERIC_MAINLINE_URL}
+        export KSELFTESTS_VERSION=${LINUX_GENERIC_MAINLINE_VERSION}
+        export KSELFTESTS_REVISION=${KERNEL_COMMIT}
+        ;;
+      linux-next-*)
+        source /srv/oe/build/lkftmetadata/packages/*/${KERNEL_RECIPE}/metadata
+        export KSELFTESTS_URL=${LINUX_GENERIC_NEXT_URL}
+        export KSELFTESTS_VERSION=${LINUX_GENERIC_NEXT_VERSION}
+        export KSELFTESTS_REVISION=${KERNEL_COMMIT}
+        ;;
+      *)
+        export KSELFTESTS_URL=${KSELFTESTS_MAINLINE_URL}
+        export KSELFTESTS_VERSION=${KSELFTESTS_MAINLINE_VERSION}
+        export KSELFTESTS_REVISION=${KSELFTESTS_MAINLINE_VERSION}
+        ;;
+    esac
+fi
 
 if [ ! -z "${KERNEL_DESCRIBE}" ]; then
     export QA_BUILD_VERSION=${KERNEL_DESCRIBE}
@@ -37,16 +47,19 @@ else
     export QA_BUILD_VERSION=${KERNEL_COMMIT:0:12}
 fi
 
-rm -rf configs
-git clone --depth 1 http://git.linaro.org/ci/job/configs.git
+if [ -z "${DRY_RUN}" ]; then
+    rm -rf configs
+    git clone --depth 1 http://git.linaro.org/ci/job/configs.git
+fi
 
 [ ! -z ${TEST_TEMPLATES} ] && unset TEST_TEMPLATES
+TEMPLATE_PATH=""
 
-for test in $(ls configs/openembedded-lkft/lava-job-definitions/testplan); do
+for test in $(ls ${BASE_PATH}/lava-job-definitions/testplan/); do
     TEST_TEMPLATES="${TEST_TEMPLATES} testplan/${test}"
 done
 
-python configs/openembedded-lkft/submit_for_testing.py \
+python ${BASE_PATH}/submit_for_testing.py \
   --device-type ${DEVICE_TYPE} \
   --build-number ${BUILD_NUMBER} \
   --lava-server ${LAVA_SERVER} \
@@ -54,4 +67,5 @@ python configs/openembedded-lkft/submit_for_testing.py \
   --qa-server-team lkft \
   --qa-server-project ${QA_SERVER_PROJECT} \
   --git-commit ${QA_BUILD_VERSION} \
+  ${DRY_RUN} \
   --test-plan ${TEST_TEMPLATES}
diff --git a/openembedded-lkft/test_submit_for_testing.sh b/openembedded-lkft/test_submit_for_testing.sh
index 7129106a33..443fced6f6 100755
--- a/openembedded-lkft/test_submit_for_testing.sh
+++ b/openembedded-lkft/test_submit_for_testing.sh
@@ -2,7 +2,7 @@
 
 virtualenv .venv
 source .venv/bin/activate
-pip install Jinja2 requests urllib3
+pip install Jinja2 requests urllib3 ruamel.yaml
 
 export BASE_URL=http://snapshots.linaro.org
 export PUB_DEST=openembedded/lkft/morty/hikey/rpb/linux-mainline/346
@@ -55,40 +55,13 @@ export DEVICE_TYPE="x86"
 export KSELFTEST_SKIPLIST="pstore"
 export QA_BUILD_VERSION=${KERNEL_DESCRIBE}
 
-[ -z "${KSELFTEST_PATH}" ] && export KSELFTEST_PATH="/opt/kselftests/mainline/"
-[ -z "${LAVA_JOB_PRIORITY}" ] && export LAVA_JOB_PRIORITY="low"
-[ -z "${SKIP_LAVA}" ] || unset DEVICE_TYPE
+export DRY_RUN=true
 
-if [ -z "${DEVICE_TYPE}" ]; then
-    echo "DEVICE_TYPE not set. Exiting"
-    exit 0
-fi
-
-if [ ! -z "${KERNEL_DESCRIBE}" ]; then
-    export QA_BUILD_VERSION=${KERNEL_DESCRIBE}
-else
-    export QA_BUILD_VERSION=${KERNEL_COMMIT:0:12}
-fi
-
-[ ! -z ${TEST_TEMPLATES} ] && unset TEST_TEMPLATES
-
-for test in $(ls lava-job-definitions/testplan); do
-    TEST_TEMPLATES="${TEST_TEMPLATES} testplan/${test}"
+for device in $(ls lava-job-definitions/devices); do
+    export DEVICE_TYPE=$device
+    bash submit_for_testing.sh
 done
 
-[ -z "${DEVICE_TYPE}" ] || \
-python submit_for_testing.py \
-  --device-type ${DEVICE_TYPE} \
-  --build-number ${BUILD_NUMBER} \
-  --lava-server ${LAVA_SERVER} \
-  --qa-server ${QA_SERVER} \
-  --qa-server-team lkft \
-  --qa-server-project ${QA_SERVER_PROJECT} \
-  --git-commit ${QA_BUILD_VERSION} \
-  --test-plan ${TEST_TEMPLATES} \
-  --testplan-path lava-job-definitions \
-  --dry-run
-
 # cleanup virtualenv
 deactivate
 rm -rf .venv