lava-pull.py: generate an HTML index file based on a template

The index file pulls in the text of template-summary.txt and
failing-tests.txt and provides a better summary of the test than
just using an apache-generated index page.

2 files changed, 120 insertions, 20 deletions

diff --git a/lava-pull/lava-pull.py b/lava-pull/lava-pull.py
index 2086e31..7f17448 100644
--- a/lava-pull/lava-pull.py
+++ b/lava-pull/lava-pull.py
@@ -2,6 +2,7 @@ import base64
 import fnmatch
 import json
 import os
+import string
 import subprocess
 import xmlrpclib
 
@@ -21,9 +22,16 @@ BUNDLES_ROOT = "/srv/tempest/bundles/"
 # other files will be written
 LOGS_ROOT = "/srv/tempest/logs/"
 
+# this is the path to the template file used to generate index.html
+# in each log directory
+INDEX_TEMPLATE_PATH = "/srv/tempest/template-index.html"
+
+# this is the subdirectory under the top-level log directory that will contain
+# all whitelisted files (see WHITELIST below)
+SUBDIR_WHITELIST = "logs"
+
 # this is the subdirectory under the top-level log directory that will contain
-# all files that are not deemed top-level important (see IMPORTANT_FILES
-# below)
+# all files that are not deemed top-level important (see WHITELIST below)
 SUBDIR_UNIMPORTANT = "xtra"
 
 # these are files that will be present in attachments for a test run that
@@ -53,6 +61,57 @@ WHITELIST = {
 
 
 #
+# create_dir - create a directory if it does not already exist
+#
+
+def create_dir(path):
+   if not os.path.exists(path):
+      os.makedirs(path)
+
+
+#
+# create_index - created an index.html for the logs directory, containing the
+# contents of tempest-summary.txt, if it exists
+#
+
+def create_index(logs_root_path, job_id, uploaded_on, attributes):
+   print "creating index page..."
+
+   tempest_summary_data = "Not Available"
+   tempest_summary_path = os.path.join(logs_root_path, SUBDIR_WHITELIST, "tempest-summary.txt")
+   if os.path.exists(tempest_summary_path):
+      print "tempest-summary.txt exists!"
+      with open(tempest_summary_path, "r") as f:
+         tempest_summary_data = f.read()
+
+   tempest_failing_tests = "Not Available"
+   tempest_failing_tests_path = os.path.join(logs_root_path, SUBDIR_WHITELIST, "failing-tests.txt")
+   if os.path.exists(tempest_failing_tests_path):
+      print "failing-tests.txt exists!"
+      with open(tempest_failing_tests_path, "r") as f:
+         tempest_failing_tests = f.read()
+
+   with open(INDEX_TEMPLATE_PATH, "r") as f:
+      template = string.Template(f.read())
+
+   output_string = template.safe_substitute(
+      JOB_ID=job_id,
+      UPLOADED_ON=uploaded_on,
+      WHITELIST_SUBDIR=SUBDIR_WHITELIST,
+      UNIMPORTANT_SUBDIR=SUBDIR_UNIMPORTANT,
+      OS_DISTRO = attributes["os-distro"],
+      OS_VERSION = attributes["os-version"],
+      DEVSTACK_BRANCH = attributes["devstack-branch"],
+      TEMPEST_SUMMARY = tempest_summary_data,
+      FAILING_TESTS = tempest_failing_tests
+   )
+
+   index_path = os.path.join(logs_root_path, "index.html")
+   with open(index_path, "w") as f:
+      f.write(output_string)
+
+
+#
 # process_bundle - process the bundle data, extracting and moving files to their
 # desired locations
 #
@@ -77,16 +136,20 @@ def process_bundle(sha1, job_id, uploaded_on, data):
       return
 
    # create the names for the root directories
-   bundle_root_path = os.path.join(LOGS_ROOT, bundle_id)
+   log_subdir = bundle_id
    if "os-distro" in attributes and "os-version" in attributes and "devstack-branch" in attributes:
-     bundle_root_path = "%s,os=%s,osver=%s,branch=%s" %   \
-        (bundle_root_path, attributes["os-distro"], attributes["os-version"], attributes["devstack-branch"])
-   bundle_unimp_path = os.path.join(bundle_root_path, "xtra")
-   print "storing output here: { root = '%s', unimportant = '%s' }" % (bundle_root_path, bundle_unimp_path)
-
-   # create the root directory
-   if not os.path.exists(bundle_root_path):
-     os.makedirs(bundle_root_path)
+      log_subdir = "%s,os=%s,osver=%s,branch=%s" %   \
+        (bundle_id, attributes["os-distro"], attributes["os-version"], attributes["devstack-branch"])
+   logs_root_path = os.path.join(LOGS_ROOT, log_subdir)
+   logs_whitelist_path = os.path.join(logs_root_path, SUBDIR_WHITELIST)
+   logs_unimp_path = os.path.join(logs_root_path, SUBDIR_UNIMPORTANT)
+   print "storing output here: { root = '%s', whitelist = '%s', unimportant = '%s' }" % \
+      (logs_root_path, logs_whitelist_path, logs_unimp_path)
+
+   # create the root and top-level subdirectories
+   create_dir(logs_root_path)
+   create_dir(logs_whitelist_path)
+   create_dir(logs_unimp_path)
 
    # loop through all of the tests in the bundle
    for test_run in obj["test_runs"]:
@@ -94,9 +157,8 @@ def process_bundle(sha1, job_id, uploaded_on, data):
       print "test [%s]:" % test_id
 
       # create directories if necessary
-      test_root_path = os.path.join(bundle_unimp_path, test_id)
-      if not os.path.exists(test_root_path):
-         os.makedirs(test_root_path)
+      test_root_path = os.path.join(logs_unimp_path, test_id)
+      create_dir(test_root_path)
 
       # see if there is a whitelist specified for the test
       test_whitelist = None
@@ -135,21 +197,22 @@ def process_bundle(sha1, job_id, uploaded_on, data):
                   # use the new name for the file
                   filename2 = matching_whitelist_filter['new-name']
                # build the full path for the whitelisted file
-               full_file_path = os.path.join(bundle_root_path, filename2)
+               full_file_path = os.path.join(logs_whitelist_path, filename2)
 
             # create the directory if necessary
             dir_name = os.path.dirname(full_file_path)
-            if not os.path.exists(dir_name):
-               print "    creating [%s]" % dir_name
-               os.makedirs(dir_name)
+            create_dir(dir_name)
 
             # finally - write the attachment
             with open(full_file_path, "w") as output_file:
                decoded_data = base64.b64decode(attachment["content"])
                output_file.write(decoded_data)
 
+   # create index page
+   create_index(logs_root_path, bundle_id, uploaded_on, attributes)
+
    # touch the directory with the original creation date
-   print subprocess.check_output(["touch", "--date=%s" % uploaded_on, bundle_root_path])
+   print subprocess.check_output(["touch", "--date=%s" % uploaded_on, logs_root_path])
 
 
 #
@@ -163,7 +226,7 @@ def retrieve_bundle(sha1, job_id, uploaded_on, output_path):
    bundle_data = server.dashboard.get(sha1)
    content = bundle_data["content"]
 
-   os.makedirs(output_path)
+   create_dir(output_path)
 
    full_bundle_path = os.path.join(output_path, "bundle.txt")
    with open(full_bundle_path, "w") as output_file:
diff --git a/lava-pull/template-index.html b/lava-pull/template-index.html
new file mode 100644
index 0000000..2ace039
--- /dev/null
+++ b/lava-pull/template-index.html
@@ -0,0 +1,37 @@
+<html>
+<head>
+  <link href='http://fonts.googleapis.com/css?family=Merriweather+Sans:400,700' rel='stylesheet' type='text/css'>
+
+  <style>
+   h1,h2,h3 { font-family: "Merriweather Sans", Arial, serif; font-weight: 700; }
+   BODY { font-family: "Merriweather Sans", Arial, serif; font-weight: 400; }
+  </style>
+
+</head>
+
+<body>
+<img src="http://www.linaro.org/app/images/linaro-logo-web.png"/>
+<h1>Linaro Openstack CI</h1>
+
+<h2>Tempest Test Run: $UPLOADED_ON (LAVA Job ID $JOB_ID)</h2>
+
+<p>The test run was performed on arm64 hardware using the following parameters:</p>
+<table>
+<tr><td>OS Distribution:</td><td><strong>$OS_DISTRO</strong></td></tr>
+<tr><td>OS Version:</td><td><strong>$OS_VERSION</strong></td></tr>
+<tr><td>Devstack Branch:</td><td><strong>$DEVSTACK_BRANCH</strong></td></tr>
+</table>
+
+<h3>Log Files</h3>
+<p>
+Click <a href="$WHITELIST_SUBDIR">here</a> for the <strong>Devstack</strong> and <strong>Tempest</strong> log files.  Additional <strong>LAVA-related</strong> log files are available <a href="$UNIMPORTANT_SUBDIR">here</a>.
+</p>
+
+<h3>Tempest Test Results Summary</h3>
+<pre>$TEMPEST_SUMMARY</pre>
+
+<h3>Failing Tests Summary</h3>
+<pre>$FAILING_TESTS</pre>
+
+</body>
+</html>