From e5e851f28128f39804f201b65deaa13cdc723a9c Mon Sep 17 00:00:00 2001 From: Clark Laughlin Date: Tue, 28 Apr 2015 21:06:23 +0000 Subject: lava-pull.py: generate an HTML index file based on a template The index file pulls in the text of template-summary.txt and failing-tests.txt and provides a better summary of the test than just using an apache-generated index page. --- lava-pull/lava-pull.py | 103 ++++++++++++++++++++++++++++++++++-------- lava-pull/template-index.html | 37 +++++++++++++++ 2 files changed, 120 insertions(+), 20 deletions(-) create mode 100644 lava-pull/template-index.html diff --git a/lava-pull/lava-pull.py b/lava-pull/lava-pull.py index 2086e31..7f17448 100644 --- a/lava-pull/lava-pull.py +++ b/lava-pull/lava-pull.py @@ -2,6 +2,7 @@ import base64 import fnmatch import json import os +import string import subprocess import xmlrpclib @@ -21,9 +22,16 @@ BUNDLES_ROOT = "/srv/tempest/bundles/" # other files will be written LOGS_ROOT = "/srv/tempest/logs/" +# this is the path to the template file used to generate index.html +# in each log directory +INDEX_TEMPLATE_PATH = "/srv/tempest/template-index.html" + +# this is the subdirectory under the top-level log directory that will contain +# all whitelisted files (see WHITELIST below) +SUBDIR_WHITELIST = "logs" + # this is the subdirectory under the top-level log directory that will contain -# all files that are not deemed top-level important (see IMPORTANT_FILES -# below) +# all files that are not deemed top-level important (see WHITELIST below) SUBDIR_UNIMPORTANT = "xtra" # these are files that will be present in attachments for a test run that @@ -52,6 +60,57 @@ WHITELIST = { } +# +# create_dir - create a directory if it does not already exist +# + +def create_dir(path): + if not os.path.exists(path): + os.makedirs(path) + + +# +# create_index - created an index.html for the logs directory, containing the +# contents of tempest-summary.txt, if it exists +# + +def create_index(logs_root_path, job_id, uploaded_on, attributes): + print "creating index page..." + + tempest_summary_data = "Not Available" + tempest_summary_path = os.path.join(logs_root_path, SUBDIR_WHITELIST, "tempest-summary.txt") + if os.path.exists(tempest_summary_path): + print "tempest-summary.txt exists!" + with open(tempest_summary_path, "r") as f: + tempest_summary_data = f.read() + + tempest_failing_tests = "Not Available" + tempest_failing_tests_path = os.path.join(logs_root_path, SUBDIR_WHITELIST, "failing-tests.txt") + if os.path.exists(tempest_failing_tests_path): + print "failing-tests.txt exists!" + with open(tempest_failing_tests_path, "r") as f: + tempest_failing_tests = f.read() + + with open(INDEX_TEMPLATE_PATH, "r") as f: + template = string.Template(f.read()) + + output_string = template.safe_substitute( + JOB_ID=job_id, + UPLOADED_ON=uploaded_on, + WHITELIST_SUBDIR=SUBDIR_WHITELIST, + UNIMPORTANT_SUBDIR=SUBDIR_UNIMPORTANT, + OS_DISTRO = attributes["os-distro"], + OS_VERSION = attributes["os-version"], + DEVSTACK_BRANCH = attributes["devstack-branch"], + TEMPEST_SUMMARY = tempest_summary_data, + FAILING_TESTS = tempest_failing_tests + ) + + index_path = os.path.join(logs_root_path, "index.html") + with open(index_path, "w") as f: + f.write(output_string) + + # # process_bundle - process the bundle data, extracting and moving files to their # desired locations @@ -77,16 +136,20 @@ def process_bundle(sha1, job_id, uploaded_on, data): return # create the names for the root directories - bundle_root_path = os.path.join(LOGS_ROOT, bundle_id) + log_subdir = bundle_id if "os-distro" in attributes and "os-version" in attributes and "devstack-branch" in attributes: - bundle_root_path = "%s,os=%s,osver=%s,branch=%s" % \ - (bundle_root_path, attributes["os-distro"], attributes["os-version"], attributes["devstack-branch"]) - bundle_unimp_path = os.path.join(bundle_root_path, "xtra") - print "storing output here: { root = '%s', unimportant = '%s' }" % (bundle_root_path, bundle_unimp_path) - - # create the root directory - if not os.path.exists(bundle_root_path): - os.makedirs(bundle_root_path) + log_subdir = "%s,os=%s,osver=%s,branch=%s" % \ + (bundle_id, attributes["os-distro"], attributes["os-version"], attributes["devstack-branch"]) + logs_root_path = os.path.join(LOGS_ROOT, log_subdir) + logs_whitelist_path = os.path.join(logs_root_path, SUBDIR_WHITELIST) + logs_unimp_path = os.path.join(logs_root_path, SUBDIR_UNIMPORTANT) + print "storing output here: { root = '%s', whitelist = '%s', unimportant = '%s' }" % \ + (logs_root_path, logs_whitelist_path, logs_unimp_path) + + # create the root and top-level subdirectories + create_dir(logs_root_path) + create_dir(logs_whitelist_path) + create_dir(logs_unimp_path) # loop through all of the tests in the bundle for test_run in obj["test_runs"]: @@ -94,9 +157,8 @@ def process_bundle(sha1, job_id, uploaded_on, data): print "test [%s]:" % test_id # create directories if necessary - test_root_path = os.path.join(bundle_unimp_path, test_id) - if not os.path.exists(test_root_path): - os.makedirs(test_root_path) + test_root_path = os.path.join(logs_unimp_path, test_id) + create_dir(test_root_path) # see if there is a whitelist specified for the test test_whitelist = None @@ -135,21 +197,22 @@ def process_bundle(sha1, job_id, uploaded_on, data): # use the new name for the file filename2 = matching_whitelist_filter['new-name'] # build the full path for the whitelisted file - full_file_path = os.path.join(bundle_root_path, filename2) + full_file_path = os.path.join(logs_whitelist_path, filename2) # create the directory if necessary dir_name = os.path.dirname(full_file_path) - if not os.path.exists(dir_name): - print " creating [%s]" % dir_name - os.makedirs(dir_name) + create_dir(dir_name) # finally - write the attachment with open(full_file_path, "w") as output_file: decoded_data = base64.b64decode(attachment["content"]) output_file.write(decoded_data) + # create index page + create_index(logs_root_path, bundle_id, uploaded_on, attributes) + # touch the directory with the original creation date - print subprocess.check_output(["touch", "--date=%s" % uploaded_on, bundle_root_path]) + print subprocess.check_output(["touch", "--date=%s" % uploaded_on, logs_root_path]) # @@ -163,7 +226,7 @@ def retrieve_bundle(sha1, job_id, uploaded_on, output_path): bundle_data = server.dashboard.get(sha1) content = bundle_data["content"] - os.makedirs(output_path) + create_dir(output_path) full_bundle_path = os.path.join(output_path, "bundle.txt") with open(full_bundle_path, "w") as output_file: diff --git a/lava-pull/template-index.html b/lava-pull/template-index.html new file mode 100644 index 0000000..2ace039 --- /dev/null +++ b/lava-pull/template-index.html @@ -0,0 +1,37 @@ + + + + + + + + + + +

Linaro Openstack CI

+ +

Tempest Test Run: $UPLOADED_ON (LAVA Job ID $JOB_ID)

+ +

The test run was performed on arm64 hardware using the following parameters:

+ + + + +
OS Distribution:$OS_DISTRO
OS Version:$OS_VERSION
Devstack Branch:$DEVSTACK_BRANCH
+ +

Log Files

+

+Click here for the Devstack and Tempest log files. Additional LAVA-related log files are available here. +

+ +

Tempest Test Results Summary

+
$TEMPEST_SUMMARY
+ +

Failing Tests Summary

+
$FAILING_TESTS
+ + + -- cgit v1.2.3