diff options
Diffstat (limited to 'lava-pull/lava-pull.py')
| -rw-r--r-- | lava-pull/lava-pull.py | 103 |
1 files changed, 83 insertions, 20 deletions
diff --git a/lava-pull/lava-pull.py b/lava-pull/lava-pull.py index 2086e31..7f17448 100644 --- a/lava-pull/lava-pull.py +++ b/lava-pull/lava-pull.py @@ -2,6 +2,7 @@ import base64 import fnmatch import json import os +import string import subprocess import xmlrpclib @@ -21,9 +22,16 @@ BUNDLES_ROOT = "/srv/tempest/bundles/" # other files will be written LOGS_ROOT = "/srv/tempest/logs/" +# this is the path to the template file used to generate index.html +# in each log directory +INDEX_TEMPLATE_PATH = "/srv/tempest/template-index.html" + +# this is the subdirectory under the top-level log directory that will contain +# all whitelisted files (see WHITELIST below) +SUBDIR_WHITELIST = "logs" + # this is the subdirectory under the top-level log directory that will contain -# all files that are not deemed top-level important (see IMPORTANT_FILES -# below) +# all files that are not deemed top-level important (see WHITELIST below) SUBDIR_UNIMPORTANT = "xtra" # these are files that will be present in attachments for a test run that @@ -53,6 +61,57 @@ WHITELIST = { # +# create_dir - create a directory if it does not already exist +# + +def create_dir(path): + if not os.path.exists(path): + os.makedirs(path) + + +# +# create_index - created an index.html for the logs directory, containing the +# contents of tempest-summary.txt, if it exists +# + +def create_index(logs_root_path, job_id, uploaded_on, attributes): + print "creating index page..." + + tempest_summary_data = "Not Available" + tempest_summary_path = os.path.join(logs_root_path, SUBDIR_WHITELIST, "tempest-summary.txt") + if os.path.exists(tempest_summary_path): + print "tempest-summary.txt exists!" + with open(tempest_summary_path, "r") as f: + tempest_summary_data = f.read() + + tempest_failing_tests = "Not Available" + tempest_failing_tests_path = os.path.join(logs_root_path, SUBDIR_WHITELIST, "failing-tests.txt") + if os.path.exists(tempest_failing_tests_path): + print "failing-tests.txt exists!" + with open(tempest_failing_tests_path, "r") as f: + tempest_failing_tests = f.read() + + with open(INDEX_TEMPLATE_PATH, "r") as f: + template = string.Template(f.read()) + + output_string = template.safe_substitute( + JOB_ID=job_id, + UPLOADED_ON=uploaded_on, + WHITELIST_SUBDIR=SUBDIR_WHITELIST, + UNIMPORTANT_SUBDIR=SUBDIR_UNIMPORTANT, + OS_DISTRO = attributes["os-distro"], + OS_VERSION = attributes["os-version"], + DEVSTACK_BRANCH = attributes["devstack-branch"], + TEMPEST_SUMMARY = tempest_summary_data, + FAILING_TESTS = tempest_failing_tests + ) + + index_path = os.path.join(logs_root_path, "index.html") + with open(index_path, "w") as f: + f.write(output_string) + + +# # process_bundle - process the bundle data, extracting and moving files to their # desired locations # @@ -77,16 +136,20 @@ def process_bundle(sha1, job_id, uploaded_on, data): return # create the names for the root directories - bundle_root_path = os.path.join(LOGS_ROOT, bundle_id) + log_subdir = bundle_id if "os-distro" in attributes and "os-version" in attributes and "devstack-branch" in attributes: - bundle_root_path = "%s,os=%s,osver=%s,branch=%s" % \ - (bundle_root_path, attributes["os-distro"], attributes["os-version"], attributes["devstack-branch"]) - bundle_unimp_path = os.path.join(bundle_root_path, "xtra") - print "storing output here: { root = '%s', unimportant = '%s' }" % (bundle_root_path, bundle_unimp_path) - - # create the root directory - if not os.path.exists(bundle_root_path): - os.makedirs(bundle_root_path) + log_subdir = "%s,os=%s,osver=%s,branch=%s" % \ + (bundle_id, attributes["os-distro"], attributes["os-version"], attributes["devstack-branch"]) + logs_root_path = os.path.join(LOGS_ROOT, log_subdir) + logs_whitelist_path = os.path.join(logs_root_path, SUBDIR_WHITELIST) + logs_unimp_path = os.path.join(logs_root_path, SUBDIR_UNIMPORTANT) + print "storing output here: { root = '%s', whitelist = '%s', unimportant = '%s' }" % \ + (logs_root_path, logs_whitelist_path, logs_unimp_path) + + # create the root and top-level subdirectories + create_dir(logs_root_path) + create_dir(logs_whitelist_path) + create_dir(logs_unimp_path) # loop through all of the tests in the bundle for test_run in obj["test_runs"]: @@ -94,9 +157,8 @@ def process_bundle(sha1, job_id, uploaded_on, data): print "test [%s]:" % test_id # create directories if necessary - test_root_path = os.path.join(bundle_unimp_path, test_id) - if not os.path.exists(test_root_path): - os.makedirs(test_root_path) + test_root_path = os.path.join(logs_unimp_path, test_id) + create_dir(test_root_path) # see if there is a whitelist specified for the test test_whitelist = None @@ -135,21 +197,22 @@ def process_bundle(sha1, job_id, uploaded_on, data): # use the new name for the file filename2 = matching_whitelist_filter['new-name'] # build the full path for the whitelisted file - full_file_path = os.path.join(bundle_root_path, filename2) + full_file_path = os.path.join(logs_whitelist_path, filename2) # create the directory if necessary dir_name = os.path.dirname(full_file_path) - if not os.path.exists(dir_name): - print " creating [%s]" % dir_name - os.makedirs(dir_name) + create_dir(dir_name) # finally - write the attachment with open(full_file_path, "w") as output_file: decoded_data = base64.b64decode(attachment["content"]) output_file.write(decoded_data) + # create index page + create_index(logs_root_path, bundle_id, uploaded_on, attributes) + # touch the directory with the original creation date - print subprocess.check_output(["touch", "--date=%s" % uploaded_on, bundle_root_path]) + print subprocess.check_output(["touch", "--date=%s" % uploaded_on, logs_root_path]) # @@ -163,7 +226,7 @@ def retrieve_bundle(sha1, job_id, uploaded_on, output_path): bundle_data = server.dashboard.get(sha1) content = bundle_data["content"] - os.makedirs(output_path) + create_dir(output_path) full_bundle_path = os.path.join(output_path, "bundle.txt") with open(full_bundle_path, "w") as output_file: |