cleanup

2 files changed, 0 insertions, 335 deletions

diff --git a/tempest-pull/tempest-lava-pull.py b/tempest-pull/tempest-lava-pull.py
deleted file mode 100644
index aa74c50..0000000
--- a/tempest-pull/tempest-lava-pull.py
+++ /dev/null
@@ -1,149 +0,0 @@
-import gc
-import os
-import string
-
-from py2neo import Graph, Node, Relationship
-from lava import LAVADashboard
-from bundle import BundleStore
-from bundle import Bundle
-
-# this is the URL of the LAVA instance to retrieve from
-LAVA_XMLRPC_ENDPOINT = "https://openstack.validation.linaro.org/RPC2"
-
-# this is the URL of the Neo4J instance
-NEO4J_ENDPOINT = "http://neo4j:linaro@localhost:7474/db/data/"
-
-# this is the name of the bundle stream to retrieve from
-BUNDLE_STREAM_NAME = "/public/team/openstack/tempest-ci/"
-
-# this is the root location where the raw bundle JSON will be downloaded
-# and stored.  Each bundle will be stored as a single .txt file inside
-# a directory named with the bundle UUID.  Any bundle that already has a
-# directory present will be skipped on subsequent runs.
-BUNDLES_ROOT = os.environ["LAVA_PULL_BUNDLEROOT"] #"/srv/tempest/bundles/"
-
-# this is the root location where the bundle's extracted log data and
-# other files will be written
-LOGS_ROOT = os.environ["LAVA_PULL_LOGROOT"] #"/srv/tempest/logs/"
-
-# these are the important files that are to be brought to the top-level of
-# the output directory
-WHITELIST = {
-   'devstack' : [
-       { 'src' : "stdout.log", 'new-name' : "stdout-devstack.log" }
-   ],
-   'tempest-summary' : [
-       { 'src' : "tempest-summary.txt" }
-   ],
-   'tempest-run' : [
-       { 'src' : "*.log.gz" },
-       { 'src' : "*.txt.gz" },
-       { 'src' : "apache2/*" },
-       { 'src' : "libvirt/*" },
-       { 'src' : "all-tests.txt" },
-       { 'src' : "results.subunit" },
-       { 'src' : "stack.sh.summary.gz" },
-       { 'src' : "stdout.log", 'new-name' : "stdout-tempest.log" },
-       { 'src' : "tempest_conf.txt" }
-   ]
-}
-
-class Neo4JDatabase(object):
-   def __init__(self, endpoint):
-      self.endpoint = endpoint
-
-   def store_bundle(self, bundle):
-      if not isinstance(bundle, Bundle):
-         raise Exception("argument is not a Bundle")
-
-      graph = Graph(self.endpoint)
-
-      print "creating graph for bundle"
-
-      # find (or create) the OS version + distro node
-      os_version = bundle.metadata["lava_job_attributes"]["os-version"]
-      os_distro = bundle.metadata["lava_job_attributes"]["os-distro"]
-      os_name = "%s/%s" % (os_distro, os_version)
-      OS_node = graph.find_one("OS", "name", os_name)
-      if not OS_node:
-         OS_node = Node("OS", name=os_name, distro=os_distro, version=os_version)
-         graph.create(OS_node)
-
-      # find (or create) the devstack branch  node
-      devstack_branch = bundle.metadata["lava_job_attributes"]["devstack-branch"]
-      Branch_node = graph.find_one("Devstack", "name", devstack_branch)
-      if not Branch_node:
-         Branch_node = Node("Devstack", name=devstack_branch)
-         graph.create(Branch_node)
-
-      # create the main tempest run node and associate with the OS and Branch
-      TempestRun_node = Node("TempestRun",          \
-         date = bundle.metadata["date_uploaded"],     \
-         lava_job = bundle.metadata["lava_job_id"],   \
-         sha1 = bundle.metadata["bundle_sha1"])
-      if bundle.all_tests:
-         TempestRun_node.properties["all_tests"] = len(bundle.all_tests)
-      if bundle.tests_run:
-         TempestRun_node.properties["tests_run"] = len(bundle.tests_run)
-      if bundle.failing_tests:
-         TempestRun_node.properties["failing_tests"] = len(bundle.failing_tests)
-      if bundle.skipped_tests:
-         TempestRun_node.properties["skipped_tests"] = len(bundle.skipped_tests)
-      if bundle.passing_tests:
-         TempestRun_node.properties["passing_tests"] = len(bundle.passing_tests)
-      OS_relationship = Relationship(TempestRun_node, "ON", OS_node)
-      Branch_relationship = Relationship(TempestRun_node, "USING", Branch_node)
-      graph.create(TempestRun_node, OS_relationship, Branch_relationship)
-
-      # create all of the tests and relate them back to the tempest node
-      for test_set in [bundle.failing_tests, bundle.passing_tests, bundle.skipped_tests]:
-         if test_set:
-            print "adding tests"
-            for test in test_set:
-               Test_node = Node("Test", test["status"],  \
-                  name=test["test"], \
-                  status=test["status"], \
-                  start_time=test["start_time"], \
-                  stop_time=test["stop_time"], \
-                  test_class=test["class"])
-               Test_relationship = Relationship(TempestRun_node, \
-                  "HAS_TEST", Test_node, status=test["status"])
-               graph.create(Test_node, Test_relationship)
-
-
-
-
-def main():
-   lava = LAVADashboard(LAVA_XMLRPC_ENDPOINT, BUNDLE_STREAM_NAME)
-   lava.connect()
-
-   store = BundleStore(BUNDLES_ROOT)
-   store_sha1_list = store.bundle_list()
-
-   bundle_sha1_list, bundle_list = lava.server_bundle_list()
-   for entry in bundle_list:
-      sha1 = entry["content_sha1"]
-      if sha1 in store_sha1_list:
-         print "[%s] skipping, already processed" % sha1
-         continue
-
-      print "------------------------------------------------------------------------------------------------------"
-
-      print "downloading new entry:"
-      print entry
-
-      bundle = lava.retrieve_bundle(entry)
-
-      print "[%s]:" % sha1
-      
-      metadata = bundle.expand(WHITELIST, LOGS_ROOT)
-
-      database = Neo4JDatabase(NEO4J_ENDPOINT)
-      database.store_bundle(bundle)
-
-      store.write_bundle_receipt(bundle)
-      del bundle
-      gc.collect()
-
-
-main()
diff --git a/tempest-pull/upload_testresults.py b/tempest-pull/upload_testresults.py
deleted file mode 100644
index c7edff4..0000000
--- a/tempest-pull/upload_testresults.py
+++ /dev/null
@@ -1,186 +0,0 @@
-import couchdb
-import csv
-import argparse
-import subprocess
-import string
-import os
-import StringIO
-import re
-from uuid import uuid4
-import datetime
-
-COUCHDB_USERNAME = 'data_upload'
-COUCHDB_PASSWORD = 'linaro'
-
-# clean up a test id
-def clean_test_id(test_id):
-    if '[' in test_id:
-        test_id = test_id[:test_id.index('[')]
-    return re.sub('[^-0-9A-Za-z_.]', '-', test_id)
-
-
-# extract a "class name" from a test id
-def get_class_name(test_id, class_depth):
-    clean_name = clean_test_id(test_id)
-    words = clean_name.split(".")
-    return ".".join(words[1:class_depth])
-
-
-# convert subunit2csv data to json
-def csv_to_json(input_csv):
-    f = StringIO.StringIO(input_csv)    
-    reader = csv.DictReader(f)
-    result = []
-    for row in reader:
-	test_id = row["test"]
-	# clean up the test name (get rid of the [uuid]) part
-        row["test"] = clean_test_id(test_id)
-        # add an entry for the class
-        row["class"] = get_class_name(test_id, 3)
-	result.append(row)
-    return result
-
-
-def get_tests_run(subunit_stream):
-    p1 = subprocess.Popen(["cat", subunit_stream], stdout=subprocess.PIPE)
-    p2 = subprocess.Popen(["subunit2csv", "--no-passthrough"], stdin=p1.stdout, stdout=subprocess.PIPE)
-    p1.stdout.close()
-    output = p2.communicate()[0]
-    return csv_to_json(output)
-
-
-def get_failing_tests(subunit_stream):
-    p1 = subprocess.Popen(["cat", subunit_stream], stdout = subprocess.PIPE)
-    p2 = subprocess.Popen(["subunit-filter", "--only-genuine-failures", "--no-passthrough"], stdin=p1.stdout, stdout=subprocess.PIPE)
-    p3 = subprocess.Popen(["subunit2csv", "--no-passthrough"], stdin=p2.stdout, stdout=subprocess.PIPE)
-    p1.stdout.close()
-    p2.stdout.close()
-    output = p3.communicate()[0]
-    return csv_to_json(output)
-
-
-def get_failing_tests_xml(subunit_stream):
-    p1 = subprocess.Popen(["cat", subunit_stream], stdout = subprocess.PIPE)
-    p2 = subprocess.Popen(["subunit-filter", "--only-genuine-failures", "--passthrough"], stdin=p1.stdout, stdout=subprocess.PIPE)
-    p3 = subprocess.Popen(["subunit2junitxml"], stdin=p2.stdout, stdout=subprocess.PIPE)
-    p1.stdout.close()
-    p2.stdout.close()
-    output = p3.communicate()[0]
-    return output
-
-
-def get_passing_tests(subunit_stream):
-    p1 = subprocess.Popen(["cat", subunit_stream], stdout = subprocess.PIPE)
-    p2 = subprocess.Popen(["subunit-filter", "--no-skip", "--no-failure", "--success", "--no-passthrough"], stdin=p1.stdout, stdout=subprocess.PIPE)
-    p3 = subprocess.Popen(["subunit2csv", "--no-passthrough"], stdin=p2.stdout, stdout=subprocess.PIPE)
-    p1.stdout.close()
-    p2.stdout.close()
-    output = p3.communicate()[0]
-    return csv_to_json(output)
-
-
-def get_skipped_tests(subunit_stream):
-    p1 = subprocess.Popen(["cat", subunit_stream], stdout = subprocess.PIPE)
-    p2 = subprocess.Popen(["subunit-filter", "--no-error", "--no-failure", "--no-success", "--no-xfail", "--no-passthrough"], stdin=p1.stdout, stdout=subprocess.PIPE)
-    p3 = subprocess.Popen(["subunit2csv", "--no-passthrough"], stdin=p2.stdout, stdout=subprocess.PIPE)
-    p1.stdout.close()
-    p2.stdout.close()
-    output = p3.communicate()[0]
-    return csv_to_json(output)
-
-
-def get_all_tests(all_tests):
-    with open(all_tests) as f:
-	return [clean_test_id(line.rstrip('\n')) for line in f]
-
-
-def create_base_document(metadata):
-    doc = dict()
-
-    # test data that needs to come from the bundle stream
-    doc['os_distro'] = 'ubuntu'
-    doc['os_version'] = 'trusty'
-    doc['devstack_branch'] = 'master'
-    doc['lava_job_id'] = '624'
-    doc['networking'] = 'nova-network'
-    doc['lava_bundle_stream'] = 'http://openstack.validation.linaro.org/dashboard/permalink/bundle/1bb4bf811506d95b2e9c09d88d05197750672902/'
-    doc['date'] = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
-
-    # create a document id, and create the base set of tags for searching
-    doc['_id'] = uuid4().hex
-    doc['tags'] = [ doc['os_distro'], doc['os_version'], doc['devstack_branch'], doc['networking'] ]
-    return doc
-
-
-
-def create_test_run_document(metadata, log_files, all_tests, tests_run, skipped_tests, failing_tests, passing_tests):
-    doc = create_base_document(metadata)
-    doc['$type'] = 'test-run'
-    
-    summary_stats = dict()
-    summary_stats['total_tests'] = len(all_tests)
-    summary_stats['tests_run'] = len(tests_run)
-    summary_stats['skipped_tests'] = len(skipped_tests)
-    summary_stats['passing_tests'] = len(passing_tests)
-    summary_stats['failing_tests'] = len(failing_tests)
-    doc['summary'] = summary_stats
-    doc['discovered_tests'] = all_tests
-
-    server = couchdb.Server()
-    server.resource.credentials = (COUCHDB_USERNAME, COUCHDB_PASSWORD)
-    doc_id, rev_id = server['tempest_results'].save(doc)
-    print "test run summary document created [%s, %s]" % (doc_id, rev_id)
-    
-    return doc_id
-
-
-def create_test_document_helper(server, metadata, summary_doc_id, test):
-    doc = create_base_document(metadata)
-    doc['$type'] = 'test'
-    doc['$test_run'] = summary_doc_id
-    doc['tags'].append(test['status'])
-    doc.update(test)
-    doc_id, rev_id = server['tempest_results'].save(doc)
-    return doc_id, rev_id
-
-
-
-def create_test_documents(metadata, summary_doc_id, skipped_tests, failing_tests, passing_tests):
-    server = couchdb.Server()
-    server.resource.credentials = (COUCHDB_USERNAME, COUCHDB_PASSWORD)
-    print "creating passing test documents:"
-    for t in passing_tests:
-       doc_id, rev_id = create_test_document_helper(server, metadata, summary_doc_id, t)
-       print ": [%s, %s]" % (doc_id, rev_id)
-    print "creating failing test documents:"
-    for t in failing_tests:
-       doc_id, rev_id = create_test_document_helper(server, metadata, summary_doc_id, t)
-       print ": [%s, %s]" % (doc_id, rev_id)
-    print "creating skipped test documents:"
-    for t in skipped_tests:
-       doc_id, rev_id = create_test_document_helper(server, metadata, summary_doc_id, t)
-       print ": [%s, %s]" % (doc_id, rev_id)
-
-
-
-# parse arguments, then process the data
-
-parser = argparse.ArgumentParser()
-parser.add_argument('--stream', help='subunit result stream', required=True)
-parser.add_argument('--all_tests', help='text file containing list of all discovered tests', \
-    required=True)
-args = parser.parse_args()
-
-print "subunit stream = %s" % args.stream
-print "discovered tests list = %s" % args.all_tests
-
-
-failing_tests = get_failing_tests(args.stream)
-passing_tests = get_passing_tests(args.stream)
-skipped_tests = get_skipped_tests(args.stream)
-all_tests = get_all_tests(args.all_tests)
-tests_run = get_tests_run(args.stream)
-
-doc_id = create_test_run_document(None, None, all_tests, tests_run, skipped_tests, failing_tests, passing_tests)
-create_test_documents(None, doc_id, skipped_tests, failing_tests, passing_tests)
-