starting using virtualenv, new graph library

4 files changed, 53 insertions, 45 deletions

diff --git a/tempest-pull/bundle.py b/tempest-pull/bundle.py
index a489fa5..6e786e4 100644
--- a/tempest-pull/bundle.py
+++ b/tempest-pull/bundle.py
@@ -116,7 +116,7 @@ class Bundle(object):
       # loop through all of the tests in the bundle
       for test_run in self.data["test_runs"]:
 	 test_id = test_run["test_id"]
-	 print "test [%s]:" % test_id
+	 print "processing test [%s]" % test_id
 
 	 # create directories if necessary
 	 test_root_path = os.path.join(unimp_path, test_id)
@@ -133,18 +133,13 @@ class Bundle(object):
 	       matching_whitelist_filter = None
 
 	       filename = attachment["pathname"]
-	       print "- %s" % filename
-
-	       if filename in self.skip_files:
-		  print "    skipping!"
-	       else:
+	       if not filename in self.skip_files:
 		  if test_whitelist:
 		     # see if the file matches one of the whitelist patterns
 		     for filter in test_whitelist:
 			pattern = filter["src"]
 			if fnmatch.fnmatch(filename, pattern):
 			   matching_whitelist_filter = filter
-			   print "    whitelist: [%s] using pattern [%s]" % (filename, pattern)
 			   break
 
 	       # build the full path to the output file, assuming it
@@ -181,8 +176,7 @@ class Bundle(object):
 	 json.dump(bundle_metadata, f)
 
       # touch the directory with the original creation date
-      print "setting date on [%s]" % full_root_path
-      print subprocess.check_output(["touch", "--date=%s" % self.upload_date, full_root_path])
+      subprocess.check_output(["touch", "--date=%s" % self.upload_date, full_root_path])
 
       self.metadata = bundle_metadata
 
diff --git a/tempest-pull/run b/tempest-pull/run
index a0da69d..11c5ca4 100644..100755
--- a/tempest-pull/run
+++ b/tempest-pull/run
@@ -1,4 +1,8 @@
+#!/bin/bash
+
 export LAVA_PULL_BUNDLEROOT=~/lava_pull/bundles/
 export LAVA_PULL_LOGROOT=~/lava_pull/logs/
-
+source venv/bin/activate
 python tempest-pull.py
+deactivate
+
diff --git a/tempest-pull/setup b/tempest-pull/setup
new file mode 100755
index 0000000..cca7e75
--- /dev/null
+++ b/tempest-pull/setup
@@ -0,0 +1,5 @@
+#!/bin/bash
+virtualenv venv
+source venv/bin/activate
+pip install py2neo
+
diff --git a/tempest-pull/tempest-pull.py b/tempest-pull/tempest-pull.py
index 2b85ed9..c3fa1c5 100644
--- a/tempest-pull/tempest-pull.py
+++ b/tempest-pull/tempest-pull.py
@@ -2,8 +2,7 @@ import gc
 import os
 import string
 
-from neo4jrestclient import client
-from neo4jrestclient.client import GraphDatabase
+from py2neo import Graph, Node, Relationship
 from lava import LAVADashboard
 from bundle import BundleStore
 from bundle import Bundle
@@ -12,7 +11,7 @@ from bundle import Bundle
 LAVA_XMLRPC_ENDPOINT = "https://openstack.validation.linaro.org/RPC2"
 
 # this is the URL of the Neo4J instance
-NEO4J_ENDPOINT = "http://localhost:7474"
+NEO4J_ENDPOINT = "http://neo4j:linaro@localhost:7474/db/data/"
 
 # this is the name of the bundle stream to retrieve from
 BUNDLE_STREAM_NAME = "/public/team/openstack/tempest-ci/"
@@ -56,48 +55,54 @@ class Neo4JDatabase(object):
    def store_bundle(self, bundle):
       if not isinstance(bundle, Bundle):
          raise Exception("argument is not a Bundle")
-      gdb = GraphDatabase(self.endpoint, username="neo4j", password="linaro")
+
+      graph = Graph(self.endpoint)
 
       # find (or create) the OS version + distro node
-      OS = None
-      results = gdb.query("match(n:OS {name:\"%s\", distro:\"%s\"}) return n" % \
-         (bundle.metadata["lava_job_attributes"]["os-version"], bundle.metadata["lava_job_attributes"]["os-distro"]), \
-         returns=client.Node)
-      if len(results) == 1:
-         OS = results[0]
-      else:
-         OS = gdb.nodes.create(
-            name=bundle.metadata["lava_job_attributes"]["os-version"], \
-            distro=bundle.metadata["lava_job_attributes"]["os-distro"])
-         OS.labels.add("OS")
+      os_version = bundle.metadata["lava_job_attributes"]["os-version"]
+      os_distro = bundle.metadata["lava_job_attributes"]["os-distro"]
+      os_name = "%s/%s" % (os_distro, os_version)
+      OS_node = graph.find_one("OS", "name", os_name)
+      if not OS_node:
+         print "creating OS node"
+         OS_node = Node("OS", name=os_name, distro=os_distro, version=os_version)
+         graph.create(OS_node)
 
       # find (or create) the devstack branch  node
-      Branch = None
-      results = gdb.query("match(n:Branch {name:\"%s\"}) return n" % \
-         bundle.metadata["lava_job_attributes"]["devstack-branch"], \
-         returns=client.Node)
-      if len(results) == 1:
-         Branch = results[0]
-      else:
-         Branch = gdb.nodes.create(
-            name=bundle.metadata["lava_job_attributes"]["devstack-branch"])     
-         Branch.labels.add("Branch")
-
-      # create the main tempest run node
-      TempestRun = gdb.nodes.create( \
-         date=bundle.metadata["date_uploaded"],     \
-         lava_job=bundle.metadata["lava_job_id"],   \
-         sha1=bundle.metadata["bundle_sha1"])
-      TempestRun.labels.add("TempestRun")
-      TempestRun.relationships.create("ON", OS)
-      TempestRun.relationships.create("USING", Branch)
+      devstack_branch = bundle.metadata["lava_job_attributes"]["devstack-branch"]
+      Branch_node = graph.find_one("Devstack", "name", devstack_branch)
+      if not Branch_node:
+         print "creating Branch node"
+         Branch_node = Node("Devstack", name=devstack_branch)
+         graph.create(Branch_node)
+
+      # create the main tempest run node and associate with the OS and Branch
+      TempestRun_node = Node("TempestRun",          \
+         date = bundle.metadata["date_uploaded"],     \
+         lava_job = bundle.metadata["lava_job_id"],   \
+         sha1 = bundle.metadata["bundle_sha1"])
+      if bundle.all_tests:
+         TempestRun_node.properties["all_tests"] = len(bundle.all_tests)
+      if bundle.tests_run:
+         TempestRun_node.properties["tests_run"] = len(bundle.tests_run)
+      if bundle.failing_tests:
+         TempestRun_node.properties["failing_tests"] = len(bundle.failing_tests)
+      if bundle.skipped_tests:
+         TempestRun_node.properties["skipped_tests"] = len(bundle.skipped_tests)
+      if bundle.passing_tests:
+         TempestRun_node.properties["passing_tests"] = len(bundle.passing_tests)
+      OS_relationship = Relationship(TempestRun_node, "ON", OS_node)
+      Branch_relationship = Relationship(TempestRun_node, "USING", Branch_node)
+      graph.create(TempestRun_node, OS_relationship, Branch_relationship)
+
+      return
 
       # create all of the tests and relate them back to the tempest node
       for test_set in [bundle.failing_tests, bundle.passing_tests, bundle.skipped_tests]:
          if test_set:
             for test in test_set:
                Test = gdb.nodes.create( \
-                  test=test["test"], \
+                  name=test["test"], \
                   status=test["status"], \
                   start_time=test["start_time"], \
                   stop_time=test["stop_time"], \