Patch from Milosz.HEADmaster

1 files changed, 65 insertions, 23 deletions

diff --git a/art_post_script.py b/art_post_script.py
index 210a665..d2042fb 100755
--- a/art_post_script.py
+++ b/art_post_script.py
@@ -13,6 +13,8 @@ import argparse
 import logging
 import requests
 import pickle
+import subprocess
+import ConfigParser
 from argparse import RawTextHelpFormatter
 from optparse import OptionParser
 from StringIO import StringIO
@@ -62,6 +64,7 @@ class ArtDb(object):
 
     def push_object(self, endpoint, params):
         url = urljoin(self.backend_url, endpoint)
+        logger.info("Submitting to URL: %s" % url)
         headers = {"Authorization": "Token %s" % self.auth_pw}
         s = requests.Session()
         r = requests.Request('POST',
@@ -69,11 +72,9 @@ class ArtDb(object):
                 data=params,
                 headers=headers)
         prep = r.prepare()
-        print r.headers
         #headers.update({'Content-Type':'application/json'})
         #response = requests.post(url, data=params, headers=headers)
         response = s.send(prep)
-        print response.request.headers
 
         #url = urljoin(self.backend_url, endpoint)
         #headers.update({'Content-Type':'application/json'})
@@ -99,7 +100,7 @@ class ArtDb(object):
         manifest = self.push_object(MANIFEST_ENDPOINT, params)
 
         params = {
-            "name" : data['gerrit_params']['GERRIT_BRANCH']
+            "name" : data['branch']
         }
         branch = self.push_object(BRANCH_ENDPOINT, params)
         for bench in data['test_result']:
@@ -118,11 +119,16 @@ class ArtDb(object):
                 "branch":                   branch['id'],
                 "manifest":                 manifest['id'],
                 "build_url":                data['build_url'],
-                "gerrit_change_id":         data['gerrit_params']['GERRIT_CHANGE_ID'],
-                "gerrit_patchset_number":   data['gerrit_params']['GERRIT_PATCHSET_NUMBER'],
-                "gerrit_change_number":     data['gerrit_params']['GERRIT_CHANGE_NUMBER'],
-                "gerrit_change_url":        data['gerrit_params']['GERRIT_CHANGE_URL']
+                #"gerrit_change_id":         data['gerrit_params']['GERRIT_CHANGE_ID'],
+                #"gerrit_patchset_number":   data['gerrit_params']['GERRIT_PATCHSET_NUMBER'],
+                #"gerrit_change_number":     data['gerrit_params']['GERRIT_CHANGE_NUMBER'],
+                #"gerrit_change_url":        data['gerrit_params']['GERRIT_CHANGE_URL']
             }
+            for key, value in data['gerrit_params'].items():
+                if value:
+                    params.update({key.lower: value})
+                if key == "LOCAL_MANIFEST_BRANCH":
+                    params.update({"branch": value})
             result = self.push_object(RESULT_ENDPOINT, params)
             params = {
                 "name": bench['benchmark_name']
@@ -146,6 +152,7 @@ class LinaroAndroidBuildSystem(object):
         self.build_job = build_job
         self.username  = username
         self.password  = password
+        self.build_params = {}
         try:
             self.jenkins    = Jenkins(self.base_url, self.username, self.password)
             self.project    = self.jenkins[self.build_job]
@@ -161,8 +168,31 @@ class LinaroAndroidBuildSystem(object):
             logger.error("Can not get build: #%d" % build_no)
             raise
 
+    def get_build_branch(self, build_no):
+        if not self.build_params:
+            self.get_build_parameters(build_no)
+        build_params = {"BUILD_CONFIG_BRANCH": "",
+                "BUILD_CONFIG_FILENAME": "",
+                "BUILD_CONFIG_REPO": ""}
+        for p in self.build_params:
+            if p['name'] in build_params:
+                build_params[p['name']] = p['value']
+        # git clone build_config_repo -b build_branch repository
+        command = ['git', 'clone', build_params["BUILD_CONFIG_REPO"], '-b', build_params["BUILD_CONFIG_BRANCH"], 'repository']
+        subprocess.call(command)
+        # workaround to ConfigParser limitation
+        ini_str = '[root]\n' + open("repository/" + build_params["BUILD_CONFIG_FILENAME"], 'r').read()
+        ini_fp = StringIO(ini_str)
+        config = ConfigParser.RawConfigParser()
+        config.readfp(ini_fp)
+        branch = config.get("root", "LOCAL_MANIFEST_BRANCH")
+        # check if not empty?
+        command = ['rm', '-rf', 'repository']
+        subprocess.call(command)
+        return branch
+
     def get_build_test_job_ids(self, build_no):
-        logger.info("Querying build #%d test job ids from Jenkins build job: %s" % 
+        logger.info("Querying build #%d test job ids from Jenkins build job: %s" %
               (build_no, self.build_job))
         job_id_list = []
         build = self.get_build(build_no)
@@ -179,12 +209,7 @@ class LinaroAndroidBuildSystem(object):
         else:
             return manifest.content
 
-    def get_build_gerrit_parameters(self, build_no):
-        gerrit_param = { "GERRIT_BRANCH": "",
-                         "GERRIT_CHANGE_ID": "",
-                         "GERRIT_PATCHSET_NUMBER": "",
-                         "GERRIT_CHANGE_NUMBER": "",
-                         "GERRIT_CHANGE_URL": "" }
+    def get_build_parameters(self, build_no):
         rest_url = "%s/job/%s/%d/api/json" % (self.base_url, self.build_job, build_no)
         req = requests.get(rest_url, auth=(self.username, self.password))
         if req.status_code != 200:
@@ -192,18 +217,23 @@ class LinaroAndroidBuildSystem(object):
         else:
             build_info = req.json()
         try:
-            params   = (a['parameters'] for a in build_info['actions'] if 'parameters' in a).next()
+            self.build_params = (a['parameters'] for a in build_info['actions'] if 'parameters' in a).next()
         except:
             logger.error("Can not get build: #%d parameters." % build_no)
             raise
-        for p in params:
+
+    def get_build_gerrit_parameters(self, build_no):
+        if not self.build_params:
+            self.get_build_parameters(build_no)
+        gerrit_param = { "GERRIT_CHANGE_ID": "",
+                         "GERRIT_PATCHSET_NUMBER": "",
+                         "GERRIT_CHANGE_NUMBER": "",
+                         "GERRIT_CHANGE_URL": ""}
+
+        for p in self.build_params:
             if p['name'] in gerrit_param:
                 gerrit_param[p['name']] = p['value']
 
-        if not ''.join(gerrit_param.values()):
-            logger.error("Can not get gerrit parameters in this build!")
-            raise InfoRequestError("Can not find any gerrir related parameters in this build")
-
         return gerrit_param
 
 
@@ -215,10 +245,14 @@ class LAVA(object):
             self.token       = token
             server_url       = "https://%s:%s@%sRPC2/" % (self.username, self.token, self.lava_server.split("//")[1])
             self.server      = xmlrpclib.ServerProxy(server_url)
+            self.result_file_name = ""
         except:
             logger.warning("Can not connect to LAVA server: %s" % lava_server)
             raise
 
+    def set_result_filename(self, result_file_name):
+        self.result_file_name = result_file_name
+
     def get_test_results(self, job_no):
         test_result_list = []
         test_result = { "board": "",
@@ -265,7 +299,11 @@ class LAVA(object):
                 # get test results for the attachment
                 test_mode        = ast.literal_eval(src['test_params'])['MODE']
                 pkl_content      = (a['content'] for a in host['attachments'] if a['pathname'].endswith('pkl')).next()
-                test_result_dict = pickle.loads(base64.b64decode(pkl_content))
+                pkl_text         = base64.b64decode(pkl_content)
+                # save pickle file locally
+                with open(self.result_file_name + "_" + str(test_mode) + ".pkl", "w") as pkl_file:
+                    pkl_file.write(pkl_text)
+                test_result_dict = pickle.loads(pkl_text)
                 # Key Format: benchmarks/micro/<BENCHMARK_NAME>.<SUBSCORE>
                 # Extract and unique them to form a benchmark name list
                 test_result_keys = list(bn.split('/')[-1].split('.')[0] for bn in test_result_dict.keys())
@@ -295,8 +333,6 @@ class LAVA(object):
 #        if not test_result['test_result']:
 #            raise InfoRequestError("No test cases in job #%s" % job_no)
 
-        import pprint
-        pprint.pprint(test_result_list)
         return test_result_list
 
 
@@ -320,6 +356,9 @@ if __name__ == '__main__':
                         help="Specify the Jenkins user. This is MANDATORY.")
     parser.add_option("--jenkins-token", dest="jenkins_token",
                         help="Specify the Jenkins token. This is MANDATORY.")
+    parser.add_option("--result-file-name", dest="result_file_name",
+                        default="ubenchmarks",
+                        help="Specify the Jenkins token. This is MANDATORY.")
 
     options, args = parser.parse_args()
 
@@ -355,6 +394,7 @@ if __name__ == '__main__':
     jenkins    = LinaroAndroidBuildSystem(jenkins_base_url, jenkins_project, options.jenkins_user, options.jenkins_token)
     #lava       = LAVA(LAVA_USERNAME, LAVA_TOKEN)
     lava       = LAVA(options.lava_user, options.lava_token)
+    lava.set_result_filename(options.result_file_name)
     backend_db = ArtDb(options.backend_url, options.backend_token)
     #except Exception as e:
     #    logger.error("Initial objects error.\n%s" % e)
@@ -362,6 +402,7 @@ if __name__ == '__main__':
     try:
         gerrit_param = jenkins.get_build_gerrit_parameters(int(jenkins_build_number))
         manifest     = jenkins.get_build_manifest(int(jenkins_build_number))
+        branch       = jenkins.get_build_branch(int(jenkins_build_number))
     except Exception as e:
         logger.error("Can not get required build information!\n%s" % e)
         exit(-1)
@@ -369,6 +410,7 @@ if __name__ == '__main__':
     data = { "build_url" : options.build_url,
              "manifest": manifest,
              "gerrit_params": gerrit_param,
+             "branch": branch,
              "test_result": {} }
 
     test_job_ids = jenkins.get_build_test_job_ids(int(jenkins_build_number))