update to cache finished jobs result with sqlite

and use the result from the latest job

Signed-off-by: Yongqin Liu <yongqin.liu@linaro.org>

9 files changed, 193 insertions, 31 deletions

diff --git a/db.sqlite3 b/db.sqlite3
index fb67489..6d3daf6 100644
--- a/db.sqlite3
+++ b/db.sqlite3
diff --git a/report/admin.py b/report/admin.py
index 13be29d..1633b41 100644
--- a/report/admin.py
+++ b/report/admin.py
@@ -4,3 +4,13 @@ from __future__ import unicode_literals
 from django.contrib import admin
 
 # Register your models here.
+from .models import TestCase
+from .models import JobCache
+
+class TestCaseAdmin(admin.ModelAdmin):
+    list_display = ('name', 'result', 'measurement', 'unit', 'suite', 'job_id')
+    search_fields = ('name', 'suite')
+    list_filter = ('name', 'suite')
+
+admin.site.register(TestCase, TestCaseAdmin)
+admin.site.register(JobCache)
diff --git a/report/migrations/0001_initial.py b/report/migrations/0001_initial.py
new file mode 100644
index 0000000..5d819cf
--- /dev/null
+++ b/report/migrations/0001_initial.py
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.5 on 2017-10-09 11:10
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TestCase',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=128)),
+                ('result', models.CharField(max_length=16)),
+                ('measurement', models.DecimalField(decimal_places=2, max_digits=11, null=True)),
+                ('unit', models.CharField(max_length=128, null=True)),
+                ('suite', models.CharField(max_length=16)),
+                ('job_id', models.CharField(max_length=16)),
+            ],
+        ),
+    ]
diff --git a/report/migrations/0002_jobcache.py b/report/migrations/0002_jobcache.py
new file mode 100644
index 0000000..4940b07
--- /dev/null
+++ b/report/migrations/0002_jobcache.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.5 on 2017-10-09 12:07
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('report', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='JobCache',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('job_id', models.CharField(max_length=16)),
+                ('cached', models.BooleanField(default=False)),
+            ],
+        ),
+    ]
diff --git a/report/models.py b/report/models.py
index 1dfab76..26587e9 100644
--- a/report/models.py
+++ b/report/models.py
@@ -4,3 +4,23 @@ from __future__ import unicode_literals
 from django.db import models
 
 # Create your models here.
+class TestCase(models.Model):
+    name = models.CharField(max_length=128)
+    result = models.CharField(max_length=16)
+    measurement = models.DecimalField( max_digits=11, decimal_places=2, null=True)
+    unit = models.CharField(max_length=128, null=True)
+    suite = models.CharField(max_length=16)
+    job_id = models.CharField(max_length=16)
+
+    def __unicode__(self):
+        if self.measurement:
+            return "%s %s %s %s" % (self.name, self.result, self.measurement, self.unit)
+        else:
+            return "%s %s" % (self.name, self.result)
+
+
+class JobCache(models.Model):
+    job_id = models.CharField(max_length=16)
+    cached = models.BooleanField(default=False)
+    def __str__(self):
+        return "%s %s" % (self.job_id, self.cached)
diff --git a/report/templates/index.html b/report/templates/index.html
index 9e59d8a..f977d97 100644
--- a/report/templates/index.html
+++ b/report/templates/index.html
@@ -6,6 +6,8 @@ Builds Info
 <table>
 <tr>
 <td align="left">
+<li>Compare 2 jobs with specified job ids</li>
+<li>Deploy it on AWS or developercloud</li>
 <li>Resubmit faild jobs(not submitted/running jobs) with one link</li>
 <li>Trigger ci builds manually or according schedule</li>
 <li>Test any change on local manifest or via patchset script</li>
diff --git a/report/templates/job-resubmit.html b/report/templates/job-resubmit.html
index ceb0f5a..f3cdc9b 100644
--- a/report/templates/job-resubmit.html
+++ b/report/templates/job-resubmit.html
@@ -1,12 +1,19 @@
 <div align="center">
 {% if errors %}
 <title>Job resubmitted failed </title>
-Please specify the job id to be resubmitted
-Back to check the <a href="/report">lava result report</a>
+Please select one job to resubmit
+Back to check the <a href="/report/jobs/">lava result report</a>
 {% else %}
 <title>Job resubmitted successfully </title>
+<table border=2>
+{% for job in new_job_ids %}
+<tr>
+<td>{{ job.0 }}</td>
+<td>{{ job.1 }}</td>
+</tr>
+{% endfor %}
+</table>
 
-The new job is: <a href="{{ lava_server_job_prefix }}/{{ job_id }}">{{ job_id }}</a> <br/>
-Back to check the <a href="/report">lava result report</a>
+Back to check the <a href="/report/jobs/">lava result report</a>
 {% endif %}
 </div>
diff --git a/report/templates/jobs.html b/report/templates/jobs.html
index 966ffdb..7a48d44 100644
--- a/report/templates/jobs.html
+++ b/report/templates/jobs.html
@@ -45,6 +45,7 @@
 {% with jobs_failed|length as number_of_failed %}
 <h2>Failed Jobs list/Total Jobs: {{ number_of_failed }}/{{ jobs_result|length|add:number_of_failed }}</h2>
 {% endwith %}
+<form action="/report/resubmit-job/" method="post">
 <table border="2">
 <tr>
     <th>Job Name</th>
@@ -66,10 +67,12 @@
             <button name="resubmit_latest" disabled="true" onclick="this.disabled=true; window.location='/report/resubmit-job/?build_name={{ build_info.build_name }}&&job_id={% with job.id_status_list|first as first_id_status %}{{ first_id_status.0 }}{% endwith %}'">
                 Resubmit {% with job.id_status_list|first as first_id_status %}{{ first_id_status.0 }}{% endwith %}
             </button>
+            <input type="checkbox" name="job_ids" value="{% with job.id_status_list|first as first_id_status %}{{ first_id_status.0 }}{% endwith %}" selected="false" disabled="false"/>
             {% else %}
             <button name="resubmit_latest" onclick="this.disabled=true; window.location='/report/resubmit-job/?build_name={{ build_info.build_name }}&&job_id={% with job.id_status_list|first as first_id_status %}{{ first_id_status.0 }}{% endwith %}'">
                 Resubmit {% with job.id_status_list|first as first_id_status %}{{ first_id_status.0 }}{% endwith %}
             </button>
+            <input type="checkbox" name="job_ids" value="{% with job.id_status_list|first as first_id_status %}{{ first_id_status.0 }}{% endwith %}" checked="true"/>
 
             {% endif %}
         {% endwith %}
@@ -81,6 +84,10 @@
 </tr>
 {% endfor %}
 </table>
+{% csrf_token %} 
+<input type="hidden" name="build_name" value="{{ build_info.build_name }}"/>
+<input type="submit" value="Resubmit All Possible Failed Jobs"/>
+</form>
 <h2>Results List from lava jobs</h2>
 <table border="2">
 <tr>
diff --git a/report/views.py b/report/views.py
index a92c64d..53ed94c 100644
--- a/report/views.py
+++ b/report/views.py
@@ -14,6 +14,7 @@ import yaml
 from lava_tool.authtoken import AuthenticatingServerProxy, KeyringAuthBackend
 
 # Create your views here.
+from models import TestCase, JobCache
 
 android_snapshot_url_base = "https://snapshots.linaro.org/android"
 ci_job_url_base = 'https://ci.linaro.org/job'
@@ -182,20 +183,35 @@ def jobs_dict_to_sorted_tuple(dict_jobs={}):
     jobs_tuple = []
     for job_name, job_details in dict_jobs.items():
         id_list = job_details.get("id_list")
+        status_list = job_details.get("status_list")
+        zip_id_status_list = zip(id_list, status_list)
+        zip_id_status_list.sort(reverse=True)
+        id_list, status_list = zip(*zip_id_status_list)
         status_string_list = []
-        for status in job_details.get("status_list"):
+        for status in status_list:
             status_string_list.append(job_status_dict[status])
         jobs_tuple.append({"name": job_name,
                            "id_list": id_list,
-                           "status_list": job_details.get("status_list"),
+                           "status_list": status_list,
                            "status_string_list": status_string_list,
                            "id_status_list": zip(id_list, status_string_list)})
     jobs_tuple.sort(key=get_job_name)
     return jobs_tuple
 
 
-def get_yaml_result(job_id, lava_server):
+def get_yaml_result(job_id):
     tests_res = {}
+    for test_case in TestCase.objects.filter(job_id=job_id):
+        tests_res[test_case.name] = {"name": test_case.name,
+                                     "result": test_case.result,
+                                     "measurement": test_case.measurement,
+                                     "unit": test_case.unit,
+                                     "suite": test_case.suite,
+                                     "job_id": job_id,
+                                    }
+    return tests_res
+
+def cache_job_result_to_db(job_id, lava_server):
     try:
         res = lava_server.results.get_testjob_results_yaml(job_id)
         for test in yaml.load(res):
@@ -208,14 +224,14 @@ def get_yaml_result(job_id, lava_server):
                 continue
             if test.get("measurement") and test.get("measurement") == "None":
                 test["measurement"] = None
-            tests_res[test.get("name")] = { "name": test.get("name"),
-                                            "result": test.get("result"),
-                                            "measurement": test.get("measurement"),
-                                            "unit": test.get("unit"),
-                                            "job_id": job_id,
-                                          }
+            TestCase.objects.create(name=test.get("name"),
+                                    result=test.get("result"),
+                                    measurement=test.get("measurement"),
+                                    unit=test.get("unit"),
+                                    suite=test.get("suite"),
+                                    job_id=job_id)
 
-        return tests_res
+        JobCache.objects.create(job_id=job_id, cached=True)
 
     except xmlrpclib.Fault as e:
         raise e
@@ -223,20 +239,37 @@ def get_yaml_result(job_id, lava_server):
         raise
 
 def resubmit_job(request):
-    job_id = request.GET.get("job_id", "")
-    build_name = request.GET.get("build_name", "")
-    if not job_id:
+    job_ids = request.POST.getlist("job_ids")
+    if len(job_ids) == 0:
+        job_id = request.GET.get("job_id", "")
+        build_name = request.GET.get("build_name", "")
+        if not job_id:
+            return render(request, 'job-resubmit.html',
+                          {
+                            'errors': True,
+                          })
+        job_ids = [job_id]
+    else:
+        build_name = request.POST.get("build_name", None)
+
+    if len(job_ids) == 0:
         return render(request, 'job-resubmit.html',
-                  {
-                   'errors': True,
-                  }
-        )
+                      {
+                        'errors': True,
+                      })
 
     lava_server = build_configs[build_name]['lava_server']
-    new_job_id = lava_server.scheduler.jobs.resubmit(job_id)
+
+    new_job_ids = []
+    for job_id in job_ids:
+        new_job_id = lava_server.scheduler.jobs.resubmit(job_id)
+        if new_job_id:
+            new_job_ids.append((job_id, new_job_id))
+        else:
+            new_job_ids.append((job_id, "--"))
     return render(request, 'job-resubmit.html',
                   {
-                   'job_id': new_job_id,
+                   'new_job_ids': new_job_ids,
                    'lava_server_job_prefix': lava_server_job_prefix["staging"],
                   }
         )
@@ -250,6 +283,14 @@ def get_default_build_no(all_build_numbers=[], defaut_build_no=None):
         return 0
 
 
+def is_job_cached(job_id):
+    try:
+        JobCache.objects.get(job_id=job_id)
+    except JobCache.DoesNotExist:
+        return False
+
+    return True
+
 def get_test_results_for_build(build_name, build_no, job_name_list=[]):
     jobs_failed = []
     total_tests_res = {}
@@ -263,13 +304,17 @@ def get_test_results_for_build(build_name, build_no, job_name_list=[]):
         for job_id, job_status in id_status_list:
             if job_status != job_status_dict[2]:
                 continue
-            tests_res = get_yaml_result(job_id, lava_server)
+            if not is_job_cached(job_id):
+                cache_job_result_to_db(job_id, lava_server)
+
+            tests_res = get_yaml_result(job_id=job_id)
             if len(tests_res) != 0:
                 # use the last to replace the first
                 # might be better to change to use the better one
                 # compare the 2 results
                 job_total_res.update(tests_res)
                 result_job_id_status = (job_id, job_status)
+                break
 
         if len(job_total_res) == 0:
             jobs_failed.append(job)
@@ -283,6 +328,17 @@ def get_test_results_for_build(build_name, build_no, job_name_list=[]):
 
     return (jobs_failed, total_tests_res)
 
+def get_build_config_value(build_config_url, key="MANIFEST_BRANCH"):
+    response = urllib2.urlopen(build_config_url)
+    html = response.read()
+
+    pat = re.compile('%s=(?P<value>android-.+)' % key)
+    all_builds = pat.findall(html)
+    if len(all_builds) > 0:
+        return all_builds[0]
+    else:
+        return None
+
 def jobs(request):
     build_name = request.GET.get("build_name", DEFAULT_BUILD_NAME)
 
@@ -292,16 +348,17 @@ def jobs(request):
     (jobs_failed, total_tests_res) = get_test_results_for_build(build_name, build_no)
 
     build_config_url = "%s/%s" % (android_build_config_url_base, build_name.replace("android-", ""))
-
+    build_android_tag = get_build_config_value(build_config_url, key="MANIFEST_BRANCH")
     build_info = {
                     "build_name": build_name,
                     "build_no": build_no,
                     "ci_url_base": ci_job_url_base,
                     "snapshot_url_base": android_snapshot_url_base,
-                    "android_tag": "android-8.0.0_r4",
+                    "android_tag": build_android_tag,
                     "build_config_url": build_config_url,
                     "build_numbers": get_possible_builds(build_name),
                  }
+
     return render(request, 'jobs.html',
                   {
                    'jobs_failed': jobs_failed,
@@ -631,12 +688,14 @@ def submit_lava_jobs(request):
 def index(request):
     builds = {}
     for build_name in build_names:
+        build_config_url = "%s/%s" % (android_build_config_url_base, build_name.replace("android-", ""))
+        build_android_tag = get_build_config_value(build_config_url, key="MANIFEST_BRANCH")
         builds[build_name] = {
                                 "build_name": build_name,
-                                "android_version": "android-8.0.0_r4",
+                                "android_version": build_android_tag,
                                 "kernel_version": "4.9",
                                 "ci_link": "%s/%s" % (ci_job_url_base, build_name),
-                                "android_build_config_link": "%s/%s" % (android_build_config_url_base, build_name.replace("android-", "")),
+                                "android_build_config_link": build_config_url,
                                 "snapshot_url": '%s/%s/' % (android_snapshot_url_base, build_name),
                                 "job_status": "--",
                              }
@@ -648,10 +707,16 @@ def index(request):
                   })
 
 if __name__ == "__main__":
-    build_name = "android-lcr-reference-x15-o"
-    job_template = get_possible_job_names(build_name=build_name)
-    print str(job_template)
+#    get_yaml_result("191778")
+    build_name = "android-lcr-reference-hikey-o"
+    build_no = '20'
+#    job_template = get_possible_job_names(build_name=build_name)
+#    print str(job_template)
 #    (all_build_numbers, checklist_results) = get_test_results_for_job(build_name, server, jobs=[])
 #    for job_name, job_result in checklist_results.items():    for test_name, test_result in job_result.items():
 #    print str(checklist_results)
 
+    lava_server = build_configs[build_name]['lava_server']
+
+    jobs = jobs_dict_to_sorted_tuple(get_jobs(build_name, build_no, lava_server, job_name_list=[]))
+    print str(jobs)