lkft: add lkftreport tool

for lkft reporting

Signed-off-by: Yongqin Liu <yongqin.liu@linaro.org>

10 files changed, 216 insertions, 2 deletions

diff --git a/lcr/qa_report.py b/lcr/qa_report.py
index e6b70b3..90f9c48 100644
--- a/lcr/qa_report.py
+++ b/lcr/qa_report.py
@@ -91,6 +91,14 @@ class JenkinsApi(RESTFullApi):
         full_api_url = '%s/api/json/' % build_url
         return self.call_with_full_url(request_url=full_api_url)
 
+    def get_job_url(self, name=None, number=None):
+        if name is None:
+            return "https://%s" % (self.domain)
+        elif number is None:
+            return "https://%s/job/%s/" % (self.domain, name)
+        else:
+            return "https://%s/job/%s/%s/" % (self.domain, name, number)
+
 
 class LAVAApi(RESTFullApi):
     def get_api_url_prefix(self):
@@ -189,4 +197,4 @@ class QAReportApi(RESTFullApi):
         return self.call_with_full_url(request_url=build_meta_url)
 
     def get_qa_job_id_with_url(self, job_url):
-        return job_url.strip('/').split('/')[-1]
\ No newline at end of file
+        return job_url.strip('/').split('/')[-1]
diff --git a/lkft/admin.py b/lkft/admin.py
index 13be29d..4e267da 100644
--- a/lkft/admin.py
+++ b/lkft/admin.py
@@ -2,5 +2,16 @@
 from __future__ import unicode_literals
 
 from django.contrib import admin
+from .models import KernelChange, CiBuild
 
 # Register your models here.
+class KernelChangeAdmin(admin.ModelAdmin):
+    list_display = ['branch', 'describe', 'trigger_name', 'trigger_number', 'reported']
+
+
+class CiBuildAdmin(admin.ModelAdmin):
+    list_display = ['name', 'number']
+
+
+admin.site.register(KernelChange, KernelChangeAdmin)
+admin.site.register(CiBuild, CiBuildAdmin)
diff --git a/lkft/management/__init__.py b/lkft/management/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lkft/management/__init__.py
diff --git a/lkft/management/commands/__init__.py b/lkft/management/commands/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lkft/management/commands/__init__.py
diff --git a/lkft/management/commands/lkftreport.py b/lkft/management/commands/lkftreport.py
new file mode 100644
index 0000000..0a5042d
--- /dev/null
+++ b/lkft/management/commands/lkftreport.py
@@ -0,0 +1,45 @@
+## https://docs.djangoproject.com/en/1.11/topics/db/managers/
+## https://docs.djangoproject.com/en/dev/howto/custom-management-commands/#howto-custom-management-commands
+## https://medium.com/@bencleary/django-scheduled-tasks-queues-part-1-62d6b6dc24f8
+## https://medium.com/@bencleary/django-scheduled-tasks-queues-part-2-fc1fb810b81d
+## https://medium.com/@kevin.michael.horan/scheduling-tasks-in-django-with-the-advanced-python-scheduler-663f17e868e6
+## https://django-background-tasks.readthedocs.io/en/latest/
+
+
+import datetime
+
+from django.core.management.base import BaseCommand, CommandError
+from lkft.models import KernelChange, CiBuild
+
+from lcr import qa_report
+
+jenkins_api = qa_report.JenkinsApi('ci.linaro.org', None)
+
+class Command(BaseCommand):
+    help = 'Check the build and test results for kernel changes, and send report if the jobs finished'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('git_describes', nargs='+', type=str)
+
+    def handle(self, *args, **options):
+        kernel_changes = KernelChange.objects_needs_report.all()
+        for kernel_change in kernel_changes:
+            trigger_url = jenkins_api.get_job_url(name=kernel_change.trigger_name, number=kernel_change.trigger_number)
+            trigger_build = jenkins_api.get_build_details_with_full_url(build_url=trigger_url)
+            trigger_build['start_timestamp'] = datetime.datetime.fromtimestamp(int(trigger_build['timestamp'])/1000)
+            trigger_build['duration'] = datetime.timedelta(milliseconds=trigger_build['duration'])
+
+            print "%s started at %s, took %s" % (kernel_change, trigger_build['start_timestamp'], trigger_build['duration'])
+            ci_builds = CiBuild.objects_kernel_change.get_builds_per_kernel_change(kernel_change=kernel_change)
+            for ci_build in ci_builds:
+                build_url = jenkins_api.get_job_url(name=ci_build.name, number=ci_build.number)
+                build = jenkins_api.get_build_details_with_full_url(build_url=build_url)
+                build['start_timestamp'] = datetime.datetime.fromtimestamp(int(build['timestamp'])/1000)
+
+                if build.get('building'):
+                    build_status = 'INPROGRESS'
+                else:
+                    build_status = build.get('result') # null or SUCCESS, FAILURE, ABORTED
+                    build['duration'] = datetime.timedelta(milliseconds=build['duration'])
+
+                print "\t %s %s started at %s, took %s" % (ci_build, build_status, build['start_timestamp'], build['duration'])
diff --git a/lkft/migrations/0001_initial.py b/lkft/migrations/0001_initial.py
new file mode 100644
index 0000000..7d30f50
--- /dev/null
+++ b/lkft/migrations/0001_initial.py
@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.5 on 2019-12-27 14:35
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CiBuild',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255)),
+                ('number', models.IntegerField()),
+            ],
+        ),
+        migrations.CreateModel(
+            name='KernelChange',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('branch', models.CharField(max_length=255)),
+                ('describe', models.CharField(max_length=255)),
+                ('reported', models.BooleanField()),
+                ('trigger_name', models.CharField(max_length=255)),
+                ('trigger_number', models.IntegerField()),
+            ],
+        ),
+        migrations.AddField(
+            model_name='cibuild',
+            name='kernel_change',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lkft.KernelChange'),
+        ),
+    ]
diff --git a/lkft/models.py b/lkft/models.py
index 1dfab76..f69d34b 100644
--- a/lkft/models.py
+++ b/lkft/models.py
@@ -3,4 +3,42 @@ from __future__ import unicode_literals
 
 from django.db import models
 
-# Create your models here.
+class KernelChangeManager(models.Manager):
+    def get_queryset(self):
+        return super(KernelChangeManager, self).get_queryset().filter(reported=False)
+
+class KernelChange(models.Model):
+    branch = models.CharField(max_length=255)
+    describe = models.CharField(max_length=255)
+    reported = models.BooleanField()
+    trigger_name = models.CharField(max_length=255)
+    trigger_number = models.IntegerField()
+
+    def __str__(self):
+        return "%s-%s" % (self.branch, self.describe)
+
+    def __unicode__(self):
+        return "%s-%s" % (self.branch, self.describe)
+
+    objects = models.Manager() # The default manager
+    objects_needs_report = KernelChangeManager() # custom managerKernelChangeManager()
+
+
+class CiBuildKernelChangeManager(models.Manager):
+    def get_builds_per_kernel_change(self, kernel_change=None):
+        return super(CiBuildKernelChangeManager, self).get_queryset().filter(kernel_change=kernel_change)
+
+
+class CiBuild(models.Model):
+    name = models.CharField(max_length=255)
+    number = models.IntegerField()
+    kernel_change = models.ForeignKey(KernelChange, on_delete=None)
+
+    def __str__(self):
+        return "%s#%s" % (self.name, self.number)
+
+    def __unicode__(self):
+        return "%s#%s" % (self.name, self.number)
+
+    objects = models.Manager()
+    objects_kernel_change = CiBuildKernelChangeManager()
diff --git a/lkft/urls.py b/lkft/urls.py
index 1ed7fef..4a07fcf 100644
--- a/lkft/urls.py
+++ b/lkft/urls.py
@@ -2,6 +2,7 @@ from django.conf.urls import url
 
 from . import views
 
+basic_pat = '[a-zA-Z0-9][a-zA-Z0-9_.-]+'
 urlpatterns = [
     url(r'^$', views.list_projects, name='home'),
     url(r'^projects/.*$', views.list_projects, name='list_projects'),
@@ -9,4 +10,8 @@ urlpatterns = [
     url(r'^jobs/.*$', views.list_jobs, name='list_jobs'),
     url(r'^file-bug/.*$', views.file_bug, name='file_bug'),
     url(r'^resubmit-job/.*$', views.resubmit_job, name='resubmit_job'),
+    # newchanges/$branch/$describe/$build_name/$build_number
+    url(r'^newchanges/(%s)/(%s)/(%s)/([0-9]+)' % (basic_pat, basic_pat, basic_pat), views.new_kernel_changes),
+    # newchanges/$branch/$describe/$build_name/$build_number
+    url(r'^newbuild/(%s)/(%s)/(%s)/([0-9]+)' % (basic_pat, basic_pat, basic_pat), views.new_build, name='new_build'),
 ]
diff --git a/lkft/views.py b/lkft/views.py
index a71cfa1..0b2c7fd 100644
--- a/lkft/views.py
+++ b/lkft/views.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 
 from django import forms
+from django.http import HttpResponse
 from django.shortcuts import render
 
 import collections
@@ -29,6 +30,8 @@ from lcr.qa_report import DotDict
 from lcr.utils import download_urllib
 from lkft.lkft_config import find_citrigger, find_cibuild, get_hardware_from_pname, get_version_from_pname, get_kver_with_pname_env
 
+from .models import KernelChange, CiBuild
+
 qa_report_def = QA_REPORT[QA_REPORT_DEFAULT]
 qa_report_api = qa_report.QAReportApi(qa_report_def.get('domain'), qa_report_def.get('token'))
 jenkins_api = qa_report.JenkinsApi('ci.linaro.org', None)
@@ -1029,3 +1032,63 @@ def resubmit_job(request):
                    'results': results,
                   }
     )
+
+
+def new_kernel_changes(request, branch, describe, trigger_name, trigger_number):
+
+    remote_addr = request.META.get("REMOTE_ADDR")
+    remote_host = request.META.get("REMOTE_HOST")
+    logger.info('request from remote_host=%s,remote_addr=%s' % (remote_host, remote_addr))
+    logger.info('request for branch=%s, describe=%s, trigger_name=%s, trigger_number=%s' % (branch, describe, trigger_name, trigger_number))
+
+    err_msg = None
+    try:
+        KernelChange.objects.get(branch=branch, describe=describe)
+        err_msg = 'request for branch=%s, describe=%s is already there' % (branch, describe)
+        logger.info(err_msg)
+    except KernelChange.DoesNotExist:
+        KernelChange.objects.create(branch=branch,
+                                    describe=describe,
+                                    reported=False,
+                                    trigger_name=trigger_name,
+                                    trigger_number=trigger_number)
+
+
+    if err_msg is None:
+        return HttpResponse(status=200)
+    else:
+        return HttpResponse("ERROR:%s" % err_msg,
+                            status=200)
+
+
+def new_build(request, branch, describe, name, number):
+
+    remote_addr = request.META.get("REMOTE_ADDR")
+    remote_host = request.META.get("REMOTE_HOST")
+    logger.info('request from %s %s' % (remote_host, remote_addr))
+    logger.info('request for branch=%s, describe=%s, trigger_name=%s, trigger_number=%s' % (branch, describe, name, number))
+
+    err_msg = None
+    try:
+        kernel_change = KernelChange.objects.get(branch=branch, describe=describe)
+
+        try:
+            CiBuild.objects.get(name=name, number=number)
+            err_msg = "The build already recorded: name=%s, number=%s" % (name, number)
+            logger.info(err_msg)
+        except CiBuild.DoesNotExist:
+            CiBuild.objects.create(name=name,
+                                    number=number,
+                                    kernel_change=kernel_change)
+            kernel_change.reported = False
+            kernel.save()
+
+    except KernelChange.DoesNotExist:
+        err_msg = "The change for the specified kernel and describe does not exist: branch=%s, describe=%s" % (branch, describe)
+        logger.info(err_msg)
+
+    if err_msg is None:
+        return HttpResponse(status=200)
+    else:
+        return HttpResponse("ERROR:%s" % err_msg,
+                            status=200)
diff --git a/lkftreport.sh b/lkftreport.sh
new file mode 100755
index 0000000..4b0c771
--- /dev/null
+++ b/lkftreport.sh
@@ -0,0 +1,3 @@
+#!/bin/bash -ex
+
+/SATA3/django_instances/workspace/bin/python /SATA3/django_instances/lcr-report/manage.py lkftreport