From b884affad4a645dca21e00313a62f9f8e000749f Mon Sep 17 00:00:00 2001 From: Yongqin Liu Date: Tue, 3 Mar 2020 11:33:01 +0800 Subject: lkft: refactory the database structure with a separate ReportProject table and add the qa-report build id into the ReportBuild table Signed-off-by: Yongqin Liu Change-Id: I7ca8c43f01dd0b2e265340d1bbe136515f7d3a73 --- lkft/admin.py | 17 ++++++++++-- lkft/management/commands/lkftreport.py | 33 +++++++++++++++++----- lkft/migrations/0004_auto_20200303_0225.py | 37 +++++++++++++++++++++++++ lkft/migrations/0005_reportbuild_qa_build_id.py | 18 ++++++++++++ lkft/models.py | 24 ++++++++++++++-- lkft/templates/lkft-describe.html | 16 +++++------ lkft/views.py | 8 +++--- 7 files changed, 128 insertions(+), 25 deletions(-) create mode 100644 lkft/migrations/0004_auto_20200303_0225.py create mode 100644 lkft/migrations/0005_reportbuild_qa_build_id.py diff --git a/lkft/admin.py b/lkft/admin.py index 7a59751..7883c2b 100644 --- a/lkft/admin.py +++ b/lkft/admin.py @@ -2,19 +2,30 @@ from __future__ import unicode_literals from django.contrib import admin -from .models import KernelChange, CiBuild, ReportBuild +from .models import KernelChange, CiBuild, ReportBuild, ReportProject # Register your models here. class KernelChangeAdmin(admin.ModelAdmin): list_display = ['branch', 'describe', 'trigger_name', 'trigger_number', 'reported'] + search_fields = ('branch', 'describe') class CiBuildAdmin(admin.ModelAdmin): - list_display = ['name', 'number'] + list_display = ['name', 'number', 'kernel_change', 'result'] + search_fields = ('name', 'number', 'kernel_change') + class ReportBuildAdmin(admin.ModelAdmin): - list_display = ['group', 'name', 'version', 'number_passed', 'number_failed', 'number_total', 'modules_done', 'modules_total'] + list_display = ['qa_project', 'version', 'number_passed', 'number_failed', 'number_total', 'modules_done', 'modules_total', 'qa_build_id'] + search_fields = ('qa_project', 'version') + + +class ReportProjectAdmin(admin.ModelAdmin): + list_display = ['group', 'name', 'slug', 'project_id'] + search_fields = ['group', 'name', 'slug', 'project_id'] + admin.site.register(KernelChange, KernelChangeAdmin) admin.site.register(CiBuild, CiBuildAdmin) admin.site.register(ReportBuild, ReportBuildAdmin) +admin.site.register(ReportProject, ReportProjectAdmin) diff --git a/lkft/management/commands/lkftreport.py b/lkft/management/commands/lkftreport.py index 61e10c5..4ac405d 100644 --- a/lkft/management/commands/lkftreport.py +++ b/lkft/management/commands/lkftreport.py @@ -16,7 +16,7 @@ from django.core.management.base import BaseCommand, CommandError from django.utils.timesince import timesince -from lkft.models import KernelChange, CiBuild, ReportBuild +from lkft.models import KernelChange, CiBuild, ReportBuild, ReportProject from lcr import qa_report @@ -88,7 +88,11 @@ class Command(BaseCommand): test_numbers = qa_report.TestNumbers() trigger_url = jenkins_api.get_job_url(name=kernel_change.trigger_name, number=kernel_change.trigger_number) - trigger_build = jenkins_api.get_build_details_with_full_url(build_url=trigger_url) + try: + trigger_build = jenkins_api.get_build_details_with_full_url(build_url=trigger_url) + except qa_report.UrlNotFoundException: + print("the build does not exist any more: %s" % trigger_url) + continue trigger_build['start_timestamp'] = qa_report_api.get_aware_datetime_from_timestamp(int(trigger_build['timestamp'])/1000) trigger_build['duration'] = datetime.timedelta(milliseconds=trigger_build['duration']) trigger_build['name'] = kernel_change.trigger_name @@ -205,6 +209,8 @@ class Command(BaseCommand): target_qareport_build['created_at'] = qa_report_api.get_aware_datetime_from_str(created_str) target_qareport_build['project_name'] = project_name target_qareport_build['project_group'] = project_group + target_qareport_build['project_slug'] = target_qareport_project.get('slug') + target_qareport_build['project_id'] = target_qareport_project.get('id') jobs = qa_report_api.get_jobs_for_build(target_qareport_build.get("id")) build_status = get_lkft_build_status(target_qareport_build, jobs) @@ -373,9 +379,21 @@ class Command(BaseCommand): result_numbers = qareport_build.get('numbers_of_result') trigger_dbci_build = CiBuild.objects.get(name=trigger_build.get('name'), number=trigger_build.get('number')) + + qa_project_group = qareport_build.get('project_group') + qa_project_name = qareport_build.get('project_name') + qa_project_slug = qareport_build.get('project_slug') + qa_project_id = qareport_build.get('project_id') + try: + qa_project = ReportProject.objects.get(group=qa_project_group, name=qa_project_name) + except ReportProject.DoesNotExist: + qa_project = ReportProject.objects.create(group=qa_project_group, + name=qa_project_name, + slug=qa_project_slug, + project_id=qa_project_id) + try: - report_build = ReportBuild.objects.get(group=qareport_build.get('project_group'), - name=qareport_build.get('project_name'), + report_build = ReportBuild.objects.get(qa_project=qa_project, version=kernel_change.describe) report_build.kernel_change = kernel_change report_build.ci_build = dbci_build @@ -387,10 +405,10 @@ class Command(BaseCommand): report_build.modules_total = result_numbers.get('modules_total') report_build.started_at = trigger_build.get('start_timestamp') report_build.fetched_at = qareport_build.get('last_fetched_timestamp') + report_build.qa_build_id = qareport_build.get('id') report_build.save() except ReportBuild.DoesNotExist: - ReportBuild.objects.create(group=qareport_build.get('project_group'), - name=qareport_build.get('project_name'), + ReportBuild.objects.create(qa_project=qa_project, version=kernel_change.describe, kernel_change=kernel_change, ci_build=dbci_build, @@ -401,7 +419,8 @@ class Command(BaseCommand): modules_done=result_numbers.get('modules_done'), modules_total=result_numbers.get('modules_total'), started_at=trigger_build.get('start_timestamp'), - fetched_at=qareport_build.get('last_fetched_timestamp')) + fetched_at=qareport_build.get('last_fetched_timestamp'), + qa_build_id=qareport_build.get('id')) return None # print out the reports diff --git a/lkft/migrations/0004_auto_20200303_0225.py b/lkft/migrations/0004_auto_20200303_0225.py new file mode 100644 index 0000000..78e0012 --- /dev/null +++ b/lkft/migrations/0004_auto_20200303_0225.py @@ -0,0 +1,37 @@ +# Generated by Django 3.0.3 on 2020-03-03 02:25 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('lkft', '0003_auto_20200225_0907'), + ] + + operations = [ + migrations.CreateModel( + name='ReportProject', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('group', models.CharField(max_length=100)), + ('name', models.CharField(max_length=100)), + ('slug', models.CharField(max_length=100)), + ('project_id', models.IntegerField(default=0)), + ], + ), + migrations.RemoveField( + model_name='reportbuild', + name='group', + ), + migrations.RemoveField( + model_name='reportbuild', + name='name', + ), + migrations.AddField( + model_name='reportbuild', + name='qa_project', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='lkft.ReportProject'), + ), + ] diff --git a/lkft/migrations/0005_reportbuild_qa_build_id.py b/lkft/migrations/0005_reportbuild_qa_build_id.py new file mode 100644 index 0000000..85bd255 --- /dev/null +++ b/lkft/migrations/0005_reportbuild_qa_build_id.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-03-03 02:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('lkft', '0004_auto_20200303_0225'), + ] + + operations = [ + migrations.AddField( + model_name='reportbuild', + name='qa_build_id', + field=models.IntegerField(default=0), + ), + ] diff --git a/lkft/models.py b/lkft/models.py index f43200f..daa67cf 100644 --- a/lkft/models.py +++ b/lkft/models.py @@ -58,14 +58,30 @@ class CiBuild(models.Model): objects_kernel_change = CiBuildKernelChangeManager() -class ReportBuild(models.Model): +class ReportProject(models.Model): # the group that this build belongs to group = models.CharField(max_length=100) # the name of the qareport project name = models.CharField(max_length=100) + # the slug of the qareport project + slug = models.CharField(max_length=100) + + project_id = models.IntegerField(default=0) + + def __str__(self): + return "%s#%s" % (self.group, self.name) + + def __unicode__(self): + return "%s#%s" % (self.group, self.name) + + objects = models.Manager() + + +class ReportBuild(models.Model): # the version of the qareport build version = models.CharField(max_length=100) + qa_project = models.ForeignKey(ReportProject, on_delete=models.CASCADE, null=True) kernel_change = models.ForeignKey(KernelChange, on_delete=models.CASCADE) ci_build = models.ForeignKey(CiBuild, on_delete=models.CASCADE, related_name="ci_build") ci_trigger_build = models.ForeignKey(CiBuild, on_delete=models.CASCADE, related_name='trigger_build') @@ -81,11 +97,13 @@ class ReportBuild(models.Model): # the time the last job was fetched fetched_at = models.DateTimeField(null=True) + # The id of the qa-report build id + qa_build_id = models.IntegerField(default=0) def __str__(self): - return "%s#%s#%s" % (self.group, self.name, self.version) + return "%s#%s" % (self.qa_project, self.version) def __unicode__(self): - return "%s#%s#%s" % (self.group, self.name, self.version) + return "%s#%s" % (self.qa_project, self.version) objects = models.Manager() \ No newline at end of file diff --git a/lkft/templates/lkft-describe.html b/lkft/templates/lkft-describe.html index 3d24186..177e59e 100644 --- a/lkft/templates/lkft-describe.html +++ b/lkft/templates/lkft-describe.html @@ -52,7 +52,7 @@
-

Test Results

+

QA Report Builds

@@ -69,15 +69,15 @@ {% for report_build in report_builds %} - - + + - - - - - + + + + + {% endfor %}
Index
{{ forloop.counter }}{{report_build.group}}{{report_build.name}}{{report_build.qa_project.group}}{{report_build.qa_project.name}}

Started at {{ report_build.started_at|date:'M. d, Y, H:i'}},
{{ report_build.started_at|timesince}} ago

 {{report_build.duration}}{{report_build.number_passed}}{{report_build.number_failed}}{{report_build.number_total}}{{report_build.modules_done}}{{report_build.modules_total}}{{report_build.number_passed}}{{report_build.number_failed}}{{report_build.number_total}}{{report_build.modules_done}}{{report_build.modules_total}}
diff --git a/lkft/views.py b/lkft/views.py index 36e81be..babb080 100644 --- a/lkft/views.py +++ b/lkft/views.py @@ -1338,7 +1338,7 @@ def list_branch_kernel_changes(request, branch): @login_required def list_describe_kernel_changes(request, branch, describe): db_kernel_change = KernelChange.objects.get(branch=branch, describe=describe) - db_report_builds = ReportBuild.objects.filter(kernel_change=db_kernel_change).order_by('group', 'name') + db_report_builds = ReportBuild.objects.filter(kernel_change=db_kernel_change).order_by('qa_project__group', 'qa_project__name') db_ci_builds = CiBuild.objects.filter(kernel_change=db_kernel_change).exclude(name=db_kernel_change.trigger_name).order_by('name', 'number') db_trigger_build = CiBuild.objects.get(name=db_kernel_change.trigger_name, kernel_change=db_kernel_change) @@ -1358,20 +1358,20 @@ def list_describe_kernel_changes(request, branch, describe): ci_build['result'] = db_ci_build.result ci_build['duration'] = datetime.timedelta(seconds=db_ci_build.duration) if db_ci_build.timestamp and db_trigger_build.timestamp: - ci_build['queued_duration'] = db_ci_build.timestamp - db_trigger_build.timestamp + ci_build['queued_duration'] = db_ci_build.timestamp - db_trigger_build.timestamp - trigger_build['duration'] ci_builds.append(ci_build) report_builds = [] for db_report_build in db_report_builds: report_build = {} - report_build['group'] = db_report_build.group - report_build['name'] = db_report_build.name + report_build['qa_project'] = db_report_build.qa_project report_build['started_at'] = db_report_build.started_at report_build['number_passed'] = db_report_build.number_passed report_build['number_failed'] = db_report_build.number_failed report_build['number_total'] = db_report_build.number_total report_build['modules_done'] = db_report_build.modules_done report_build['modules_total'] = db_report_build.modules_total + report_build['qa_build_id'] = db_report_build.qa_build_id if db_report_build.fetched_at and db_report_build.started_at: report_build['duration'] = db_report_build.fetched_at - db_report_build.started_at -- cgit v1.2.3