Refactor report emails logic.

  * Move all reports logic into its own module, under utils.
  * Separate boot and build report logic and collect common
    functions.
  * Add some unit tests.
  * Fix celery tasks.

Change-Id: Ib10a3ac436d8a596aae31c3088af32a509098198

9 files changed, 761 insertions, 245 deletions

diff --git a/app/models/__init__.py b/app/models/__init__.py
index 9dc5db0..d96953e 100644
--- a/app/models/__init__.py
+++ b/app/models/__init__.py
@@ -16,6 +16,9 @@ DB_NAME = "kernel-ci"
 
 DEFAULT_SCHEMA_VERSION = "1.0"
 
+# i18n domain name.
+I18N_DOMAIN = "kernelci-backend"
+
 # The default ID key, and other keys, for mongodb documents and queries.
 ACCEPTED_KEYS = "accepted"
 ADDRESS_KEY = "address"
diff --git a/app/taskqueue/tasks.py b/app/taskqueue/tasks.py
index 4be8899..4bf64f0 100644
--- a/app/taskqueue/tasks.py
+++ b/app/taskqueue/tasks.py
@@ -16,7 +16,6 @@
 from __future__ import absolute_import
 
 import celery
-import types
 
 import models
 import taskqueue.celery as taskc
@@ -27,7 +26,9 @@ import utils.bisect.defconfig as defconfigb
 import utils.bootimport
 import utils.docimport
 import utils.emails
-import utils.report
+import utils.report.boot
+import utils.report.build
+import utils.report.common
 
 
 @taskc.app.task(name='send-emails', ignore_result=True)
@@ -160,61 +161,6 @@ def defconfig_bisect_compared_to(doc_id, compare_to, db_options, fields=None):
 
 
 @taskc.app.task(
-    name="schedule-boot-report",
-    acks_late=True,
-    track_started=True,
-    ignore_result=False)
-def schedule_boot_report(json_obj, db_options, mail_options, countdown):
-    """Schedule a second task to send the boot report.
-
-    :param json_obj: The json data as sent in the request.
-    :type json_obj: dict
-    :param db_options: The options necessary to connect to the database.
-    :type db_options: dict
-    :param mail_options: The options necessary to connect to the SMTP server.
-    :type mail_options: dict
-    :param countdown: The delay in seconds to send the email.
-    :type countdown: int
-    """
-    j_get = json_obj.get
-    to_addrs = []
-    status = 400
-
-    if bool(j_get(models.SEND_BOOT_REPORT_KEY, False)):
-        job = j_get(models.JOB_KEY)
-        kernel = j_get(models.KERNEL_KEY)
-        lab_name = j_get(models.LAB_NAME_KEY, None)
-
-        boot_emails = j_get(models.BOOT_REPORT_SEND_TO_KEY, None)
-        generic_emails = j_get(models.REPORT_SEND_TO_KEY, None)
-
-        if boot_emails is not None:
-            if isinstance(boot_emails, types.ListType):
-                to_addrs.extend(boot_emails)
-            elif isinstance(boot_emails, types.StringTypes):
-                to_addrs.append(boot_emails)
-
-        if generic_emails is not None:
-            if isinstance(generic_emails, types.ListType):
-                to_addrs.extend(generic_emails)
-            elif isinstance(generic_emails, types.StringTypes):
-                to_addrs.append(generic_emails)
-
-        if to_addrs:
-            status = 200
-            send_boot_report.apply_async(
-                [job, kernel, lab_name, to_addrs, db_options, mail_options],
-                countdown=countdown)
-        else:
-            status = 500
-            utils.LOG.warn(
-                "No send email addresses specified for '%s-%s': boot report "
-                "cannot be sent", job, kernel)
-
-    return status
-
-
-@taskc.app.task(
     name="send-boot-report",
     acks_late=True,
     track_started=True,
@@ -236,9 +182,9 @@ def send_boot_report(job, kernel, lab_name, to_addrs, db_options, mail_options):
     :type mail_options: dict
     """
     utils.LOG.info("Preparing boot report email for '%s-%s'", job, kernel)
-    status = 400
+    status = "ERROR"
 
-    body, subject = utils.report.create_boot_report(
+    body, subject = utils.report.boot.create_boot_report(
         job,
         kernel,
         lab_name,
@@ -250,8 +196,43 @@ def send_boot_report(job, kernel, lab_name, to_addrs, db_options, mail_options):
         utils.LOG.info("Sending boot report email for '%s-%s'", job, kernel)
         status, errors = utils.emails.send_email(
             to_addrs, subject, body, mail_options)
-        utils.report.save_report(
+        utils.report.common.save_report(
             job, kernel, models.BOOT_REPORT, status, errors, db_options)
+    else:
+        utils.LOG.error(
+            "No email body nor subject found for boot report '%s-%s'",
+            job, kernel)
+
+    return status
+
+
+@taskc.app.task(
+    name="send-build-report",
+    acks_late=True,
+    track_started=True,
+    ignore_result=False)
+def send_build_report(job, kernel, to_addrs, db_options, mail_options):
+    utils.LOG.info("Preparing build report email for '%s-%s'", job, kernel)
+    status = "ERROR"
+
+    body, subject = utils.report.build.create_build_report(
+        job,
+        kernel,
+        db_options=db_options,
+        mail_options=mail_options
+    )
+
+    if all([body is not None, subject is not None]):
+        utils.LOG.info("Sending build report email for '%s-%s'", job, kernel)
+        status, errors = utils.emails.send_email(
+            to_addrs, subject, body, mail_options)
+        utils.report.common.save_report(
+            job, kernel, models.BOOT_REPORT, status, errors, db_options)
+    else:
+        utils.LOG.error(
+            "No email body nor subject found for build report '%s-%s'",
+            job, kernel)
+
     return status
 
 
diff --git a/app/tests/__init__.py b/app/tests/__init__.py
index f6ea79d..5dce3c5 100644
--- a/app/tests/__init__.py
+++ b/app/tests/__init__.py
@@ -44,6 +44,7 @@ def test_modules():
         'models.tests.test_token_model',
         'utils.batch.tests.test_batch_common',
         'utils.bisect.tests.test_bisect',
+        'utils.report.tests.test_report_common',
         'utils.tests.test_base',
         'utils.tests.test_bootimport',
         'utils.tests.test_docimport',
diff --git a/app/utils/report/__init__.py b/app/utils/report/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/app/utils/report/__init__.py
diff --git a/app/utils/report.py b/app/utils/report/boot.py
index a0c3341..30a7e8f 100644
--- a/app/utils/report.py
+++ b/app/utils/report/boot.py
@@ -11,32 +11,33 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Create and send email reports."""
+"""Create the boot email report."""
 
 import gettext
 import io
 import itertools
 import pymongo
-import types
 
 import models
-import models.report as mreport
-import utils
 import utils.db
+import utils.report.common as rcommon
 
 # Register the translation domain and fallback safely, at the moment we do
 # not care if we have translations or not, we just use gettext to exploit its
 # plural forms capabilities. We mark the email string as translatable though
 # so we might give that feature in the future.
-t = gettext.translation("kernelci-backed", fallback=True)
+L10N = gettext.translation(models.I18N_DOMAIN, fallback=True)
 # Register normal Unicode gettext.
-_ = t.ugettext
+_ = L10N.ugettext
 # Register plural forms Unicode gettext.
-_p = t.ungettext
+_p = L10N.ungettext
 
-DEFAULT_BASE_URL = u"http://kernelci.org"
-DEFAULT_BOOT_URL = u"http://kernelci.org/boot/all/job"
-DEFAULT_BUILD_URL = u"http://kernelci.org/build"
+
+BOOT_SEARCH_SORT = [
+    (models.ARCHITECTURE_KEY, pymongo.ASCENDING),
+    (models.DEFCONFIG_FULL_KEY, pymongo.ASCENDING),
+    (models.BOARD_KEY, pymongo.ASCENDING)
+]
 
 BOOT_SEARCH_FIELDS = [
     models.ARCHITECTURE_KEY,
@@ -47,71 +48,9 @@ BOOT_SEARCH_FIELDS = [
     models.STATUS_KEY
 ]
 
-BUILD_SEARCH_FIELDS = [
-    models.GIT_COMMIT_KEY,
-    models.GIT_URL_KEY,
-    models.GIT_BRANCH_KEY
-]
-
-BOOT_SEARCH_SORT = [
-    (models.ARCHITECTURE_KEY, pymongo.ASCENDING),
-    (models.DEFCONFIG_FULL_KEY, pymongo.ASCENDING),
-    (models.BOARD_KEY, pymongo.ASCENDING)
-]
-
-
-# pylint: disable=too-many-arguments
-# pylint: disable=star-args
-def save_report(job, kernel, r_type, status, errors, db_options):
-    """Save the report in the database.
-
-    :param job: The job name.
-    :type job: str
-    :param kernel: The kernel name.
-    :type kernel: str
-    :param r_type: The type of report to save.
-    :type r_type: str
-    :param status: The status of the send action.
-    :type status: str
-    :param errors: A list of errors from the send action.
-    :type errors: list
-    :param db_options: The mongodb database connection parameters.
-    :type db_options: dict
-    """
-    utils.LOG.info("Saving '%s' report for '%s-%s'", r_type, job, kernel)
-
-    name = "%s-%s" % (job, kernel)
-
-    spec = {
-        models.JOB_KEY: job,
-        models.KERNEL_KEY: kernel,
-        models.NAME_KEY: name,
-        models.TYPE_KEY: r_type
-    }
-
-    database = utils.db.get_db_connection(db_options)
-
-    prev_doc = utils.db.find_one2(
-        database[models.REPORT_COLLECTION], spec_or_id=spec)
-
-    if prev_doc:
-        report = mreport.ReportDocument.from_json(prev_doc)
-        report.status = status
-        report.errors = errors
-
-        utils.db.save(database, report)
-    else:
-        report = mreport.ReportDocument(name)
-        report.job = job
-        report.kernel = kernel
-        report.r_type = r_type
-        report.status = status
-        report.errors = errors
-
-        utils.db.save(database, report, manipulate=True)
-
 
 # pylint: disable=too-many-locals
+# pylint: disable=star-args
 def create_boot_report(job, kernel, lab_name, db_options, mail_options=None):
     """Create the boot report email to be sent.
 
@@ -139,42 +78,27 @@ def create_boot_report(job, kernel, lab_name, db_options, mail_options=None):
     if mail_options:
         info_email = mail_options.get("info_email", None)
 
-    database = utils.db.get_db_connection(db_options)
+    total_count, total_unique_data = rcommon.get_total_results(
+        job,
+        kernel,
+        models.BOOT_COLLECTION,
+        db_options,
+        lab_name=lab_name
+    )
+
+    git_commit, git_url, git_branch = rcommon.get_git_data(
+        job, kernel, db_options)
 
     spec = {
         models.JOB_KEY: job,
         models.KERNEL_KEY: kernel,
+        models.STATUS_KEY: models.OFFLINE_STATUS
     }
 
     if lab_name is not None:
         spec[models.LAB_NAME_KEY] = lab_name
 
-    total_results, total_count = utils.db.find_and_count(
-        database[models.BOOT_COLLECTION],
-        0,
-        0,
-        spec=spec,
-        fields=[models.ID_KEY])
-
-    total_unique_data = _get_unique_data(total_results.clone())
-
-    git_results = utils.db.find(
-        database[models.JOB_COLLECTION],
-        0,
-        0,
-        spec=spec,
-        fields=BUILD_SEARCH_FIELDS)
-
-    git_data = _parse_job_results(git_results)
-    if git_data:
-        git_commit = git_data[models.GIT_COMMIT_KEY]
-        git_url = git_data[models.GIT_URL_KEY]
-        git_branch = git_data[models.GIT_BRANCH_KEY]
-    else:
-        git_commit = git_url = git_branch = u"Unknown"
-
-    spec[models.STATUS_KEY] = models.OFFLINE_STATUS
-
+    database = utils.db.get_db_connection(db_options)
     offline_results, offline_count = utils.db.find_and_count(
         database[models.BOOT_COLLECTION],
         0,
@@ -185,10 +109,9 @@ def create_boot_report(job, kernel, lab_name, db_options, mail_options=None):
 
     # MongoDB cursor gets overwritten somehow by the next query. Extract the
     # data before this happens.
+    offline_data = None
     if offline_count > 0:
         offline_data, _, _, _ = _parse_boot_results(offline_results.clone())
-    else:
-        offline_data = None
 
     spec[models.STATUS_KEY] = models.FAIL_STATUS
 
@@ -206,9 +129,9 @@ def create_boot_report(job, kernel, lab_name, db_options, mail_options=None):
 
     # Fill the data structure for the email report creation.
     kwargs = {
-        "base_url": DEFAULT_BASE_URL,
-        "boot_url": DEFAULT_BOOT_URL,
-        "build_url": DEFAULT_BUILD_URL,
+        "base_url": rcommon.DEFAULT_BASE_URL,
+        "boot_url": rcommon.DEFAULT_BOOT_URL,
+        "build_url": rcommon.DEFAULT_BUILD_URL,
         "conflict_count": conflict_count,
         "conflict_data": conflict_data,
         "fail_count": fail_count - conflict_count,
@@ -298,55 +221,8 @@ def create_boot_report(job, kernel, lab_name, db_options, mail_options=None):
     return email_body, subject
 
 
-def _parse_job_results(results):
-    """Parse the job results from the database creating a new data structure.
-
-    This is done to provide a simpler data structure to create the email
-    body.
-
-
-    :param results: The job results to parse.
-    :type results: `pymongo.cursor.Cursor` or a list of dict
-    :return A tuple with the parsed data as dictionary.
-    """
-    parsed_data = None
-
-    for result in results:
-        res_get = result.get
-
-        git_commit = res_get(models.GIT_COMMIT_KEY)
-        git_url = res_get(models.GIT_URL_KEY)
-        git_branch = res_get(models.GIT_BRANCH_KEY)
-
-        parsed_data = {
-            models.GIT_COMMIT_KEY: git_commit,
-            models.GIT_URL_KEY: git_url,
-            models.GIT_BRANCH_KEY: git_branch
-        }
-
-    return parsed_data
-
-
-def _get_unique_data(results):
-    """Get a dictionary with the unique values in the results.
-
-    :param results: The `Cursor` to analyze.
-    :type results: pymongo.cursor.Cursor
-    :return A dictionary with the unique data found in the results.
-    """
-    unique_data = {}
-
-    if isinstance(results, pymongo.cursor.Cursor):
-        unique_data = {
-            models.ARCHITECTURE_KEY: results.distinct(models.ARCHITECTURE_KEY),
-            models.BOARD_KEY: results.distinct(models.BOARD_KEY),
-            models.DEFCONFIG_FULL_KEY: results.distinct(
-                models.DEFCONFIG_FULL_KEY),
-            models.MACH_KEY: results.distinct(models.MACH_KEY)
-        }
-    return unique_data
-
-
+# pylint: disable=too-many-branches
+# pylint: disable=too-many-statements
 def _parse_boot_results(results, intersect_results=None, get_unique=False):
     """Parse the boot results from the database creating a new data structure.
 
@@ -370,12 +246,11 @@ def _parse_boot_results(results, intersect_results=None, get_unique=False):
     """
     parsed_data = {}
     parsed_get = parsed_data.get
-    result_struct = None
     unique_data = None
     intersections = 0
 
     if get_unique:
-        unique_data = _get_unique_data(results)
+        unique_data = rcommon.get_unique_data(results)
 
     for result in results:
         res_get = result.get
@@ -476,24 +351,6 @@ def _search_conflicts(failed, passed):
     return conflict
 
 
-def _count_unique(to_count):
-    """Count the number of values in a list.
-
-    Traverse the list and consider only the valid values (non-None).
-
-    :param to_count: The list to count.
-    :type to_count: list
-    :return The number of element in the list.
-    """
-    total = 0
-    if isinstance(to_count, (types.ListType, types.TupleType)):
-        filtered_list = None
-        filtered_list = [x for x in to_count if x is not None]
-        total = len(filtered_list)
-    return total
-
-
-# pylint: disable=too-many-arguments
 def _create_boot_email(**kwargs):
     """Parse the results and create the email text body to send.
 
@@ -533,12 +390,6 @@ def _create_boot_email(**kwargs):
     :type boot_url: string
     :param build_url: The base URL for the build section of the dashboard.
     :type build_url: string
-    :param git_branch: The name of the branch.
-    :type git_branch: string
-    :param git_commit: The git commit SHA.
-    :type git_commit: string
-    :param git_url: The URL to the git repository
-    :type git_url: string
     :param info_email: The email address for the footer note.
     :type info_email: string
     :return A tuple with the email body and subject as strings.
@@ -556,9 +407,9 @@ def _create_boot_email(**kwargs):
 
     tested_string = None
     if total_unique_data:
-        unique_boards = _count_unique(
+        unique_boards = rcommon.count_unique(
             total_unique_data.get(models.BOARD_KEY, None))
-        unique_socs = _count_unique(
+        unique_socs = rcommon.count_unique(
             total_unique_data.get(models.MACH_KEY, None))
 
         kwargs["unique_boards"] = unique_boards
@@ -577,9 +428,9 @@ def _create_boot_email(**kwargs):
 
         if all([unique_boards > 0, unique_socs > 0]):
             tested_string = tested_two % (boards_str, soc_str)
-        elif unique_boards > 0:
+        elif all([unique_boards > 0, unique_socs == 0]):
             tested_string = tested_one % boards_str
-        elif unique_socs > 0:
+        elif all([unique_socs > 0, unique_boards == 0]):
             tested_string = tested_one % soc_str
 
         if tested_string:
@@ -624,8 +475,9 @@ def _create_boot_email(**kwargs):
     return email_body, subject_str
 
 
+# pylint: disable=invalid-name
 def _get_boot_subject_string(**kwargs):
-    """Create the email subject line.
+    """Create the boot email subject line.
 
     This is used to created the custom email report line based on the number
     of values we have.
@@ -638,12 +490,15 @@ def _get_boot_subject_string(**kwargs):
     :type offline_count: integer
     :param conflict_count: The number of boot reports in conflict.
     :type conflict_count: integer
+    :param pass_count: The number of successful boot reports.
+    :type pass_count: integer
     :param lab_name: The name of the lab.
     :type lab_name: string
     :param job: The name of the job.
     :type job: string
     :param kernel: The name of the kernel.
     :type kernel: string
+    :return The subject string.
     """
     k_get = kwargs.get
     lab_name = k_get("lab_name", None)
diff --git a/app/utils/report/build.py b/app/utils/report/build.py
new file mode 100644
index 0000000..a3e7eac
--- /dev/null
+++ b/app/utils/report/build.py
@@ -0,0 +1,356 @@
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Create the build email report."""
+
+import gettext
+import io
+import pymongo
+
+import models
+import utils.db
+import utils.report.common as rcommon
+
+# Register the translation domain and fallback safely, at the moment we do
+# not care if we have translations or not, we just use gettext to exploit its
+# plural forms capabilities. We mark the email string as translatable though
+# so we might give that feature in the future.
+L10N = gettext.translation(models.I18N_DOMAIN, fallback=True)
+# Register normal Unicode gettext.
+_ = L10N.ugettext
+# Register plural forms Unicode gettext.
+_p = L10N.ungettext
+
+
+BUILD_SEARCH_FIELDS = [
+    models.ARCHITECTURE_KEY,
+    models.DEFCONFIG_FULL_KEY,
+    models.STATUS_KEY
+]
+
+BUILD_SEARCH_SORT = [
+    (models.DEFCONFIG_KEY, pymongo.ASCENDING),
+    (models.DEFCONFIG_FULL_KEY, pymongo.ASCENDING),
+    (models.ARCHITECTURE_KEY, pymongo.ASCENDING)
+]
+
+
+# pylint: disable=too-many-locals
+# pylint: disable=star-args
+def create_build_report(job, kernel, db_options, mail_options=None):
+    """Create the build report email to be sent.
+
+    :param job: The name of the job.
+    :type job: str
+    :param  kernel: The name of the kernel.
+    :type kernel: str
+    :param db_options: The mongodb database connection parameters.
+    :type db_options: dict
+    :param mail_options: The options necessary to connect to the SMTP server.
+    :type mail_options: dict
+    :return A tuple with the email body and subject as strings or None.
+    """
+    kwargs = {}
+    email_body = None
+    subject = None
+    # This is used to provide a footer note in the email report.
+    info_email = None
+
+    fail_count = total_count = 0
+    fail_results = []
+
+    if mail_options:
+        info_email = mail_options.get("info_email", None)
+
+    unique_keys = [models.ARCHITECTURE_KEY, models.DEFCONFIG_FULL_KEY]
+    total_count, total_unique_data = rcommon.get_total_results(
+        job,
+        kernel,
+        models.DEFCONFIG_COLLECTION, db_options, unique_keys=unique_keys
+    )
+
+    git_commit, git_url, git_branch = rcommon.get_git_data(
+        job, kernel, db_options)
+
+    spec = {
+        models.JOB_KEY: job,
+        models.KERNEL_KEY: kernel,
+        models.STATUS_KEY: models.FAIL_STATUS
+    }
+
+    database = utils.db.get_db_connection(db_options)
+    fail_results, fail_count = utils.db.find_and_count(
+        database[models.DEFCONFIG_COLLECTION],
+        0,
+        0,
+        spec=spec,
+        fields=BUILD_SEARCH_FIELDS,
+        sort=BUILD_SEARCH_SORT)
+
+    failed_data = _parse_build_data(fail_results.clone())
+
+    kwargs = {
+        "base_url": rcommon.DEFAULT_BASE_URL,
+        "build_url": rcommon.DEFAULT_BUILD_URL,
+        "fail_count": fail_count,
+        "total_count": total_count,
+        "pass_count": total_count - fail_count,
+        "failed_data": failed_data,
+        "git_branch": git_branch,
+        "git_commit": git_commit,
+        "git_url": git_url,
+        "info_email": info_email,
+        "total_unique_data": total_unique_data,
+        models.JOB_KEY: job,
+        models.KERNEL_KEY: kernel,
+    }
+
+    if all([fail_count == 0, total_count == 0]):
+        utils.LOG.warn(
+            "Nothing found for '%s-%s': no build email report sent",
+            job, kernel)
+    else:
+        email_body, subject = _create_build_email(**kwargs)
+
+    return email_body, subject
+
+
+def _parse_build_data(results):
+    """Parse the build data to provide a writable data structure.
+
+    Loop through the build data found, and create a new dictionary whose keys
+    are the architectures and their values a list of tuples of
+    (defconfig, status).
+
+    :param results: The results to parse.
+    :type results: pymongo.cursor.Cursor.
+    :return A dictionary.
+    """
+    parsed_data = {}
+    arch_keys = parsed_data.viewkeys()
+
+    for result in results:
+        res_get = result.get
+
+        arch = res_get(models.ARCHITECTURE_KEY)
+        defconfig_full = res_get(models.DEFCONFIG_FULL_KEY, None)
+        defconfig = res_get(models.DEFCONFIG_KEY)
+        status = res_get(models.STATUS_KEY)
+
+        struct = ((defconfig_full or defconfig), status)
+
+        if arch in arch_keys:
+            parsed_data[arch].append(struct)
+        else:
+            parsed_data[arch] = []
+            parsed_data[arch].append(struct)
+
+    return parsed_data
+
+
+# pylint: disable=too-many-statements
+def _create_build_email(**kwargs):
+    """Parse the results and create the email text body to send.
+
+    :param job: The name of the job.
+    :type job: str
+    :param  kernel: The name of the kernel.
+    :type kernel: str
+    :param git_commit: The git commit.
+    :type git_commit: str
+    :param git_url: The git url.
+    :type git_url: str
+    :param git_branch: The git branch.
+    :type git_branch: str
+    :param failed_data: The parsed failed results.
+    :type failed_data: dict
+    :param fail_count: The total number of failed results.
+    :type fail_count: int
+    :param total_count: The total number of results.
+    :type total_count: int
+    :param total_unique_data: The unique values data structure.
+    :type total_unique_data: dictionary
+    :param pass_count: The total number of passed results.
+    :type pass_count: int
+    :param base_url: The base URL to build the dashboard links.
+    :type base_url: string
+    :param boot_url: The base URL for the boot section of the dashboard.
+    :type boot_url: string
+    :param build_url: The base URL for the build section of the dashboard.
+    :type build_url: string
+    :param info_email: The email address for the footer note.
+    :type info_email: string
+    :return A tuple with the email body and subject as strings.
+    """
+    k_get = kwargs.get
+    total_unique_data = k_get("total_unique_data", None)
+    failed_data = k_get("failed_data", None)
+    info_email = k_get("info_email", None)
+    fail_count = k_get("fail_count", 0)
+
+    email_body = u""
+    subject_str = _get_build_subject_string(**kwargs)
+
+    built_unique_one = "Built: %s"
+    built_unique_two = "Built: %s, %s"
+
+    built_unique_string = None
+    if total_unique_data:
+        unique_defconfigs = rcommon.count_unique(
+            total_unique_data.get("defconfig_full", None))
+        unique_archs = rcommon.count_unique(
+            total_unique_data.get("arch", None))
+
+        kwargs["unique_defconfigs"] = unique_defconfigs
+        kwargs["unique_archs"] = unique_archs
+
+        defconfig_str = _p(
+            u"%(unique_defconfigs)d unique defconfig",
+            u"%(unique_defconfigs)d unique defconfigs",
+            unique_defconfigs
+        )
+        arch_str = _p(
+            u"%(unique_archs)d unique architecture",
+            u"%(unique_archs)d unique architectures",
+            unique_archs
+        )
+
+        if all([unique_defconfigs > 0, unique_archs > 0]):
+            built_unique_string = built_unique_two % (defconfig_str, arch_str)
+        elif unique_defconfigs > 0:
+            built_unique_string = built_unique_one % defconfig_str
+        elif unique_archs > 0:
+            built_unique_string = built_unique_one % arch_str
+
+        if built_unique_string:
+            built_unique_string = built_unique_string % kwargs
+
+    build_summary_url = u"%(build_url)s/%(job)s/kernel/%(kernel)s/" % kwargs
+
+    tree = _(u"Tree: %(job)s\n") % kwargs
+    branch = _(u"Branch: %(git_branch)s\n") % kwargs
+    git_describe = _(u"Git Describe: %(kernel)s\n") % kwargs
+    git_commit = _(u"Git Commit: %(git_commit)s\n") % kwargs
+    git_url = _(u"Git URL: %(git_url)s\n") % kwargs
+
+    with io.StringIO() as m_string:
+        m_string.write(subject_str)
+        m_string.write(u"\n")
+        m_string.write(u"\n")
+        m_string.write(
+            _(u"Full Build Summary: %s\n") % build_summary_url)
+        m_string.write(u"\n")
+        m_string.write(tree)
+        m_string.write(branch)
+        m_string.write(git_describe)
+        m_string.write(git_commit)
+        m_string.write(git_url)
+
+        if built_unique_string:
+            m_string.write(built_unique_string)
+            m_string.write(u"\n")
+
+        if failed_data:
+            m_string.write(u"\n")
+            m_string.write(
+                _p(
+                    u"Build Failure Detected:\n",
+                    u"Build Failures Detected:\n", fail_count
+                ))
+
+            f_get = failed_data.get
+            for arch in failed_data.viewkeys():
+                m_string.write(u"\n")
+                m_string.write(
+                    _(u"%s:\n") % arch
+                )
+
+                for struct in f_get(arch):
+                    m_string.write(u"\n")
+                    m_string.write(
+                        _(u"    %s: %s") % (struct[0], struct[1])
+                    )
+                m_string.write(u"\n")
+
+        if info_email:
+            m_string.write(u"\n")
+            m_string.write(u"---\n")
+            m_string.write(_(u"For more info write to <%s>") % info_email)
+
+        email_body = m_string.getvalue()
+
+    return email_body, subject_str
+
+
+def _get_build_subject_string(**kwargs):
+    """Create the build email subject line.
+
+    This is used to created the custom email report line based on the number
+    of values we have.
+
+    :param total_count: The total number of build reports.
+    :type total_count: integer
+    :param fail_count: The number of failed build reports.
+    :type fail_count: integer
+    :param pass_count: The number of successful build reports.
+    :type pass_count: integer
+    :param job: The name of the job.
+    :type job: string
+    :param kernel: The name of the kernel.
+    :type kernel: string
+    :return The subject string.
+    """
+    k_get = kwargs.get
+    fail_count = k_get("fail_count", 0)
+    total_count = k_get("total_count", 0)
+
+    subject_str = u""
+
+    base_subject = _(u"%(job)s build")
+    kernel_name = _(u"(%(kernel)s)")
+    failed_builds = _(u"%(fail_count)d failed")
+    passed_builds = _(u"%(pass_count)d passed")
+    total_builds = _p(
+        u"%(total_count)d build", u"%(total_count)d builds", total_count)
+
+    subject_substitutions = {
+        "build_name": base_subject,
+        "total_builds": total_builds,
+        "passed_builds": passed_builds,
+        "failed_builds": failed_builds,
+        "kernel_name": kernel_name,
+    }
+
+    subject_all_pass = _(
+        u"%(build_name)s: %(total_builds)s: %(passed_builds)s %(kernel_name)s")
+    subject_all_fail = _(
+        u"%(build_name)s: %(total_builds)s: %(failed_builds)s %(kernel_name)s"
+    )
+    subject_pass_and_fail = _(
+        u"%(build_name)s: %(total_builds)s: %(passed_builds)s, "
+        "%(failed_builds)s %(kernel_name)s"
+    )
+
+    if all([fail_count > 0, fail_count != total_count]):
+        subject_str = subject_pass_and_fail
+    elif all([fail_count > 0, fail_count == total_count]):
+        subject_str = subject_all_fail
+    elif fail_count == 0:
+        subject_str = subject_all_pass
+
+    # Perform all the normal placeholder substitutions.
+    subject_str = subject_str % subject_substitutions
+    # Now fill in the values.
+    subject_str = subject_str % kwargs
+
+    return subject_str
diff --git a/app/utils/report/common.py b/app/utils/report/common.py
new file mode 100644
index 0000000..e271d7a
--- /dev/null
+++ b/app/utils/report/common.py
@@ -0,0 +1,242 @@
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Common functions for email reports creation."""
+
+import pymongo
+import types
+
+import models
+import models.report as mreport
+import utils
+
+
+DEFAULT_UNIQUE_KEYS = [
+    models.ARCHITECTURE_KEY,
+    models.BOARD_KEY,
+    models.DEFCONFIG_FULL_KEY,
+    models.MACH_KEY
+]
+
+JOB_SEARCH_FIELDS = [
+    models.GIT_COMMIT_KEY,
+    models.GIT_URL_KEY,
+    models.GIT_BRANCH_KEY
+]
+
+DEFAULT_BASE_URL = u"http://kernelci.org"
+DEFAULT_BOOT_URL = u"http://kernelci.org/boot/all/job"
+DEFAULT_BUILD_URL = u"http://kernelci.org/build"
+
+
+def save_report(job, kernel, r_type, status, errors, db_options):
+    """Save the email report status in the database.
+
+    It does not save the actual email report sent.
+
+    :param job: The job name.
+    :type job: str
+    :param kernel: The kernel name.
+    :type kernel: str
+    :param r_type: The type of report to save.
+    :type r_type: str
+    :param status: The status of the send action.
+    :type status: str
+    :param errors: A list of errors from the send action.
+    :type errors: list
+    :param db_options: The mongodb database connection parameters.
+    :type db_options: dict
+    """
+    utils.LOG.info("Saving '%s' report for '%s-%s'", r_type, job, kernel)
+
+    name = "%s-%s" % (job, kernel)
+
+    spec = {
+        models.JOB_KEY: job,
+        models.KERNEL_KEY: kernel,
+        models.NAME_KEY: name,
+        models.TYPE_KEY: r_type
+    }
+
+    database = utils.db.get_db_connection(db_options)
+
+    prev_doc = utils.db.find_one2(
+        database[models.REPORT_COLLECTION], spec_or_id=spec)
+
+    if prev_doc:
+        report = mreport.ReportDocument.from_json(prev_doc)
+        report.status = status
+        report.errors = errors
+
+        utils.db.save(database, report)
+    else:
+        report = mreport.ReportDocument(name)
+        report.job = job
+        report.kernel = kernel
+        report.r_type = r_type
+        report.status = status
+        report.errors = errors
+
+        utils.db.save(database, report, manipulate=True)
+
+
+def get_unique_data(results, unique_keys=None):
+    """Get a dictionary with the unique values in the results.
+
+    :param results: The `Cursor` to analyze.
+    :type results: pymongo.cursor.Cursor
+    :return A dictionary with the unique data found in the results.
+    """
+    unique_data = {}
+
+    if not unique_keys:
+        unique_keys = DEFAULT_UNIQUE_KEYS
+
+    def _unique_value(results, keys):
+        """Internal generator to return the unique values.
+
+        :param results: The pymongo Cursor to iterate.
+        :param keys: The list of keys.
+        :type keys: list
+        :return A tuple (key, value).
+        """
+        for key in keys:
+            yield key, results.distinct(key)
+
+    if isinstance(results, pymongo.cursor.Cursor):
+        unique_data = {
+            k: v for k, v in _unique_value(results, unique_keys)
+        }
+    return unique_data
+
+
+def count_unique(to_count):
+    """Count the number of values in a list.
+
+    Traverse the list and consider only the valid values (non-None).
+
+    :param to_count: The list to count.
+    :type to_count: list
+    :return The number of element in the list.
+    """
+    total = 0
+    if isinstance(to_count, (types.ListType, types.TupleType)):
+        filtered_list = None
+        filtered_list = [x for x in to_count if x is not None]
+        total = len(filtered_list)
+    return total
+
+
+def parse_job_results(results):
+    """Parse the job results from the database creating a new data structure.
+
+    This is done to provide a simpler data structure to create the email
+    body.
+
+
+    :param results: The job results to parse.
+    :type results: `pymongo.cursor.Cursor` or a list of dict
+    :return A tuple with the parsed data as dictionary.
+    """
+    parsed_data = None
+
+    for result in results:
+        if result:
+            res_get = result.get
+
+            git_commit = res_get(models.GIT_COMMIT_KEY, u"Unknown")
+            git_url = res_get(models.GIT_URL_KEY, u"Unknown")
+            git_branch = res_get(models.GIT_BRANCH_KEY, u"Unknown")
+
+            parsed_data = {
+                models.GIT_COMMIT_KEY: git_commit,
+                models.GIT_URL_KEY: git_url,
+                models.GIT_BRANCH_KEY: git_branch
+            }
+
+    return parsed_data
+
+
+def get_git_data(job, kernel, db_options):
+    """Retrieve the git data from a job.
+
+    :param job: The job name.
+    :type job: string
+    :param kernel: The kernel name.
+    :type kernel: string
+    :param db_options: The mongodb database connection parameters.
+    :type db_options: dict
+    :return A tuple with git commit, url and branch.
+    """
+    spec = {
+        models.JOB_KEY: job,
+        models.KERNEL_KEY: kernel
+    }
+
+    database = utils.db.get_db_connection(db_options)
+
+    git_results = utils.db.find(
+        database[models.JOB_COLLECTION],
+        0,
+        0,
+        spec=spec,
+        fields=JOB_SEARCH_FIELDS)
+
+    git_data = parse_job_results(git_results)
+    if git_data:
+        git_commit = git_data[models.GIT_COMMIT_KEY]
+        git_url = git_data[models.GIT_URL_KEY]
+        git_branch = git_data[models.GIT_BRANCH_KEY]
+    else:
+        git_commit = git_url = git_branch = u"Unknown"
+
+    return (git_commit, git_url, git_branch)
+
+
+def get_total_results(
+        job, kernel, collection, db_options, lab_name=None, unique_keys=None):
+    """Retrieve the total count and the unique data for a collection.
+
+    :param job: The job name.
+    :type job: string
+    :param kernel: The kernel name.
+    :type kernel: string
+    :param collection: The collection name.
+    :type collection: string.
+    :param db_options: The mongodb database connection parameters.
+    :type db_options: dict
+    :param lab_name: The lab name.
+    :type lab_name: string
+    :param unique_keys: The unique keys to count.
+    :type unique_keys: list
+    :return A tuple with the total count and the unique elements.
+    """
+    spec = {
+        models.JOB_KEY: job,
+        models.KERNEL_KEY: kernel,
+    }
+
+    if lab_name:
+        spec[models.LAB_NAME_KEY] = lab_name
+
+    database = utils.db.get_db_connection(db_options)
+
+    total_results, total_count = utils.db.find_and_count(
+        database[collection],
+        0,
+        0,
+        spec=spec)
+
+    total_unique_data = get_unique_data(total_results.clone(), unique_keys)
+
+    return (total_count, total_unique_data)
diff --git a/app/utils/report/tests/__init__.py b/app/utils/report/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/app/utils/report/tests/__init__.py
diff --git a/app/utils/report/tests/test_report_common.py b/app/utils/report/tests/test_report_common.py
new file mode 100644
index 0000000..0c8d829
--- /dev/null
+++ b/app/utils/report/tests/test_report_common.py
@@ -0,0 +1,78 @@
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test class for the email report functions."""
+
+import unittest
+
+import utils.report.common as rcommon
+
+
+class TestReportCommon(unittest.TestCase):
+
+    def test_count_unique_with_string(self):
+        to_count = ""
+        self.assertEqual(0, rcommon.count_unique(to_count))
+
+    def test_count_unique_with_dict(self):
+        to_count = {}
+        self.assertEqual(0, rcommon.count_unique(to_count))
+
+    def test_count_unique_with_list(self):
+        to_count = [None, "a", "b", None]
+        self.assertEqual(2, rcommon.count_unique(to_count))
+
+    def test_count_unique_with_tuple(self):
+        to_count = (None, "a", "b", None, None, None, "c")
+        self.assertEqual(3, rcommon.count_unique(to_count))
+
+    def test_parse_job_results_empty(self):
+        results = [{}]
+        self.assertIsNone(rcommon.parse_job_results(results))
+
+    def test_parse_job_results_with_git_commit(self):
+        results = [{"git_commit": "12345"}]
+        expected = {
+            "git_commit": "12345",
+            "git_branch": "Unknown",
+            "git_url": "Unknown"
+        }
+        self.assertDictEqual(expected, rcommon.parse_job_results(results))
+
+    def test_parse_job_results_with_git_branch(self):
+        results = [{"git_branch": "branch"}]
+        expected = {
+            "git_commit": "Unknown",
+            "git_branch": "branch",
+            "git_url": "Unknown"
+        }
+        self.assertDictEqual(expected, rcommon.parse_job_results(results))
+
+    def test_parse_job_results_with_git_url(self):
+        results = [{"git_url": "url"}]
+        expected = {
+            "git_commit": "Unknown",
+            "git_branch": "Unknown",
+            "git_url": "url"
+        }
+        self.assertDictEqual(expected, rcommon.parse_job_results(results))
+
+    def test_parse_job_results_with_git_all(self):
+        results = [
+            {"git_url": "url", "git_commit": "12345", "git_branch": "branch"}]
+        expected = {
+            "git_commit": "12345",
+            "git_branch": "branch",
+            "git_url": "url"
+        }
+        self.assertDictEqual(expected, rcommon.parse_job_results(results))