aboutsummaryrefslogtreecommitdiff
path: root/lava_results_app/dbutils.py
blob: 435c27c6b2c6d94e9ee1269fb6553c8cce379eee (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
# Copyright (C) 2015 Linaro Limited
#
# Author: Neil Williams <neil.williams@linaro.org>
#
# This file is part of Lava Server.
#
# Lava Server is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License version 3
# as published by the Free Software Foundation
#
# Lava Server is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Lava Server.  If not, see <http://www.gnu.org/licenses/>.

# pylint: disable=no-member,too-many-locals,too-many-nested-blocks,
# pylint: disable=too-many-return-statements,ungrouped-imports

import hashlib
import os
import yaml
import sys
import logging
import decimal

from collections import OrderedDict  # pylint: disable=unused-import
from lava_results_app.models import (
    TestSuite,
    TestSet,
    TestCase,
    TestData,
    ActionData,
    MetaType,
)
from lava_results_app.utils import debian_package_version
from django.core.exceptions import MultipleObjectsReturned
from lava_common.timeout import Timeout

if sys.version_info[0] == 2:
    # Python 2.x
    from urllib import quote
elif sys.version_info[0] == 3:
    # For Python 3.0 and later
    from urllib.parse import quote
    basestring = str


def _check_for_testset(result_dict, suite):
    """
    The presence of the test_set key indicates the start and usage of a TestSet.
    Get or create and populate the definition based on that set.
    # {date: pass, test_definition: install-ssh, test_set: first_set}
    :param result_dict: lava-test-shell results
    :param suite: current test suite
    """
    logger = logging.getLogger('lava-master')
    testset = None
    if 'set' in result_dict:
        set_name = result_dict['set']
        if set_name != quote(set_name):
            msg = "Invalid testset name '%s', ignoring." % set_name
            suite.job.set_failure_comment(msg)
            logger.warning(msg)
            return None
        testset, _ = TestSet.objects.get_or_create(name=set_name, suite=suite)
        logger.debug("%s", testset)
    return testset


def append_failure_comment(job, msg):
    if not job.failure_comment:
        job.failure_comment = ''
    job.failure_comment += msg[:256]
    job.save(update_fields=["failure_comment"])


def create_metadata_store(results, job):
    """
    Uses the OrderedDict import to correctly handle
    the yaml.load
    """
    if 'extra' not in results:
        return None
    level = results.get('level')
    if level is None:
        return None

    logger = logging.getLogger('lava-master')
    stub = "%s-%s-%s.yaml" % (results['definition'], results['case'], level)
    meta_filename = os.path.join(job.output_dir, 'metadata', stub)
    if not os.path.exists(os.path.dirname(meta_filename)):
        os.makedirs(os.path.dirname(meta_filename), mode=0o755)
    if os.path.exists(meta_filename):
        with open(meta_filename, 'r') as existing_store:
            data = yaml.load(existing_store)
        data.update(results['extra'])
    else:
        data = results['extra']
    try:
        with open(meta_filename, 'w') as extra_store:
            yaml.dump(data, extra_store)
    except (OSError, IOError) as exc:  # LAVA-847
        msg = "[%d] Unable to create metadata store: %s" % (job.id, exc)
        logger.error(msg)
        append_failure_comment(job, msg)
        return None
    return meta_filename


def map_scanned_results(results, job, meta_filename):  # pylint: disable=too-many-branches,too-many-statements,too-many-return-statements
    """
    Sanity checker on the logged results dictionary
    :param results: results logged via the slave
    :param job: the current test job
    :param meta_filename: YAML store for results metadata
    :return: the TestCase object that should be saved to the database.
             None on error.
    """
    logger = logging.getLogger('lava-master')

    if not isinstance(results, dict):
        append_failure_comment(job, "[%d] %s is not a dictionary" % (job.id, results))
        return None

    if not {"definition", "case", "result"}.issubset(set(results.keys())):
        append_failure_comment(job, "Missing some keys (\"definition\", \"case\" or \"result\") in %s" % results)
        return None

    if 'extra' in results:
        results['extra'] = meta_filename

    metadata = yaml.dump(results)
    if len(metadata) > 4096:  # bug 2471 - test_length unit test
        msg = "[%d] Result metadata is too long. %s" % (job.id, metadata)
        logger.error(msg)
        append_failure_comment(job, msg)
        metadata = ""

    suite, _ = TestSuite.objects.get_or_create(name=results["definition"], job=job)
    testset = _check_for_testset(results, suite)

    name = results["case"].strip()

    test_case = None
    if suite.name == "lava":
        try:
            result_val = TestCase.RESULT_MAP[results['result']]
        except KeyError:
            logger.error("[%d] Unable to MAP result \"%s\"", job.id, results['result'])
            return None

        measurement = None
        units = ''
        if 'duration' in results:
            measurement = results['duration']
            units = 'seconds'
        test_case = TestCase(name=name,
                             suite=suite,
                             test_set=testset,
                             metadata=metadata,
                             measurement=measurement,
                             units=units,
                             result=result_val)

    else:
        result = results["result"]
        measurement = None
        units = ''
        if testset:
            logger.debug("%s/%s/%s %s", suite, testset, name, result)
        else:
            logger.debug("%s/%s %s", suite, name, result)
        if 'measurement' in results:
            measurement = results['measurement']
        if 'units' in results:
            units = results['units']
            logger.debug("%s/%s %s%s", suite, name, measurement, units)
        if result not in TestCase.RESULT_MAP:
            logger.warning("[%d] Unrecognised result: '%s' for test case '%s'", job.id, result, name)
            return None
        try:
            test_case = TestCase(name=name,
                                 suite=suite,
                                 test_set=testset,
                                 result=TestCase.RESULT_MAP[result],
                                 metadata=metadata,
                                 measurement=measurement,
                                 units=units)
        except decimal.InvalidOperation:
            logger.exception("[%d] Unable to create test case %s", job.id, name)
    return test_case


def _add_parameter_metadata(prefix, definition, dictionary, label):
    if 'parameters' in definition and isinstance(definition['parameters'], dict):
        for paramkey, paramvalue in definition['parameters'].items():
            if paramkey == 'yaml_line':
                continue
            dictionary['%s.%s.parameters.%s' % (prefix, label, paramkey)] = paramvalue


def _get_job_metadata(job):
    retval = {}
    # Add original_definition checksum to metadata
    retval.update({
        'definition-checksum': hashlib.md5(
            job.original_definition.encode('utf-8')).hexdigest()
    })
    # Add lava-server-version to metadata
    packaged = debian_package_version()
    if packaged:
        retval.update({
            'lava-server-version': packaged
        })
    return retval


def _get_action_metadata(data):  # pylint: disable=too-many-branches,too-many-nested-blocks,too-many-statements
    if not isinstance(data, list):
        return None
    retval = {}
    for action in data:
        deploy = [dict.get(action, 'deploy')]
        count = 0
        for block in deploy:
            if not block:
                continue
            namespace = block.get('namespace', None)
            prefix = "deploy.%d.%s" % (count, namespace) if namespace else 'deploy.%d' % count
            value = block.get('method', None)
            if value:
                retval['%s.method' % prefix] = value
                count += 1
        boot = [dict.get(action, 'boot')]
        count = 0
        for block in boot:
            if not block:
                continue
            namespace = block.get('namespace', None)
            prefix = "boot.%d.%s" % (count, namespace) if namespace else 'boot.%d' % count
            value = block.get('commands', None)
            if value:
                retval['%s.commands' % prefix] = value
            value = block.get('method', None)
            if value:
                retval['%s.method' % prefix] = value
            value = block.get('type', None)
            if value:
                retval['%s.type' % prefix] = value
            count += 1
        test = [dict.get(action, 'test')]
        count = 0
        for block in test:
            if not block:
                continue
            namespace = block.get('namespace', None)
            definitions = [dict.get(block, 'definitions')][0]
            if not definitions:
                monitors = [dict.get(block, 'monitors')][0]
                if monitors:
                    if isinstance(monitors, list):
                        for monitor in monitors:
                            prefix = "test.%d.%s" % (count, namespace) if namespace else 'test.%d' % count
                            retval['%s.monitor.name' % prefix] = monitor['name']
                            count += 1
            else:
                for definition in definitions:
                    if definition['from'] == 'inline':
                        run = definition['repository'].get('run', None)
                        # an inline repo without test cases will not get reported.
                        steps = [dict.get(run, 'steps')][0] if run else None
                        if steps is not None and 'lava-test-case' in steps:
                            prefix = "test.%d.%s" % (count, namespace) if namespace else 'test.%d' % count
                        else:
                            # store the fact that an inline exists but would not generate any testcases
                            prefix = 'omitted.%d.%s' % (count, namespace) if namespace else 'omitted.%d' % count
                        retval['%s.inline.name' % prefix] = definition['name']
                        _add_parameter_metadata(prefix=prefix, definition=definition,
                                                dictionary=retval, label='inline')
                        retval['%s.inline.path' % prefix] = definition['path']
                    else:
                        prefix = "test.%d.%s" % (count, namespace) if namespace else 'test.%d' % count
                        # FIXME: what happens with remote definition without lava-test-case?
                        retval['%s.definition.name' % prefix] = definition['name']
                        retval['%s.definition.path' % prefix] = definition['path']
                        retval['%s.definition.from' % prefix] = definition['from']
                        retval['%s.definition.repository' % prefix] = definition['repository']
                        _add_parameter_metadata(prefix=prefix, definition=definition,
                                                dictionary=retval, label='definition')
                    count += 1
    return retval


def _get_device_metadata(data):
    devicetype = data.get('device_type', None)
    return {
        'target.device_type': devicetype
    }


def build_action(action_data, testdata, submission):
    # test for a known section
    logger = logging.getLogger('lava-master')
    if 'section' not in action_data:
        logger.warning("Invalid action data - missing section")
        return

    metatype = MetaType.get_section(action_data['section'])
    if metatype is None:  # 0 is allowed
        logger.debug("Unrecognised metatype in action_data: %s", action_data['section'])
        return
    # lookup the type from the job definition.
    type_name = MetaType.get_type_name(action_data, submission)
    if not type_name:
        logger.debug(
            "type_name failed for %s metatype %s",
            action_data['section'], MetaType.TYPE_CHOICES[metatype])
        return
    action_meta, _ = MetaType.objects.get_or_create(name=type_name,
                                                    metatype=metatype)
    max_retry = action_data.get('max_retries')

    # find corresponding test case
    match_case = None
    test_cases = TestCase.objects.filter(suite__job=testdata.testjob, suite__name='lava')
    for case in test_cases:
        if 'level' in case.action_metadata:
            if case.action_metadata['level'] == action_data['level']:
                match_case = case

    # maps the static testdata derived from the definition to the runtime pipeline construction
    ActionData.objects.create(
        action_name=action_data['name'],
        action_level=action_data['level'],
        action_summary=action_data['summary'],
        testdata=testdata,
        action_description=action_data['description'],
        meta_type=action_meta,
        max_retries=max_retry,
        timeout=int(Timeout.parse(action_data['timeout'])),
        testcase=match_case
    )


def walk_actions(data, testdata, submission):
    for action in data:
        build_action(action, testdata, submission)
        if 'pipeline' in action:
            walk_actions(action['pipeline'], testdata, submission)


def map_metadata(description, job):
    """
    Generate metadata from the combination of the pipeline definition
    file (after any parsing for protocols) and the pipeline description
    into static metadata (TestData) related to this specific job
    The description itself remains outside the database - it will need
    to be made available as a download link.
    :param description: the pipeline description output
    :param job: the TestJob to associate
    :return: True on success, False on error
    """
    logger = logging.getLogger('lava-master')
    try:
        submission_data = yaml.load(job.definition)
        description_data = yaml.load(description)
    except yaml.YAMLError as exc:
        logger.exception("[%s] %s", job.id, exc)
        return False
    try:
        testdata, created = TestData.objects.get_or_create(testjob=job)
    except MultipleObjectsReturned:
        # only happens for small number of jobs affected by original bug.
        logger.info("[%s] skipping alteration of duplicated TestData", job.id)
        return False
    if not created:
        # prevent updates of existing TestData
        logger.debug("[%s] skipping alteration of existing TestData", job.id)
        return False

    # get job-action metadata
    if description is None:
        logger.warning("[%s] skipping empty description", job.id)
        return
    if not description_data:
        logger.warning("[%s] skipping invalid description data", job.id)
        return
    if 'job' not in description_data:
        logger.warning("[%s] skipping description without a job.", job.id)
        return
    action_values = _get_action_metadata(description_data['job']['actions'])
    for key, value in action_values.items():
        if not key or not value:
            logger.warning('[%s] Missing element in job. %s: %s', job.id, key, value)
            continue
        testdata.attributes.create(name=key, value=value)

    # get common job metadata
    job_metadata = _get_job_metadata(job)
    for key, value in job_metadata.items():
        testdata.attributes.create(name=key, value=value)

    # get metadata from device
    device_values = _get_device_metadata(description_data['device'])
    for key, value in device_values.items():
        if not key or not value:
            logger.warning('[%s] Missing element in device. %s: %s', job.id, key, value)
            continue
        testdata.attributes.create(name=key, value=value)

    # Add metadata from job submission data.
    if "metadata" in submission_data:
        for key in submission_data["metadata"]:
            value = submission_data["metadata"][key]
            if not key or not value:
                logger.warning('[%s] Missing element in job. %s: %s', job.id, key, value)
                continue
            testdata.attributes.create(name=key, value=value)

    walk_actions(description_data['pipeline'], testdata, submission_data)
    return True


def testcase_export_fields():
    """
    Keep this list in sync with the keys in export_testcase
    :return: list of fields used in export_testcase
    """
    return [
        'job', 'suite', 'result', 'measurement', 'unit',
        'duration', 'timeout',
        'logged', 'level', 'metadata', 'url', 'name', 'id'
    ]


def export_testcase(testcase, with_buglinks=False):
    """
    Returns string versions of selected elements of a TestCase
    Unicode causes issues with CSV and can complicate YAML parsing
    with non-python parsers.
    :param testcase: list of TestCase objects
    :return: Dictionary containing relevant information formatted for export
    """
    metadata = dict(testcase.action_metadata) if testcase.action_metadata else {}
    extra_source = []
    extra_data = metadata.get('extra', None)
    if isinstance(extra_data, basestring) and os.path.exists(extra_data):
        with open(metadata['extra'], 'r') as extra_file:
            items = yaml.load(extra_file, Loader=yaml.CLoader)
        # hide the !!python OrderedDict prefix from the output.
        for key, value in items.items():
            extra_source.append({key: value})
        metadata['extra'] = extra_source
    casedict = {
        'name': str(testcase.name),
        'job': str(testcase.suite.job_id),
        'suite': str(testcase.suite.name),
        'result': str(testcase.result_code),
        'measurement': str(testcase.measurement),
        'unit': str(testcase.units),
        'level': metadata.get('level', ''),
        'url': str(testcase.get_absolute_url()),
        'id': str(testcase.id),
        'logged': str(testcase.logged),
        'metadata': metadata,
    }
    if with_buglinks:
        casedict['buglinks'] = [str(url) for url in testcase.buglinks.values_list('url', flat=True)]

    return casedict


def testsuite_export_fields():
    """
    Keep this list in sync with the keys in export_testsuite
    :return: list of fields used in export_testsuite
    """
    return [
        'name', 'job', 'id'
    ]


def export_testsuite(testsuite):
    """
    Returns string versions of selected elements of a TestSuite
    :param testsuite: TestSuite object
    :return: Dictionary containing relevant information formatted for export
    """
    suitedict = {
        'name': str(testsuite.name),
        'job': str(testsuite.job_id),
        'id': str(testsuite.id),
    }
    return suitedict