run-jjb.py

#!/usr/bin/python3

import os
import shutil
import signal
import string
import subprocess
import sys
import xml.etree.ElementTree
from distutils.spawn import find_executable


# provide a convenience wrapper for running a command.  If command
# fails for any reason, it should raise a ValueError and let caller
# handle it
def safe_run(args, check=True):
    try:
        proc = subprocess.run(
                        args,
                        stdout=subprocess.PIPE,
                        stderr=subprocess.STDOUT,
                        check=check)
    except(OSError, ValueError) as e:
        raise ValueError("command '%s' failed: %s" % (" ".join(args), e))
    except(subprocess.CalledProcessError) as e:
        raise ValueError("command '%s' failed with %s:\n%s" % (
            e.cmd, e.returncode, e.output.decode()))
    except(Exception) as e:
        raise ValueError("unknown error running command '%s':\n%s" %
            (" ".join(args), e))

    return proc


def findparentfiles(fname):
    filelist = []
    newlist = []
    args = ['grep', '-rl', '--exclude-dir=.git', fname]

    # Set False for the check argument to subprocess run... we don't care
    # if grep doesn't find any files
    proc = safe_run(args, False)
    data = proc.stdout.decode()

    for filename in data.splitlines():
        if filename.endswith('.yaml') and '/' not in filename:
            filelist.append(filename)
        else:
            newlist = findparentfiles(filename)
            for tempname in newlist:
                filelist.append(tempname)
    return filelist


jjb_cmd = find_executable('jenkins-jobs') or sys.exit('jenkins-jobs is not found.')
jjb_args = [jjb_cmd]

jjb_user = os.environ.get('JJB_USER')
jjb_password = os.environ.get('JJB_PASSWORD')
if jjb_user is not None and jjb_password is not None:
    jenkins_jobs_ini = ('[job_builder]\n'
                        'ignore_cache=True\n'
                        'keep_descriptions=False\n'
                        '\n'
                        '[jenkins]\n'
                        'user=%s\n'
                        'password=%s\n'
                        'url=https://ci.linaro.org/\n' % (jjb_user, jjb_password))
    with open('jenkins_jobs.ini', 'w') as f:
        f.write(jenkins_jobs_ini)
    jjb_args.append('--conf=jenkins_jobs.ini')

jjb_test_args = list(jjb_args)
jjb_delete_args = list(jjb_args)

# !!! "update" below and through out this file is replaced by "test" (using sed)
# !!! in the sanity-check job.
main_action = 'update'
jjb_args.extend([main_action, 'template.yaml'])
jjb_test_args.extend(['test', '-o', 'out/', 'template.yaml'])
jjb_delete_args.extend(['delete'])

if main_action == 'test':
    # Dry-run, don't delete jobs.
    jjb_delete_args.insert(0, 'echo')

try:
    git_args = ['git', 'diff', '--raw',
                os.environ.get('GIT_PREVIOUS_COMMIT'),
                os.environ.get('GIT_COMMIT')]
    proc = safe_run(git_args)
    data = proc.stdout.decode()
except (ValueError) as e:
    raise ValueError("%s" % e)

filelist = []
deletelist = []
files = []
for line in data.splitlines():
    # Format of the git-diff; we only need OPERATION and FILE1
    #
    # :<OLD MODE> <NEW MODE> <OLD REF> <NEW REF> <OPERATION> <FILE1> <FILE2>
    elems = line.split()
    operation = elems[4][0]
    filename = elems[5]

    if filename.endswith('.yaml') and '/' not in filename:
        # No point trying to test deleted jobs because they don't exist any
        # more.
        if operation == 'D':
            deletelist.append(filename[:-5])
            continue
        # operation R100 is 100% rename, which means sixth element is the renamed file
        if operation == 'R':
            filename = elems[6]
            # delete old job name
            deletelist.append(elems[5][:-5])
        filelist.append(filename)
    else:
        files = findparentfiles(filename)
        for tempname in files:
            filelist.append(tempname)

# Remove duplicate entries in the list
filelist = list(set(filelist))

for conf_filename in filelist:
    with open(conf_filename) as f:
        buffer = f.read()
        template = string.Template(buffer)
        buffer = template.safe_substitute(
            AUTH_TOKEN=os.environ.get('AUTH_TOKEN'),
            LT_QCOM_KEY=os.environ.get('LT_QCOM_KEY'),
            LAVA_USER=os.environ.get('LAVA_USER'),
            LAVA_TOKEN=os.environ.get('LAVA_TOKEN'))
        with open('template.yaml', 'w') as f:
            f.write(buffer)

        proc = safe_run(jjb_args)
        data = proc.stdout.decode()

        try:
            shutil.rmtree('out/', ignore_errors=True)

            proc = safe_run(jjb_test_args)
            data = proc.stdout.decode()

            proc = safe_run(['ls', 'out/'])
            data = proc.stdout.decode()

            for filename in data.splitlines():
                # old job conf might have been removed because the job is now generated through the template
                # do not delete the job in this case
                if filename in deletelist:
                    deletelist.remove(filename)

                conf_name=os.path.splitext(conf_filename)[0]
                conf_name=conf_name[:len(filename)]
                if not filename.startswith(conf_name):
                    raise ValueError("Job name %s does not match the file it is in: %s" % (filename, conf_name))
                try:
                    xmlroot = xml.etree.ElementTree.parse('out/' + filename).getroot()
                    disabled = next(xmlroot.iterfind('disabled')).text
                    if disabled != 'true':
                        continue
                    displayName = next(xmlroot.iterfind('displayName')).text
                    if displayName != 'DELETE ME':
                        continue
                except:
                    continue

                deletelist.append(filename)

        except (OSError, ValueError) as e:
            raise ValueError("%s" % e)

        shutil.rmtree('out/', ignore_errors=True)
        os.remove('template.yaml')


for deletejob in deletelist:
    delete_args = list(jjb_delete_args)
    delete_args.extend([deletejob])

    proc = safe_run(delete_args)
    data = proc.stdout.decode()

    print(data)

if os.path.exists('jenkins_jobs.ini'):
    os.remove('jenkins_jobs.ini')