build_images.py

#!/usr/bin/python

# Copyright (C) 2012 Linaro
#
# Author: Andy Doan <andy.doan@linaro.org>
#
# This file is part of Linaro Daily Prebuilt Images.
#
# Linaro Daily Prebuilt Images is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# Linaro Daily Prebuilt Images is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Linaro Image Tools; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
# USA.

from linaro_fetch_image import fetch_image
from linaro_image_tools import cmd_runner

import crawler

import argparse
import datetime
import time
import hashlib
import os
import re
import urlparse
import string

class HwPack:
    def __init__(self, lmcname, eula=False, pre_inst_script=None):
        self.lmcname = lmcname
        if eula:
            self.eula = 'EULA.txt'
        else:
            self.eula = 'OPEN-EULA.txt'
        self.pre_inst_script = pre_inst_script

    def do_eula(self, imgfile):
        fname = '%s/%s' % (os.path.dirname(imgfile), self.eula)
        with file(fname, 'a'):
            pass #just need the file to exist

# a mapping of hwpack name to l-m-c name
HWPACKS = {
    'arndale': HwPack('arndale'),
    'arndale-be': HwPack('arndale'),
    'beagleboard': HwPack('beagle'),
    'beaglebone': HwPack('beaglebone'),
    'efikamx': HwPack('efikamx'),
    'highbank': HwPack('highbank'),
    'igep': HwPack('igep'),
    'leb-origen': HwPack('origen'),
    'lsk-vexpress': HwPack('vexpress'),
    'lsk-vexpress64': HwPack('vexpress'),
    'lt-mx5': HwPack('mx53loco'),
    'lt-mx6': HwPack('mx6qsabrelite'),
    'lt-origen': HwPack('origen'),
    'lt-panda': HwPack('panda'),
    'lt-panda-x11-base': HwPack('panda'),
    'lt-snowball': HwPack('snowball_sd', True, 'ste-preinstall.sh'),
    'lt-snowball-x11-base': HwPack('snowball_sd', True, 'ste-preinstall.sh'),
    'lt-vexpress': HwPack('vexpress'),
    'origen': HwPack('origen'),
    'overo': HwPack('overo'),
    'panda': HwPack('panda'),
    'panda-be': HwPack('panda'),
    'snowball': HwPack('snowball_sd', True),
    'ti-panda-x11-base': HwPack('panda'),
    'vexpress': HwPack('vexpress'),
    'vexpress64': HwPack('vexpress'),
}
# a mapping of binary image to its image_file size
BINARIES = {
    'alip': '2G',
    'developer': '1G',
    'lamp-armv8': '2G',
    'leg-java-armv8': '4G',
    'linarotv-xbmc': '3G',
    'minimal-armv8': '512M',
    'nano': '1G',
    'nano-lava': '1G',
    'sdk-armv8': '2G',
    'server': '1G',
    'ubuntu-desktop': '3G',
}

SNAPSHOTS_URL = "http://snapshots.linaro.org"

def today():
    d = datetime.date.today()
    return "%d%02d%02d" % (d.year, d.month, d.day)

def download(dm, url):
    print "INFO: Fetching", url
    before = time.time()
    dfile = dm.download(url, None, verbose=False)
    delta_min, delta_secs = divmod(int(time.time() - before), 60)
    print "INFO: Downloaded file '%s' in %d min(s) and %d sec(s)" % (
                    dfile, delta_min, delta_secs)
    return dfile

def compress_image(imgfile):
    print "INFO: compressing %s" % imgfile
    before = time.time()
    args = ('gzip', imgfile)
    cmd_runner.run(args).wait()
    delta_min, delta_secs = divmod(int(time.time() - before), 60)
    print "INFO: file compressed in %d min(s) and %d sec(s)" % (
                    delta_min, delta_secs)
    return '%s.gz' % imgfile

def zsync_image(platform, imgfile):
    print "INFO: making zsync file for %s" % imgfile
    before = time.time()
    basepath = "%s/%s/pre-built" % (SNAPSHOTS_URL, platform)
    imgpath=os.sep.join(imgfile.split(os.sep)[-3:])
    zurl=basepath + os.sep + imgpath
    args = ('zsyncmake', '-b 2048', '-u', zurl, '-o', '%s.zsync' % imgfile,
        imgfile)
    cmd_runner.run(args).wait()
    delta_min, delta_secs = divmod(int(time.time() - before), 60)
    print "INFO: zsync generated in %d min(s) and %d sec(s)" % (
                    delta_min, delta_secs)

def md5sum(fname):
    md5 = hashlib.md5()
    with open(fname, 'rb') as f:
        while True:
            data = f.read(4096)
            if not data: break
            md5.update(data)
    return md5.hexdigest()

def create_image_info(infofile, gzfile, binary_file, hwpack_url, \
                      rootfs_url, platform):
    ''' prints out the md5sum and hwpack/rootfs info for the given image
    '''
    print "INFO: building image info for: %s" % gzfile
    md5 = md5sum(gzfile)

    binary_file = os.path.basename(binary_file)
    tfile = '%s/image_info_template.html' % os.path.dirname(__file__)

    tmpl = None
    with open(tfile) as f:
        buff = f.read()
        tmpl = string.Template(buff)

    title = 'Pre-Built Image Info'

    # build hwpack manifest and rootfs packages links
    hwpack_name = os.path.basename(hwpack_url)
    hwpack_manifest_url = ("%s" % hwpack_url).replace(".tar.gz", ".manifest.txt")
    hwpack_manifest = os.path.basename(hwpack_manifest_url)

    rootfs_name = os.path.basename(rootfs_url)
    rootfs_packages_url = ("%s" % rootfs_url).replace(".tar.gz", ".packages")
    rootfs_packages = os.path.basename(rootfs_packages_url)

    if platform == "openembedded":
        rootfs_packages_url = "n/a"
        rootfs_packages = "n/a"

    # try to get information from Jenkins, if available
    jenkins_buildn = os.environ.get('BUILD_NUMBER') or 'Not Found'
    jenkins_build_url = os.environ.get('BUILD_URL') or 'http://ci.linaro.org'

    buff = tmpl.substitute(
            title=title, md5=md5, image_name=os.path.basename(gzfile),
            hwpack_name=hwpack_name, hwpack_url=hwpack_url,
            rootfs_name=rootfs_name, rootfs_url=rootfs_url,
            hwpack_manifest_name=hwpack_manifest,
            hwpack_manifest_url=hwpack_manifest_url,
            rootfs_packages_name=rootfs_packages,
            rootfs_packages_url=rootfs_packages_url,
            jenkins_buildn=jenkins_buildn,
            jenkins_build_url=jenkins_build_url)
    with open(infofile, 'w') as f:
        f.write(buff)

def build_image(lmc, imgfile, hwpack, hwpack_file, binary,
                            binary_file, fastmodel=False):
    print "INFO: building image: %s" % imgfile
    hwpi = HWPACKS[hwpack]
    try:
        size = BINARIES[binary]
    except:
        size = '3G'

    args = [lmc,
                '--image-size', size,
                '--hwpack-force-yes',
                '--hwpack', hwpack_file,
                '--binary', binary_file,
            ]

    if fastmodel:
        args += ['--dev', 'fastmodel',
                '--output-directory', os.path.dirname(imgfile),
                '--image-file', os.path.basename(imgfile)]
    else:
        args += ['--dev', hwpi.lmcname,
                '--image-file', imgfile]

    if hwpi.pre_inst_script is not None:
        sdir = os.path.abspath(os.path.dirname(__file__))
        script = "%s/%s" % (sdir, hwpi.pre_inst_script)
        print "INFO: using preinstall script: %s" % script
        args.append('--preinstall-script')
        args.append(script)

    bootloader = os.environ.get("BOOTLOADER")
    if bootloader is None:
        print 'INFO: Use default bootloader (u_boot)'
    else:
        print 'INFO: Use specified bootlooader (%s)' % bootloader
        args.append('--bootloader')
        args.append(bootloader)

    print "INFO: running l-m-c:", " ".join([str(arg) for arg in args])

    hwpi.do_eula(imgfile)

    # some useful information so we know how much time it took per lmc run
    before = time.time()
    cmd_runner.run(args, as_root=True).wait()
    delta_min, delta_secs = divmod(int(time.time() - before), 60)
    print "INFO: linaro-media-create took %d min(s) and %d sec(s) to run" % (
                    delta_min, delta_secs)

class SnapshotCrawler:
    def __init__(self, platform, hwpacks, binaries, platform_hwpack):
        self.platform = platform
        self.platform_hwpack = platform_hwpack
        self.hwpacks = hwpacks
        self.binaries = binaries

    def get_binaries(self):
        '''return a hash table of binary->url'''
        binaryf = {}
        for binary in self.binaries:
            url = "%s/%s/images/%s" % (SNAPSHOTS_URL, self.platform, binary)
            (date, url) = crawler.latest_rfs(url)
            binaryf[binary] = url

        return binaryf

    def get_hwpacks(self):
        '''return a hash table of hwpack->urls (date, url)'''
        hwpackf = {}
        for hwpack in self.hwpacks:
            url = "%s/%s/hwpacks/%s" % (SNAPSHOTS_URL, self.platform_hwpack, hwpack)
            hwpackf[hwpack] = crawler.latest_hwpacks(url)

        return hwpackf

    def get_image_name(self, odir, hwpack, hwpack_url, binary):

        #convert a url like:
        # http://snapshots.linaro.org/SERIE/hwpacks/HWPACK_NAME/HWPACK_BUILD_NUMBER/HWPACK_FILE_NAME
        # to:
        # SERIE/pre-built/HWPACK_NAME/HWPACK_BUILD_NUMBER/HWPACK_FILE_NAME.img
        path = urlparse.urlparse(hwpack_url).path
        parts = path.split('/')
        name = parts[-1] #use for the file name below
        path = '/'.join(parts[-3:-1])

        path = '%s/%s' % (odir, path)
        if not os.path.exists(path):
            os.makedirs(path)

        date = re.search(r'_(\d+-\d+)_', name).group(1)
        return "%s/%s-%s_%s_%s.img" % (path, hwpack, self.platform, binary, date)

def main():
    hwpacks = HWPACKS.keys()
    binaries = BINARIES.keys()

    p = argparse.ArgumentParser(description=
        'Builds a matrix of builds from the latest build on '
        'snapshots.linaro.org or release images on releases.linaro.org')

    p.add_argument('-w', dest='hwpacks', action='append',
                   help='The hwpacks to generate for, default=%s' %
                   ', '.join(hwpacks))
    p.add_argument('-b', dest='binaries', action='append',
                   help='The binaries to generate for, default=%s' %
                   ', '.join(binaries))
    p.add_argument('-p', dest='platform', default='ubuntu',
                   help='The platform, default=ubuntu')
    p.add_argument('-pw', dest='platform_hwpack', default=False,
                   help='The platform for hwpack, default - same as platform')
    p.add_argument('-o', dest='out_dir', default='./out',
                   help='The out directory for downloaded and built files, default=./out')
    p.add_argument('-d', dest='date', default="",
                   help='The date (YYYYMMDD) as reference for hwpack')
    p.add_argument('-r', dest='release', action='store_true', default=False,
                   help='If this is for release images. NOTE: the "date" arg'
                   'will then become the cycle ie -d 12.03')
    p.add_argument('-f', dest='fastmodel', action='store_true',
                   help='Build for the fastmodel target (for vexpress hwpack)')


    args = p.parse_args()

    if args.hwpacks:
        hwpacks = args.hwpacks
    if args.binaries:
        binaries = args.binaries
    if not args.platform_hwpack:
        platform_hwpack = args.platform
    else:
        platform_hwpack = args.platform_hwpack

    crawler.cookie_setup()

    dm = fetch_image.DownloadManager(args.out_dir)

    if args.release:
        #TODO: add platform_hwpack
        site = ReleaseCrawler(args.date, args.platform, hwpacks, binaries)
    else:
        site = SnapshotCrawler(args.platform, hwpacks, binaries, platform_hwpack)

    hwpackf = site.get_hwpacks()
    binaryf = site.get_binaries()

    lmc = '%s/linaro-image-tools/linaro-media-create' % args.out_dir
    odir = '%s/pre-built' % args.out_dir

    for hwpack in hwpacks:
        url = None
        # check if the user wanted a hwpack from a specific date
        if args.date:
            try:
                dates = [elem[0] for elem in hwpackf[hwpack]]
                index = dates.index(args.date)
                url = hwpackf[hwpack][index][1]
            except:
                print "ERROR: Could not find hwpack '%s' with date '%s'" \
                      ", skipping pre-built image for it" % (hwpack, args.date)
        else:
            # just grab the latest hwpack
            try:
                url = hwpackf[hwpack][0][1]
            except:
                print "ERROR: Could not find a valid hwpack url for '%s'" \
                      ", skipping pre-built image for it" % (hwpack)

        if url is not None and url is not False:
            hwpf = download(dm, url)
            for binary in binaries:
                # create the image and metadata
                imgfile = site.get_image_name(odir, hwpack, url, binary)
                infofile = '%s.html' % os.path.splitext(imgfile)[0]
                rfsf = download(dm, binaryf[binary])
                build_image(lmc, imgfile, hwpack, hwpf, binary, rfsf,
                                        args.fastmodel)
                gzfile = compress_image(imgfile)
                zsync_image(args.platform, gzfile)
                create_image_info(infofile, gzfile, imgfile, url, \
                                  binaryf[binary], args.platform)

if __name__ == '__main__':
    main()