build-images.py

#!/usr/bin/env python
# Copyright (C) 2012 Linaro
#
# Author: Andy Doan <andy.doan@linaro.org>
#
# This file is part of Linaro Daily Prebuilt Images.
#
# Linaro Daily Prebuilt Images is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# Linaro Daily Prebuilt Images is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Linaro Image Tools; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
# USA.

from linaro_fetch_image import fetch_image
from linaro_image_tools import cmd_runner

import crawler

import argparse
import bz2
import datetime
import os
import re
import urlparse

# a mapping of hwpack name to l-m-c name
HWPACKS = {
    'omap3': 'beagle',
    'overo': 'overo',
    'lt-panda': 'panda',
    'lt-mx5':'mx53loco',
    'lt-mx6': 'mx6qsabrelite',
    'lt-origen': 'origen',
    'lt-snowball': 'snowball_sd',
    'lt-vexpress-a9': 'vexpress-a9',
    'efikamx': 'efikamx',
}
# a mapping of binary image to its image_file size
BINARIES = {
    'linaro-o-nano': '512M',
    'linaro-o-alip': '2G',
    'linaro-o-ubuntu-desktop': '3G',
}

PREINST_SCRIPTS = {
    'lt-snowball': 'ste-preinstall.sh',
}

def today():
    d = datetime.date.today()
    return "%d%02d%02d" % (d.year, d.month, d.day)

def hwpack_available(date, hwpack, platform):
    url = 'http://snapshots.linaro.org/%s/%s-%s' % (platform, hwpack, platform)
    hwpacks = crawler.latest_hwpacks(url)
    for hwpack in hwpacks:
        if hwpack[0] == date:
            return hwpack[1]

    return False

def compress_image(imgfile):
    print "compressing %s" % imgfile
    args = ('bzip2', imgfile)
    cmd_runner.run(args).wait()
    return '%s.bz2' % imgfile

def zsync_image(imgfile):
    print "making zsync file for %s" % imgfile
    args = ('zsyncmake', '-o', '%s.zsync'%imgfile, imgfile)
    cmd_runner.run(args).wait()

def build_image(lmc, imgfile, hwpack, hwpack_file, binary, binary_file):
    print "building image: %s" % imgfile
    dev = HWPACKS[hwpack]
    size = BINARIES[binary]

    print 'running l-m-c: %s' % lmc
    args = [lmc,
                '--dev', dev,
                '--image_file', imgfile,
                '--image_size', size,
                '--hwpack-force-yes',
                '--hwpack', hwpack_file,
                '--binary', binary_file,
            ]

    if PREINST_SCRIPTS.has_key(hwpack):
        sdir = os.path.abspath(os.path.dirname(__file__))
        script = "%s/%s" % (sdir, PREINST_SCRIPTS[hwpack])
        print "using presintall script: %s" % script
        args.append('--preinstall-script')
        args.append(script)

    cmd_runner.run(args, as_root=True).wait()
    bz2file = compress_image(imgfile)
    zsync_image(bz2file)

def get_image_dir(outdir, url):
    '''covert a url like:
      http://snapshots.linaro.org/oneiric/lt-origen-oneiric/20120321/1/images/hwpack/hwpack_linaro-lt-origen_20120321-1_armel_supported.tar.gz
    to:
     oneiric/lt-origen-oneiric/20120321/1/images
    '''
    path = urlparse.urlparse(url).path
    path = os.path.dirname(path)
    path = os.path.split(path)[0]

    path = '%s/%s' % (outdir, path)
    if not os.path.exists(path):
        os.makedirs(path)
    return path

def get_snapshot_binaries(binaries, platform):
    binaryf = {}
    for binary in binaries:
        url = 'http://snapshots.linaro.org/%s/%s' % (platform, binary)
        (date, url) = crawler.latest_rfs(url)
        binaryf[binary] = url

    return binaryf

def get_snapshot_hwpack(date, hwpack, platform):
    ''' returns the url to the hwpack on snapshots.l.o if it exists'''
    hwpackname = "%s-%s" % (hwpack, platform)
    return hwpack_available(date, hwpack, platform)

def main():
    day = today()
    hwpacks = HWPACKS.keys()
    binaries = BINARIES.keys()

    p = argparse.ArgumentParser(description=
        'Builds a matrix of builds from the latest build on '
        'snapshots.linaro.org or release images on releases.linaro.org')

    p.add_argument('-o', dest='out_dir', default='./out',
                   help='The out directory for downloaded and built files, default=./')
    p.add_argument('-d', dest='date', default=day,
                   help='The date, default=%s' % day)
    p.add_argument('-w', dest='hwpacks', action='append',
                   help='The hwpacks to generate for, default=%s' %
                        ', '.join(hwpacks))
    p.add_argument('-b', dest='binaries', action='append',
                   help='The binaries to generate for, default=%s' %
                         ', '.join(binaries))
    p.add_argument('-p', dest='platform', default='oneiric',
                   help='The platform, default=oneiric')

    args = p.parse_args()

    if args.hwpacks:
        hwpacks = args.hwpacks
    if args.binaries:
        binaries = args.binaries

    dm = fetch_image.DownloadManager(args.out_dir)

    binaryf = get_snapshot_binaries(binaries, args.platform)

    lmc = '%s/linaro-image-tools/linaro-media-create' % args.out_dir
    odir_root = '%s/pre-built' % args.out_dir

    for hwpack in hwpacks:
        url = get_snapshot_hwpack(args.date, hwpack, args.platform)

        if url is not None and url is not False:
            odir = get_image_dir(odir_root, url)

            hwpf = dm.download(url, None)
            for binary in binaries:
                imgfile = "%s/%s-%s_%s.img" %(odir,hwpack,args.platform,binary)
                rfsf = dm.download(binaryf[binary], None)
                build_image(lmc, imgfile, hwpack, hwpf, binary, rfsf)
        else:
            print '%s hwpack not available for %s' % (args.date,hwpack)

if __name__ == '__main__':
    main()