build-images.py

#!/usr/bin/env python
# Copyright (C) 2012 Linaro
#
# Author: Andy Doan <andy.doan@linaro.org>
#
# This file is part of Linaro Daily Prebuilt Images.
#
# Linaro Daily Prebuilt Images is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# Linaro Daily Prebuilt Images is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Linaro Image Tools; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
# USA.

from linaro_fetch_image import fetch_image
from linaro_image_tools import cmd_runner

import crawler

import argparse
import bz2
import datetime
import os
import re
import urlparse
import sys

# a mapping of hwpack name to l-m-c name
HWPACKS = {
    'omap3': 'beagle',
    'overo': 'overo',
    'lt-panda': 'panda',
    'lt-mx5':'mx53loco',
    'lt-mx6': 'mx6qsabrelite',
    'lt-origen': 'origen',
    'lt-snowball': 'snowball_sd',
    'lt-vexpress-a9': 'vexpress-a9',
    'efikamx': 'efikamx',
}
# a mapping of binary image to its image_file size
BINARIES = {
    # daily oneiric snapshots
    'linaro-o-nano': '512M',
    'linaro-o-alip': '2G',
    'linaro-o-ubuntu-desktop': '3G',
    # release images
    'nano': '512M',
    'alip': '2G',
    'ubuntu-desktop': '3G',
    'linarotv-xbmc': '3G',
}

PREINST_SCRIPTS = {
    'lt-snowball': 'ste-preinstall.sh',
}

def today():
    d = datetime.date.today()
    return "%d%02d%02d" % (d.year, d.month, d.day)

def hwpack_available(date, hwpack, platform):
    url = 'http://snapshots.linaro.org/%s/%s-%s' % (platform, hwpack, platform)
    hwpacks = crawler.latest_hwpacks(url)
    for hwpack in hwpacks:
        if hwpack[0] == date:
            return hwpack[1]

    return False

def compress_image(imgfile):
    print "compressing %s" % imgfile
    args = ('bzip2', imgfile)
    cmd_runner.run(args).wait()
    return '%s.bz2' % imgfile

def zsync_image(imgfile):
    print "making zsync file for %s" % imgfile
    args = ('zsyncmake', '-o', '%s.zsync'%imgfile, imgfile)
    cmd_runner.run(args).wait()

def build_image(lmc, imgfile, hwpack, hwpack_file, binary, binary_file):
    print "building image: %s" % imgfile
    dev = HWPACKS[hwpack]
    size = BINARIES[binary]

    print 'running l-m-c: %s' % lmc
    args = [lmc,
                '--dev', dev,
                '--image_file', imgfile,
                '--image_size', size,
                '--hwpack-force-yes',
                '--hwpack', hwpack_file,
                '--binary', binary_file,
            ]

    if PREINST_SCRIPTS.has_key(hwpack):
        sdir = os.path.abspath(os.path.dirname(__file__))
        script = "%s/%s" % (sdir, PREINST_SCRIPTS[hwpack])
        print "using presintall script: %s" % script
        args.append('--preinstall-script')
        args.append(script)

    cmd_runner.run(args, as_root=True).wait()
    bz2file = compress_image(imgfile)
    zsync_image(bz2file)

class SnapshotCrawler:
    def __init__(self, date, platform, hwpacks, binaries):
        self.date = date
        self.platform = platform
        self.hwpacks = hwpacks
        self.binaries = binaries

    def get_binaries(self):
        '''return a hash table of binary->url'''
        binaryf = {}
        for binary in self.binaries:
            url = 'http://snapshots.linaro.org/%s/%s' % (self.platform, binary)
            (date, url) = crawler.latest_rfs(url)
            binaryf[binary] = url

        return binaryf

    def get_hwpack_url(self, hwpack):
        hwpackname = "%s-%s" % (hwpack, self.platform)
        return hwpack_available(self.date, hwpack, self.platform)

    def get_image_name(self, odir, hwpack, hwpack_url, binary):

        #covert a url like:
        # http://snapshots.linaro.org/oneiric/lt-origen-oneiric/20120321/1/images/hwpack/hwpack_linaro-lt-origen_20120321-1_armel_supported.tar.gz
        # to:
        #  oneiric/lt-origen-oneiric/20120321/1/images/<file>.img
        path = urlparse.urlparse(hwpack_url).path
        path = os.path.dirname(path)
        path = os.path.split(path)[0]

        path = '%s/%s' % (odir, path)
        if not os.path.exists(path):
            os.makedirs(path)
        return "%s/%s-%s_%s.img" %(path,hwpack,self.platform,binary)

class ReleaseCrawler:
    def __init__(self, cycle, platform, hwpacks, binaries):
        self.cycle = cycle
        self.platform = platform
        self.hwpacks = hwpacks
        self.binaries = binaries

        #inialize our list of hwpacks
        url = 'http://releases.linaro.org/%s/ubuntu/%s-hwpacks' % (cycle,platform)
        self.hwpack_urls = crawler.list_hwpacks(url)

    def get_hwpack_url(self, hwpack):
        pat = re.compile('hwpack_linaro-%s_' % hwpack)
        for url in self.hwpack_urls:
            if pat.search(url):
                return url
        return None

    def get_binaries(self):
        '''return a hash table of binary->url'''
        binaryf = {}
        for binary in self.binaries:
            url = 'http://releases.linaro.org/%s/ubuntu/%s-images/%s' % (
                    self.cycle, self.platform, binary)
            url = crawler.list_rfs(url)
            if url is not None:
                binaryf[binary] = url
        return binaryf

    def get_image_name(self, odir, hwpack, hwpack_url, binary):
        path = '%s/%s/%s/%s' % (odir, self.cycle, self.platform, binary)
        if not os.path.exists(path):
            os.makedirs(path)
        return "%s/%s-%s.img" %(path,hwpack,binary)

def main():
    day = today()
    hwpacks = HWPACKS.keys()
    binaries = BINARIES.keys()

    p = argparse.ArgumentParser(description=
        'Builds a matrix of builds from the latest build on '
        'snapshots.linaro.org or release images on releases.linaro.org')

    p.add_argument('-o', dest='out_dir', default='./out',
                   help='The out directory for downloaded and built files, default=./')
    p.add_argument('-d', dest='date', default=day,
                   help='The date, default=%s' % day)
    p.add_argument('-w', dest='hwpacks', action='append',
                   help='The hwpacks to generate for, default=%s' %
                        ', '.join(hwpacks))
    p.add_argument('-b', dest='binaries', action='append',
                   help='The binaries to generate for, default=%s' %
                         ', '.join(binaries))
    p.add_argument('-p', dest='platform', default='oneiric',
                   help='The platform, default=oneiric')
    p.add_argument('-r', dest='release', action='store_true', default=False,
                   help='If this is for release images. NOTE: the "date" arg'
                   'will then become the cycle ie -d 12.03')

    args = p.parse_args()

    if args.hwpacks:
        hwpacks = args.hwpacks
    if args.binaries:
        binaries = args.binaries

    dm = fetch_image.DownloadManager(args.out_dir)

    if args.release:
        site = ReleaseCrawler(args.date, args.platform, hwpacks, binaries)
    else:
        site = SnapshotCrawler(args.date, args.platform, hwpacks, binaries)

    binaryf = site.get_binaries()

    lmc = '%s/linaro-image-tools/linaro-media-create' % args.out_dir
    odir = '%s/pre-built' % args.out_dir

    for hwpack in hwpacks:
        url = site.get_hwpack_url(hwpack)

        if url is not None and url is not False:
            hwpf = dm.download(url, None)
            for binary in binaries:
                imgfile = site.get_image_name(odir,hwpack,url,binary)
                rfsf = dm.download(binaryf[binary], None)
                build_image(lmc, imgfile, hwpack, hwpf, binary, rfsf)
        else:
            print '%s hwpack not available for %s' % (args.date,hwpack)

if __name__ == '__main__':
    main()