build-images.py

#!/usr/bin/env python
# Copyright (C) 2012 Linaro
#
# Author: Andy Doan <andy.doan@linaro.org>
#
# This file is part of Linaro Daily Prebuilt Images.
#
# Linaro Daily Prebuilt Images is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# Linaro Daily Prebuilt Images is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Linaro Image Tools; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
# USA.

from linaro_fetch_image import fetch_image
from linaro_image_tools import cmd_runner

import crawler

import argparse
import bz2
import datetime
import hashlib
import os
import re
import urlparse
import sys

class HwPack:
    def __init__(self, lmcname, eula=False, pre_inst_script=None):
        self.lmcname = lmcname
        if eula:
            self.eula = 'EULA.txt'
        else:
            self.eula = 'OPEN_EULA.txt'
        self.pre_inst_script = pre_inst_script

    def do_eula(self, imgfile):
        fname = '%s/%s' % (os.path.dirname(imgfile), self.eula)
        with file(fname, 'a'):
            pass #just need the file to exist

# a mapping of hwpack name to l-m-c name
HWPACKS = {
    'beagleboard': HwPack('beagle'),
    'overo': HwPack('overo'),
    'lt-panda': HwPack('panda'),
    'lt-mx5': HwPack('mx53loco'),
    'lt-mx6': HwPack('mx6qsabrelite'),
    'lt-origen': HwPack('origen', True),
    'leb-origen': HwPack('origen', True),
    'lt-snowball': HwPack('snowball_sd', True, 'ste-preinstall.sh'),
    'lt-snowball-x11-base': HwPack('snowball_sd', True, 'ste-preinstall.sh'),
    'vexpress-a9': HwPack('vexpress-a9'),
    'efikamx': HwPack('efikamx'),
    'igep': HwPack('igep'),
}
# a mapping of binary image to its image_file size
BINARIES = {
    # release images
    'nano': '512M',
    'alip': '2G',
    'ubuntu-desktop': '3G',
    'linarotv-xbmc': '3G',
}

def today():
    d = datetime.date.today()
    return "%d%02d%02d" % (d.year, d.month, d.day)

def hwpack_available(date, hwpack, platform):
    url = 'http://snapshots.linaro.org/%s/hwpacks/%s' % (platform, hwpack)
    hwpacks = crawler.latest_hwpacks(url)
    for hwpack in hwpacks:
        if hwpack[0] == date:
            return hwpack[1]

    return False

def compress_image(imgfile):
    print "compressing %s" % imgfile
    args = ('bzip2', imgfile)
    cmd_runner.run(args).wait()
    return '%s.bz2' % imgfile

def zsync_image(imgfile):
    print "making zsync file for %s" % imgfile
    args = ('zsyncmake', '-o', '%s.zsync'%imgfile, imgfile)
    cmd_runner.run(args).wait()

def md5sum(fname):
    md5 = hashlib.md5()
    with open(fname, 'rb') as f:
        while True:
            data = f.read(4096)
            if not data: break
            md5.update(data)
    return md5.hexdigest()

def image_info(bz2file, binary_file):
    ''' prints out the md5sum and rfs info for the given image
    '''
    print "building image info for: %s" % bz2file
    md5 = md5sum(bz2file)

    binary_file = os.path.basename(binary_file)

    with open('%s.info.txt' % bz2file, 'w') as f:
        f.write("%s md5sum(%s)\n" % (os.path.basename(bz2file), md5))
        f.write("root file system: %s\n" % binary_file)

def build_image(lmc, imgfile, hwpack, hwpack_file, binary, binary_file):
    print "building image: %s" % imgfile
    hwpi = HWPACKS[hwpack]
    size = BINARIES[binary]

    print 'running l-m-c: %s' % lmc
    args = [lmc,
                '--dev', hwpi.lmcname,
                '--image_file', imgfile,
                '--image_size', size,
                '--hwpack-force-yes',
                '--hwpack', hwpack_file,
                '--binary', binary_file,
            ]

    if hwpi.pre_inst_script is not None:
        sdir = os.path.abspath(os.path.dirname(__file__))
        script = "%s/%s" % (sdir, hwpi.pre_inst_script)
        print "using presintall script: %s" % script
        args.append('--preinstall-script')
        args.append(script)

    hwpi.do_eula(imgfile)
    cmd_runner.run(args, as_root=True).wait()
    bz2file = compress_image(imgfile)
    zsync_image(bz2file)
    image_info(bz2file, binary_file)

class SnapshotCrawler:
    def __init__(self, date, platform, hwpacks, binaries):
        self.date = date
        self.platform = platform
        self.hwpacks = hwpacks
        self.binaries = binaries

    def get_binaries(self):
        '''return a hash table of binary->url'''
        binaryf = {}
        for binary in self.binaries:
            url = 'http://snapshots.linaro.org/%s/images/%s' % (self.platform, binary)
            (date, url) = crawler.latest_rfs(url)
            binaryf[binary] = url

        return binaryf

    def get_hwpack_url(self, hwpack):
        hwpackname = "%s-%s" % (hwpack, self.platform)
        return hwpack_available(self.date, hwpack, self.platform)

    def get_image_name(self, odir, hwpack, hwpack_url, binary):

        #covert a url like:
        # http://snapshots.linaro.org/precise/hwpacks/lt-panda/87/hwpack_linaro-lt-panda_20120417-87_armhf_supported.tar.gz
        # to:
        # precise/pre-built/lt-panda/87/<file>.img
        path = urlparse.urlparse(hwpack_url).path
        parts = path.split('/')
        name = parts[-1] #use for the file name below
        path = '/'.join(parts[-3:-1])

        path = '%s/%s' % (odir, path)
        if not os.path.exists(path):
            os.makedirs(path)

        date = re.search(r'_(\d+-\d+)_', name).group(1)
        return "%s/%s-%s_%s_%s.img" %(path,hwpack,self.platform,binary,date)

# XXX: this is now broken for precise. need to wait for release images to see
# what changes are needed
class ReleaseCrawler:
    def __init__(self, cycle, platform, hwpacks, binaries):
        self.cycle = cycle
        self.platform = platform
        self.hwpacks = hwpacks
        self.binaries = binaries

        #inialize our list of hwpacks
        url = 'http://releases.linaro.org/%s/ubuntu/%s-hwpacks' % (cycle,platform)
        self.hwpack_urls = crawler.list_hwpacks(url)

    def get_hwpack_url(self, hwpack):
        pat = re.compile('hwpack_linaro-%s_' % hwpack)
        for url in self.hwpack_urls:
            if pat.search(url):
                return url
        return None

    def get_binaries(self):
        '''return a hash table of binary->url'''
        binaryf = {}
        for binary in self.binaries:
            url = 'http://releases.linaro.org/%s/ubuntu/%s-images/%s' % (
                    self.cycle, self.platform, binary)
            url = crawler.list_rfs(url)
            if url is not None:
                binaryf[binary] = url
        return binaryf

    def get_image_name(self, odir, hwpack, hwpack_url, binary):
        path = '%s/%s/%s/%s' % (odir, self.cycle, self.platform, binary)
        if not os.path.exists(path):
            os.makedirs(path)
        return "%s/%s-%s.img" %(path,hwpack,binary)

def main():
    day = today()
    hwpacks = HWPACKS.keys()
    binaries = BINARIES.keys()

    p = argparse.ArgumentParser(description=
        'Builds a matrix of builds from the latest build on '
        'snapshots.linaro.org or release images on releases.linaro.org')

    p.add_argument('-o', dest='out_dir', default='./out',
                   help='The out directory for downloaded and built files, default=./')
    p.add_argument('-d', dest='date', default=day,
                   help='The date, default=%s' % day)
    p.add_argument('-w', dest='hwpacks', action='append',
                   help='The hwpacks to generate for, default=%s' %
                        ', '.join(hwpacks))
    p.add_argument('-b', dest='binaries', action='append',
                   help='The binaries to generate for, default=%s' %
                         ', '.join(binaries))
    p.add_argument('-p', dest='platform', default='precise',
                   help='The platform, default=precise')
    p.add_argument('-r', dest='release', action='store_true', default=False,
                   help='If this is for release images. NOTE: the "date" arg'
                   'will then become the cycle ie -d 12.03')

    args = p.parse_args()

    if args.hwpacks:
        hwpacks = args.hwpacks
    if args.binaries:
        binaries = args.binaries

    dm = fetch_image.DownloadManager(args.out_dir)

    if args.release:
        site = ReleaseCrawler(args.date, args.platform, hwpacks, binaries)
    else:
        site = SnapshotCrawler(args.date, args.platform, hwpacks, binaries)

    binaryf = site.get_binaries()

    lmc = '%s/linaro-image-tools/linaro-media-create' % args.out_dir
    odir = '%s/pre-built' % args.out_dir

    for hwpack in hwpacks:
        url = site.get_hwpack_url(hwpack)

        if url is not None and url is not False:
            hwpf = dm.download(url, None)
            for binary in binaries:
                imgfile = site.get_image_name(odir,hwpack,url,binary)
                rfsf = dm.download(binaryf[binary], None)
                build_image(lmc, imgfile, hwpack, hwpf, binary, rfsf)
        else:
            print '%s hwpack not available for %s' % (args.date,hwpack)

if __name__ == '__main__':
    main()