#!/usr/bin/python # Copyright (C) 2012 Linaro # # Author: Andy Doan # # This file is part of Linaro Daily Prebuilt Images. # # Linaro Daily Prebuilt Images is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # Linaro Daily Prebuilt Images is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Linaro Image Tools; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. from linaro_fetch_image import fetch_image from linaro_image_tools import cmd_runner import crawler import argparse import datetime import time import hashlib import os import re import urlparse import string class HwPack: def __init__(self, lmcname, eula=False, pre_inst_script=None): self.lmcname = lmcname if eula: self.eula = 'EULA.txt' else: self.eula = 'OPEN-EULA.txt' self.pre_inst_script = pre_inst_script def do_eula(self, imgfile): fname = '%s/%s' % (os.path.dirname(imgfile), self.eula) with file(fname, 'a'): pass #just need the file to exist # a mapping of hwpack name to l-m-c name HWPACKS = { 'arndale': HwPack('arndale'), 'arndale-be': HwPack('arndale'), 'beagleboard': HwPack('beagle'), 'beaglebone': HwPack('beaglebone'), 'efikamx': HwPack('efikamx'), 'highbank': HwPack('highbank'), 'igep': HwPack('igep'), 'leb-origen': HwPack('origen'), 'linux-linaro-stable-lsk-vexpress-debug': HwPack('vexpress'), 'lsk-vexpress': HwPack('vexpress'), 'lsk-vexpress64': HwPack('vexpress'), 'lt-mx5': HwPack('mx53loco'), 'lt-mx6': HwPack('mx6qsabrelite'), 'lt-origen': HwPack('origen'), 'lt-panda': HwPack('panda'), 'lt-panda-x11-base': HwPack('panda'), 'lt-snowball': HwPack('snowball_sd', True, 'ste-preinstall.sh'), 'lt-snowball-x11-base': HwPack('snowball_sd', True, 'ste-preinstall.sh'), 'lt-vexpress': HwPack('vexpress'), 'lt-vexpress64': HwPack('vexpress'), 'origen': HwPack('origen'), 'overo': HwPack('overo'), 'panda': HwPack('panda'), 'panda-be': HwPack('panda'), 'snowball': HwPack('snowball_sd', True), 'ti-panda-x11-base': HwPack('panda'), 'vexpress': HwPack('vexpress'), 'vexpress-debug': HwPack('vexpress'), 'vexpress64': HwPack('vexpress'), } # a mapping of binary image to its image_file size BINARIES = { 'alip': '2G', 'developer': '1G', 'lamp-armv8': '2G', 'leg-java-armv8': '4G', 'linarotv-xbmc': '3G', 'minimal-armv8': '512M', 'nano': '1G', 'nano-lava': '1G', 'sdk-armv8': '2G', 'server': '1G', 'ubuntu-desktop': '3G', } SNAPSHOTS_URL = "http://snapshots.linaro.org" def today(): d = datetime.date.today() return "%d%02d%02d" % (d.year, d.month, d.day) def download(dm, url): print "INFO: Fetching", url before = time.time() dfile = dm.download(url, None, verbose=False) delta_min, delta_secs = divmod(int(time.time() - before), 60) print "INFO: Downloaded file '%s' in %d min(s) and %d sec(s)" % ( dfile, delta_min, delta_secs) return dfile def compress_image(imgfile): print "INFO: compressing %s" % imgfile before = time.time() args = ('gzip', imgfile) cmd_runner.run(args).wait() delta_min, delta_secs = divmod(int(time.time() - before), 60) print "INFO: file compressed in %d min(s) and %d sec(s)" % ( delta_min, delta_secs) return '%s.gz' % imgfile def zsync_image(platform, imgfile): print "INFO: making zsync file for %s" % imgfile before = time.time() basepath = "%s/%s/pre-built" % (SNAPSHOTS_URL, platform) imgpath=os.sep.join(imgfile.split(os.sep)[-3:]) zurl=basepath + os.sep + imgpath args = ('zsyncmake', '-b 2048', '-u', zurl, '-o', '%s.zsync' % imgfile, imgfile) cmd_runner.run(args).wait() delta_min, delta_secs = divmod(int(time.time() - before), 60) print "INFO: zsync generated in %d min(s) and %d sec(s)" % ( delta_min, delta_secs) def md5sum(fname): md5 = hashlib.md5() with open(fname, 'rb') as f: while True: data = f.read(4096) if not data: break md5.update(data) return md5.hexdigest() def create_image_info(infofile, gzfile, binary_file, hwpack_url, \ rootfs_url, platform): ''' prints out the md5sum and hwpack/rootfs info for the given image ''' print "INFO: building image info for: %s" % gzfile md5 = md5sum(gzfile) binary_file = os.path.basename(binary_file) tfile = '%s/image_info_template.html' % os.path.dirname(__file__) tmpl = None with open(tfile) as f: buff = f.read() tmpl = string.Template(buff) title = 'Pre-Built Image Info' # build hwpack manifest and rootfs packages links hwpack_name = os.path.basename(hwpack_url) hwpack_manifest_url = ("%s" % hwpack_url).replace(".tar.gz", ".manifest.txt") hwpack_manifest = os.path.basename(hwpack_manifest_url) rootfs_name = os.path.basename(rootfs_url) rootfs_packages_url = ("%s" % rootfs_url).replace(".tar.gz", ".packages") rootfs_packages = os.path.basename(rootfs_packages_url) if platform == "openembedded": rootfs_packages_url = "n/a" rootfs_packages = "n/a" # try to get information from Jenkins, if available jenkins_buildn = os.environ.get('BUILD_NUMBER') or 'Not Found' jenkins_build_url = os.environ.get('BUILD_URL') or 'http://ci.linaro.org' buff = tmpl.substitute( title=title, md5=md5, image_name=os.path.basename(gzfile), hwpack_name=hwpack_name, hwpack_url=hwpack_url, rootfs_name=rootfs_name, rootfs_url=rootfs_url, hwpack_manifest_name=hwpack_manifest, hwpack_manifest_url=hwpack_manifest_url, rootfs_packages_name=rootfs_packages, rootfs_packages_url=rootfs_packages_url, jenkins_buildn=jenkins_buildn, jenkins_build_url=jenkins_build_url) with open(infofile, 'w') as f: f.write(buff) def build_image(lmc, imgfile, hwpack, hwpack_file, binary, binary_file, fastmodel=False): print "INFO: building image: %s" % imgfile hwpi = HWPACKS[hwpack] try: size = BINARIES[binary] except: size = '3G' args = [lmc, '--image-size', size, '--hwpack-force-yes', '--hwpack', hwpack_file, '--binary', binary_file, ] if fastmodel: args += ['--dev', 'fastmodel', '--output-directory', os.path.dirname(imgfile), '--image-file', os.path.basename(imgfile)] else: args += ['--dev', hwpi.lmcname, '--image-file', imgfile] if hwpi.pre_inst_script is not None: sdir = os.path.abspath(os.path.dirname(__file__)) script = "%s/%s" % (sdir, hwpi.pre_inst_script) print "INFO: using preinstall script: %s" % script args.append('--preinstall-script') args.append(script) bootloader = os.environ.get("BOOTLOADER") if bootloader is None: print 'INFO: Use default bootloader (u_boot)' else: print 'INFO: Use specified bootlooader (%s)' % bootloader args.append('--bootloader') args.append(bootloader) print "INFO: running l-m-c:", " ".join([str(arg) for arg in args]) hwpi.do_eula(imgfile) # some useful information so we know how much time it took per lmc run before = time.time() cmd_runner.run(args, as_root=True).wait() delta_min, delta_secs = divmod(int(time.time() - before), 60) print "INFO: linaro-media-create took %d min(s) and %d sec(s) to run" % ( delta_min, delta_secs) class SnapshotCrawler: def __init__(self, platform, hwpacks, binaries, platform_hwpack): self.platform = platform self.platform_hwpack = platform_hwpack self.hwpacks = hwpacks self.binaries = binaries def get_binaries(self): '''return a hash table of binary->url''' binaryf = {} for binary in self.binaries: url = "%s/%s/images/%s" % (SNAPSHOTS_URL, self.platform, binary) (date, url) = crawler.latest_rfs(url) binaryf[binary] = url return binaryf def get_hwpacks(self): '''return a hash table of hwpack->urls (date, url)''' hwpackf = {} for hwpack in self.hwpacks: if self.platform_hwpack is "kernel_hwpacks": url = "%s/%s/%s" % (SNAPSHOTS_URL, self.platform_hwpack, hwpack) else: url = "%s/%s/hwpacks/%s" % (SNAPSHOTS_URL, self.platform_hwpack, hwpack) hwpackf[hwpack] = crawler.latest_hwpacks(url) return hwpackf def get_image_name(self, odir, hwpack, hwpack_url, binary): #convert a url like: # http://snapshots.linaro.org/SERIE/hwpacks/HWPACK_NAME/HWPACK_BUILD_NUMBER/HWPACK_FILE_NAME # to: # SERIE/pre-built/HWPACK_NAME/HWPACK_BUILD_NUMBER/HWPACK_FILE_NAME.img path = urlparse.urlparse(hwpack_url).path parts = path.split('/') name = parts[-1] #use for the file name below path = '/'.join(parts[-3:-1]) path = '%s/%s' % (odir, path) if not os.path.exists(path): os.makedirs(path) date = re.search(r'_(\d+-\d+)_', name).group(1) return "%s/%s-%s_%s_%s.img" % (path, hwpack, self.platform, binary, date) def main(): hwpacks = HWPACKS.keys() binaries = BINARIES.keys() p = argparse.ArgumentParser(description= 'Builds a matrix of builds from the latest build on ' 'snapshots.linaro.org or release images on releases.linaro.org') p.add_argument('-w', dest='hwpacks', action='append', help='The hwpacks to generate for, default=%s' % ', '.join(hwpacks)) p.add_argument('-b', dest='binaries', action='append', help='The binaries to generate for, default=%s' % ', '.join(binaries)) p.add_argument('-p', dest='platform', default='ubuntu', help='The platform, default=ubuntu') p.add_argument('-pw', dest='platform_hwpack', default=False, help='The platform for hwpack, default - same as platform') p.add_argument('-o', dest='out_dir', default='./out', help='The out directory for downloaded and built files, default=./out') p.add_argument('-d', dest='date', default="", help='The date (YYYYMMDD) as reference for hwpack') p.add_argument('-r', dest='release', action='store_true', default=False, help='If this is for release images. NOTE: the "date" arg' 'will then become the cycle ie -d 12.03') p.add_argument('-f', dest='fastmodel', action='store_true', help='Build for the fastmodel target (for vexpress hwpack)') args = p.parse_args() if args.hwpacks: hwpacks = args.hwpacks if args.binaries: binaries = args.binaries if not args.platform_hwpack: platform_hwpack = args.platform else: platform_hwpack = args.platform_hwpack crawler.cookie_setup() dm = fetch_image.DownloadManager(args.out_dir) if args.release: #TODO: add platform_hwpack site = ReleaseCrawler(args.date, args.platform, hwpacks, binaries) else: site = SnapshotCrawler(args.platform, hwpacks, binaries, platform_hwpack) hwpackf = site.get_hwpacks() binaryf = site.get_binaries() lmc = '%s/linaro-image-tools/linaro-media-create' % args.out_dir odir = '%s/pre-built' % args.out_dir for hwpack in hwpacks: url = None # check if the user wanted a hwpack from a specific date if args.date: try: dates = [elem[0] for elem in hwpackf[hwpack]] index = dates.index(args.date) url = hwpackf[hwpack][index][1] except: print "ERROR: Could not find hwpack '%s' with date '%s'" \ ", skipping pre-built image for it" % (hwpack, args.date) else: # just grab the latest hwpack try: url = hwpackf[hwpack][0][1] except: print "ERROR: Could not find a valid hwpack url for '%s'" \ ", skipping pre-built image for it" % (hwpack) if url is not None and url is not False: hwpf = download(dm, url) for binary in binaries: # create the image and metadata imgfile = site.get_image_name(odir, hwpack, url, binary) infofile = '%s.html' % os.path.splitext(imgfile)[0] rfsf = download(dm, binaryf[binary]) build_image(lmc, imgfile, hwpack, hwpf, binary, rfsf, args.fastmodel) gzfile = compress_image(imgfile) zsync_image(args.platform, gzfile) create_image_info(infofile, gzfile, imgfile, url, \ binaryf[binary], args.platform) if __name__ == '__main__': main()