| #!/usr/bin/python |
| |
| # Copyright (C) 2012 Linaro |
| # |
| # Author: Andy Doan <andy.doan@linaro.org> |
| # |
| # This file is part of Linaro Daily Prebuilt Images. |
| # |
| # Linaro Daily Prebuilt Images is free software; you can redistribute it and/or |
| # modify it under the terms of the GNU General Public License |
| # as published by the Free Software Foundation; either version 2 |
| # of the License, or (at your option) any later version. |
| # |
| # Linaro Daily Prebuilt Images is distributed in the hope that it will be useful, |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| # GNU General Public License for more details. |
| # |
| # You should have received a copy of the GNU General Public License |
| # along with Linaro Image Tools; if not, write to the Free Software |
| # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, |
| # USA. |
| |
| from linaro_fetch_image import fetch_image |
| from linaro_image_tools import cmd_runner |
| |
| import crawler |
| |
| import argparse |
| import datetime |
| import time |
| import hashlib |
| import os |
| import re |
| import urlparse |
| import string |
| |
| class HwPack: |
| def __init__(self, lmcname, eula=False, pre_inst_script=None): |
| self.lmcname = lmcname |
| if eula: |
| self.eula = 'EULA.txt' |
| else: |
| self.eula = 'OPEN-EULA.txt' |
| self.pre_inst_script = pre_inst_script |
| |
| def do_eula(self, imgfile): |
| fname = '%s/%s' % (os.path.dirname(imgfile), self.eula) |
| with file(fname, 'a'): |
| pass #just need the file to exist |
| |
| # a mapping of hwpack name to l-m-c name |
| HWPACKS = { |
| 'beagleboard': HwPack('beagle'), |
| 'efikamx': HwPack('efikamx'), |
| 'igep': HwPack('igep'), |
| 'leb-origen': HwPack('origen'), |
| 'lt-mx5': HwPack('mx53loco'), |
| 'lt-mx6': HwPack('mx6qsabrelite'), |
| 'lt-origen': HwPack('origen'), |
| 'lt-panda': HwPack('panda'), |
| 'lt-panda-x11-base': HwPack('panda'), |
| 'lt-snowball': HwPack('snowball_sd', True, 'ste-preinstall.sh'), |
| 'lt-snowball-x11-base': HwPack('snowball_sd', True, 'ste-preinstall.sh'), |
| 'lt-vexpress': HwPack('vexpress'), |
| 'origen': HwPack('origen'), |
| 'overo': HwPack('overo'), |
| 'panda': HwPack('panda'), |
| 'snowball': HwPack('snowball_sd', True), |
| 'ti-panda-x11-base': HwPack('panda'), |
| 'vexpress': HwPack('vexpress'), |
| 'vexpress64': HwPack('vexpress'), |
| } |
| # a mapping of binary image to its image_file size |
| BINARIES = { |
| 'alip': '2G', |
| 'developer': '1G', |
| 'lamp-armv8': '1G', |
| 'linarotv-xbmc': '3G', |
| 'minimal-armv8': '512M', |
| 'nano': '1G', |
| 'nano-lava': '1G', |
| 'sdk-armv8': '2G', |
| 'server': '1G', |
| 'ubuntu-desktop': '3G', |
| } |
| |
| SNAPSHOTS_URL = "http://snapshots.linaro.org" |
| |
| def today(): |
| d = datetime.date.today() |
| return "%d%02d%02d" % (d.year, d.month, d.day) |
| |
| def download(dm, url): |
| print "INFO: Fetching", url |
| before = time.time() |
| dfile = dm.download(url, None, verbose=False) |
| delta_min, delta_secs = divmod(int(time.time() - before), 60) |
| print "INFO: Downloaded file '%s' in %d min(s) and %d sec(s)" % ( |
| dfile, delta_min, delta_secs) |
| return dfile |
| |
| def compress_image(imgfile): |
| print "INFO: compressing %s" % imgfile |
| before = time.time() |
| args = ('gzip', imgfile) |
| cmd_runner.run(args).wait() |
| delta_min, delta_secs = divmod(int(time.time() - before), 60) |
| print "INFO: file compressed in %d min(s) and %d sec(s)" % ( |
| delta_min, delta_secs) |
| return '%s.gz' % imgfile |
| |
| def zsync_image(platform, imgfile): |
| print "INFO: making zsync file for %s" % imgfile |
| before = time.time() |
| basepath = "%s/%s/pre-built" % (SNAPSHOTS_URL, platform) |
| imgpath=os.sep.join(imgfile.split(os.sep)[-3:]) |
| zurl=basepath + os.sep + imgpath |
| args = ('zsyncmake', '-b 2048', '-u', zurl, '-o', '%s.zsync' % imgfile, |
| imgfile) |
| cmd_runner.run(args).wait() |
| delta_min, delta_secs = divmod(int(time.time() - before), 60) |
| print "INFO: zsync generated in %d min(s) and %d sec(s)" % ( |
| delta_min, delta_secs) |
| |
| def md5sum(fname): |
| md5 = hashlib.md5() |
| with open(fname, 'rb') as f: |
| while True: |
| data = f.read(4096) |
| if not data: break |
| md5.update(data) |
| return md5.hexdigest() |
| |
| def create_image_info(infofile, gzfile, binary_file, hwpack_url, \ |
| rootfs_url, platform): |
| ''' prints out the md5sum and hwpack/rootfs info for the given image |
| ''' |
| print "INFO: building image info for: %s" % gzfile |
| md5 = md5sum(gzfile) |
| |
| binary_file = os.path.basename(binary_file) |
| tfile = '%s/image_info_template.html' % os.path.dirname(__file__) |
| |
| tmpl = None |
| with open(tfile) as f: |
| buff = f.read() |
| tmpl = string.Template(buff) |
| |
| title = 'Pre-Built Image Info' |
| |
| # build hwpack manifest and rootfs packages links |
| hwpack_name = os.path.basename(hwpack_url) |
| hwpack_manifest_url = ("%s" % hwpack_url).replace(".tar.gz", ".manifest.txt") |
| hwpack_manifest = os.path.basename(hwpack_manifest_url) |
| |
| rootfs_name = os.path.basename(rootfs_url) |
| rootfs_packages_url = ("%s" % rootfs_url).replace(".tar.gz", ".packages") |
| rootfs_packages = os.path.basename(rootfs_packages_url) |
| |
| if platform == "openembedded": |
| rootfs_packages_url = "n/a" |
| rootfs_packages = "n/a" |
| |
| # try to get information from Jenkins, if available |
| jenkins_buildn = os.environ.get('BUILD_NUMBER') or 'Not Found' |
| jenkins_build_url = os.environ.get('BUILD_URL') or 'http://ci.linaro.org' |
| |
| buff = tmpl.substitute( |
| title=title, md5=md5, image_name=os.path.basename(gzfile), |
| hwpack_name=hwpack_name, hwpack_url=hwpack_url, |
| rootfs_name=rootfs_name, rootfs_url=rootfs_url, |
| hwpack_manifest_name=hwpack_manifest, |
| hwpack_manifest_url=hwpack_manifest_url, |
| rootfs_packages_name=rootfs_packages, |
| rootfs_packages_url=rootfs_packages_url, |
| jenkins_buildn=jenkins_buildn, |
| jenkins_build_url=jenkins_build_url) |
| with open(infofile, 'w') as f: |
| f.write(buff) |
| |
| def build_image(lmc, imgfile, hwpack, hwpack_file, binary, |
| binary_file, fastmodel=False): |
| print "INFO: building image: %s" % imgfile |
| hwpi = HWPACKS[hwpack] |
| size = BINARIES[binary] |
| |
| args = [lmc, |
| '--image-size', size, |
| '--hwpack-force-yes', |
| '--hwpack', hwpack_file, |
| '--binary', binary_file, |
| ] |
| |
| if fastmodel: |
| args += ['--dev', 'fastmodel', |
| '--output-directory', os.path.dirname(imgfile), |
| '--image-file', os.path.basename(imgfile)] |
| else: |
| args += ['--dev', hwpi.lmcname, |
| '--image-file', imgfile] |
| |
| if hwpi.pre_inst_script is not None: |
| sdir = os.path.abspath(os.path.dirname(__file__)) |
| script = "%s/%s" % (sdir, hwpi.pre_inst_script) |
| print "using presintall script: %s" % script |
| args.append('--preinstall-script') |
| args.append(script) |
| |
| print "INFO: running l-m-c:", " ".join([str(arg) for arg in args]) |
| |
| hwpi.do_eula(imgfile) |
| |
| # some useful information so we know how much time it took per lmc run |
| before = time.time() |
| cmd_runner.run(args, as_root=True).wait() |
| delta_min, delta_secs = divmod(int(time.time() - before), 60) |
| print "INFO: linaro-media-create took %d min(s) and %d sec(s) to run" % ( |
| delta_min, delta_secs) |
| |
| class SnapshotCrawler: |
| def __init__(self, platform, hwpacks, binaries): |
| self.platform = platform |
| self.hwpacks = hwpacks |
| self.binaries = binaries |
| |
| def get_binaries(self): |
| '''return a hash table of binary->url''' |
| binaryf = {} |
| for binary in self.binaries: |
| url = "%s/%s/images/%s" % (SNAPSHOTS_URL, self.platform, binary) |
| (date, url) = crawler.latest_rfs(url) |
| binaryf[binary] = url |
| |
| return binaryf |
| |
| def get_hwpacks(self): |
| '''return a hash table of hwpack->urls (date, url)''' |
| hwpackf = {} |
| for hwpack in self.hwpacks: |
| url = "%s/%s/hwpacks/%s" % (SNAPSHOTS_URL, self.platform, hwpack) |
| hwpackf[hwpack] = crawler.latest_hwpacks(url) |
| |
| return hwpackf |
| |
| def get_image_name(self, odir, hwpack, hwpack_url, binary): |
| |
| #convert a url like: |
| # http://snapshots.linaro.org/SERIE/hwpacks/HWPACK_NAME/HWPACK_BUILD_NUMBER/HWPACK_FILE_NAME |
| # to: |
| # SERIE/pre-built/HWPACK_NAME/HWPACK_BUILD_NUMBER/HWPACK_FILE_NAME.img |
| path = urlparse.urlparse(hwpack_url).path |
| parts = path.split('/') |
| name = parts[-1] #use for the file name below |
| path = '/'.join(parts[-3:-1]) |
| |
| path = '%s/%s' % (odir, path) |
| if not os.path.exists(path): |
| os.makedirs(path) |
| |
| date = re.search(r'_(\d+-\d+)_', name).group(1) |
| return "%s/%s-%s_%s_%s.img" % (path, hwpack, self.platform, binary, date) |
| |
| def main(): |
| hwpacks = HWPACKS.keys() |
| binaries = BINARIES.keys() |
| |
| p = argparse.ArgumentParser(description= |
| 'Builds a matrix of builds from the latest build on ' |
| 'snapshots.linaro.org or release images on releases.linaro.org') |
| |
| p.add_argument('-w', dest='hwpacks', action='append', |
| help='The hwpacks to generate for, default=%s' % |
| ', '.join(hwpacks)) |
| p.add_argument('-b', dest='binaries', action='append', |
| help='The binaries to generate for, default=%s' % |
| ', '.join(binaries)) |
| p.add_argument('-p', dest='platform', default='quantal', |
| help='The platform, default=quantal') |
| p.add_argument('-o', dest='out_dir', default='./out', |
| help='The out directory for downloaded and built files, default=./out') |
| p.add_argument('-d', dest='date', default="", |
| help='The date (YYYYMMDD) as reference for hwpack') |
| p.add_argument('-r', dest='release', action='store_true', default=False, |
| help='If this is for release images. NOTE: the "date" arg' |
| 'will then become the cycle ie -d 12.03') |
| p.add_argument('-f', dest='fastmodel', action='store_true', |
| help='Build for the fastmodel target (for vexpress hwpack)') |
| |
| |
| args = p.parse_args() |
| |
| if args.hwpacks: |
| hwpacks = args.hwpacks |
| if args.binaries: |
| binaries = args.binaries |
| |
| crawler.cookie_setup() |
| |
| dm = fetch_image.DownloadManager(args.out_dir) |
| |
| if args.release: |
| site = ReleaseCrawler(args.date, args.platform, hwpacks, binaries) |
| else: |
| site = SnapshotCrawler(args.platform, hwpacks, binaries) |
| |
| hwpackf = site.get_hwpacks() |
| binaryf = site.get_binaries() |
| |
| lmc = '%s/linaro-image-tools/linaro-media-create' % args.out_dir |
| odir = '%s/pre-built' % args.out_dir |
| |
| for hwpack in hwpacks: |
| url = None |
| # check if the user wanted a hwpack from a specific date |
| if args.date: |
| try: |
| dates = [elem[0] for elem in hwpackf[hwpack]] |
| index = dates.index(args.date) |
| url = hwpackf[hwpack][index][1] |
| except: |
| print "ERROR: Could not find hwpack '%s' with date '%s'" \ |
| ", skipping pre-built image for it" % (hwpack, args.date) |
| else: |
| # just grab the latest hwpack |
| try: |
| url = hwpackf[hwpack][0][1] |
| except: |
| print "ERROR: Could not find a valid hwpack url for '%s'" \ |
| ", skipping pre-built image for it" % (hwpack) |
| |
| if url is not None and url is not False: |
| hwpf = download(dm, url) |
| for binary in binaries: |
| # create the image and metadata |
| imgfile = site.get_image_name(odir, hwpack, url, binary) |
| infofile = '%s.html' % os.path.splitext(imgfile)[0] |
| rfsf = download(dm, binaryf[binary]) |
| build_image(lmc, imgfile, hwpack, hwpf, binary, rfsf, |
| args.fastmodel) |
| gzfile = compress_image(imgfile) |
| zsync_image(args.platform, gzfile) |
| create_image_info(infofile, gzfile, imgfile, url, \ |
| binaryf[binary], args.platform) |
| |
| if __name__ == '__main__': |
| main() |