crawler.py

#!/usr/bin/python

import cookielib
import os
import re
import urllib2
import sys

def cookie_setup():
    cookies = os.getenv('LMC_COOKIES')
    if cookies:
        cj = cookielib.LWPCookieJar()
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
        opener.addheaders.append(('Cookie', cookies))
        urllib2.install_opener(opener)

def list_links(url, regex=r'<a\s*href=[\'"].*[\'"]>(.*)/?</a>'):
    try:
        response = urllib2.urlopen(url)
        msg = response.read()
        links = re.compile(regex).findall(msg)
    except urllib2.HTTPError,e:
        print "ERROR: finding links for (%s): %s" % (url, e)
        links = [] #return empty array
    return links

def list_hwpack(url):
    ''' returns tuple of (buildate, url)
    '''
    urls = list_links(url, r'<a\s*href=[\'|"].*[\'"]>(hwpack.*?\.tar\.gz)</a>')
    for link in urls:
        try:
            build_date = re.compile('_(\d+)-').findall(link)
            return (build_date[0], '%s/%s' % (url,link))
        except:
            return None
    return None

def latest_hwpacks(url, limit=7):
    '''returns an array of tuples (build-date, hwpack url) like:
       [ (20120210, http://foo.bar/hwpack.tar.gz), (20120209, blah.tar.gz) ]
    '''
    # only analyze the last few builds
    links = list_links(url, r'<a\s*href=[\'|"].*[\'"]>(\d+)/?</a>')
    links = sorted(links, reverse=True, key=int)[:limit]
    hwpacks = []
    for link in links:
        build = list_hwpack('%s/%s'% (url, link))
        if build is not None:
            hwpacks.append(build)
    return hwpacks

def list_rfs(url):
    links = list_links(url,
       r'<a\s*href=[\'|"].*[\'"]>(.*\-\d+\.(?!config)(?:rootfs\.)?tar\.gz)</a>')
    if len(links) is 1:
        return "%s/%s" %(url,links[0])
    return None

def latest_rfs(url, limit=7):
    '''
    Returns a tuple of (builddate, url)
    '''
    # only analyze the last few builds
    links = list_links(url, r'<a\s*href=[\'"].*[\'"]>(\d+)/?</a>')
    links = sorted(links, reverse=True, key=int)[:limit]
    for link in links:
        build = list_rfs('%s/%s' %(url, link))
        if build is not None:
            return (link, build)

    return None

if __name__ == '__main__':
    cookie_setup()

    for arg in sys.argv[1:]:
        print "HWPACKS for: %s" % arg
        hwpacks = latest_hwpacks(arg, 4)
        for hwpack in hwpacks:
            print "  %s: %s" % hwpack

    print "latest nano:"
    print "  %s %s" % latest_rfs('http://snapshots.linaro.org/quantal/images/nano')