crawler.py

#!/usr/bin/python

import re
import urllib2

def list_links(url, regex=r'<a\s*href=[\'|"](.*?)[\'"].*?>'):
    try:
        response = urllib2.urlopen(url)
        msg = response.read()
        links = re.compile(regex).findall(msg)
    except urllib2.HTTPError,e:
        print "ERROR finding links for (%s): %s" % (url, e)
        links = [] #return empty array
    return links

def list_hwpack(url):
    ''' returns tuple of (buildate, url)
    '''
    urls = list_links(url, r'<a\s*href=[\'|"](hwpack.*?\.tar\.gz)[\'"].*?>')
    for link in urls:
        build_date = re.compile('_(\d+)-').findall(link)
        return (build_date[0], '%s/%s' % (url,link))
    return None

def latest_hwpacks(url, limit=7):
    '''returns an array of tuples (build-date, hwpack url) like:
       [ (20120210, http://foo.bar/hwpack.tar.gz), (20120209, blah.tar.gz) ]
    '''
    # only analyze the last few builds
    links = list_links(url, r'<a\s*href=[\'|"](\d+)\/[\'"].*?>')[:limit]
    hwpacks = []
    for link in sorted(links, reverse=True):
        build = list_hwpack('%s/%s'% (url, link))
        if build is not None:
            hwpacks.append(build)
    return hwpacks

def list_rfs(url):
    links = list_links(url, r'<a\s*href=[\'|"](linaro-.*?\d+(?!config)\.tar\.gz)[\'"].*?>')
    if len(links) is 1:
        return "%s/%s" %(url,links[0])
    return None

def latest_rfs(url, limit=7):
    '''
    Returns a tuple of (builddate, url)
    '''
    # only analyze the last few builds
    links = list_links(url, r'<a\s*href=[\'|"](\d+)\/[\'"].*?>')[:limit]
    for link in sorted(links, reverse=True, key=int):
        build = list_rfs('%s/%s' %(url, link))
        if build is not None:
            return (link, build)

    return None

if __name__ == '__main__':
    import sys
    for arg in sys.argv[1:]:
        print "HWPACKS for: %s" % arg
        hwpacks = latest_hwpacks(arg, 4)
        for hwpack in hwpacks:
            print "  %s: %s" % hwpack

    print "latest nano:"
    print "  %s %s" % latest_rfs('http://snapshots.linaro.org/precise/images/nano')