fetch-from-snapshots.py

#!/usr/bin/python
#
# (C) 2013 Linaro Ltd.
#
# Based on scripts/download.py from lp:linaro-licence-protection
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
# 
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
# details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import json
import urlparse
import shutil
import urllib2
import os
import sys
import re

def download(api_urls, file_regexp):
    """Example of how to use the API to download a/all files in a directory."""

    # Get listing for file(s) pointed to by URL we were given
    request = urllib2.urlopen(api_urls.ls())
    listing = json.loads(request.read())["files"]

    for file_info in listing:
        if file_info["type"] == "folder":
            # Skip folders...
            continue

        if not re.match(file_regexp, file_info["name"]):
            continue

        print "downloading %s" % file_info["url"]
        # just download the file.
        req = urllib2.urlopen(urllib2.Request(api_urls.file(file_info["url"])))
        with open(os.path.basename(file_info["url"]), 'wb') as fp:
            shutil.copyfileobj(req, fp)

class ApiUrls():
    """Since we want to manipulate URLS, but urlsplit returns an immutable
    object this is a convenience object to perform the manipulations for us"""
    def __init__(self, input_url):
        self.parsed_url = [c for c in urlparse.urlsplit(input_url)]
        self.path = self.parsed_url[2]

    def ls(self, path=None):
        if not path:
            path = self.path
        self.parsed_url[2] = "/api/ls" + path
        return urlparse.urlunsplit(self.parsed_url)

    def license(self, path):
        self.parsed_url[2] = "/api/license" + path
        return urlparse.urlunsplit(self.parsed_url)

    def file(self, path):
        self.parsed_url[2] = path
        return urlparse.urlunsplit(self.parsed_url)

def check_for_latest(api_urls):
    
    request = urllib2.urlopen(api_urls.ls())
    listing = json.loads(request.read())["files"]

    latest = 0
    for file_info in listing:
        if file_info["name"] != "latest" and file_info["name"] > latest:
            latest = file_info["name"]

    print "latest is %s" % latest
    return latest

if __name__ == '__main__':
    if len(sys.argv) != 3:
        print >> sys.stderr, "Usage: fetch-from-snapshots.py base_url filenameregexp"
        print >> sys.stderr, ""
        print >> sys.stderr, "Example: fetch-from-snapshots.py http://snapshots.linaro.org/openembedded/images/minimal-armv8 linaro-image-minimal.*tar.gz"
        exit(1)

    latest = check_for_latest(ApiUrls(sys.argv[1]))
    download(ApiUrls(sys.argv[1] + "/" + latest), sys.argv[2])