1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
#!/usr/bin/python
import json
import urlparse
import shutil
import urllib2
import os
from html2text import html2text
import sys
import xdg.BaseDirectory as xdgBaseDir
def download(api_urls, accepted_licenses):
"""Example of how to use the API to download a/all files in a directory."""
# Get listing for file(s) pointed to by URL we were given
request = urllib2.urlopen(api_urls.ls())
listing = json.loads(request.read())["files"]
for file_info in listing:
if file_info["type"] == "folder":
# Skip folders...
continue
# Get the licenses. They are returned as a JSON document in the form:
# {"licenses":
# [{"text": "<license text>", "digest": "<digest of license>"},
# {"text": "<license text>", "digest": "<digest of license>"},
# ...
# ]}
# Each license has a digest associated with it.
request = urllib2.urlopen(api_urls.license(file_info["url"]))
licenses = json.loads(request.read())["licenses"]
if licenses[0] == "Open":
headers = {}
else:
# Present each license to the user...
for lic in licenses:
if lic["digest"] not in accepted_licenses:
skip_license = os.environ.get('SKIP_LICENSE')
if skip_license is None:
# Licenses are stored as HTML. Convert them to markdown
# (text) and print it to the terminal.
print html2text(lic["text"])
# Ask the user if they accept the license. If they don't we
# terminate the script.
user_response = raw_input(
"Do you accept this license? (y/N)")
if user_response != "y":
exit(1)
# Remember this license acceptance for another download.
accepted_licenses.append(lic["digest"])
# To accept a license, place the digest in the LICENSE_ACCEPTED
# header. For multiple licenses, they are stored space separated.
digests = [lic["digest"] for lic in licenses]
headers = {"LICENSE_ACCEPTED": " ".join(digests)}
# Once the header has been generated, just download the file.
req = urllib2.urlopen(urllib2.Request(api_urls.file(file_info["url"]),
headers=headers))
with open(os.path.basename(file_info["url"]), 'wb') as fp:
shutil.copyfileobj(req, fp)
class ApiUrls():
"""Since we want to manipulate URLS, but urlsplit returns an immutable
object this is a convenience object to perform the manipulations for us"""
def __init__(self, input_url):
self.parsed_url = [c for c in urlparse.urlsplit(input_url)]
self.path = self.parsed_url[2]
def ls(self, path=None):
if not path:
path = self.path
self.parsed_url[2] = "/api/ls" + path
return urlparse.urlunsplit(self.parsed_url)
def license(self, path):
self.parsed_url[2] = "/api/license" + path
return urlparse.urlunsplit(self.parsed_url)
def file(self, path):
self.parsed_url[2] = path
return urlparse.urlunsplit(self.parsed_url)
if __name__ == '__main__':
if len(sys.argv) != 2:
# Check that a URL has been supplied.
print >> sys.stderr, "Usage: download.py <URL>"
exit(1)
dest = os.path.join(xdgBaseDir.xdg_data_home,"linaro")
accepted_licenses_path = os.path.join(dest, "accepted_licenses")
# Later we ask the user to accept each license in turn. Store which
# licenses are accepted so the user only has to accept them once.
if os.path.isfile(accepted_licenses_path):
with open(accepted_licenses_path) as accepted_licenses_file:
accepted_licenses = accepted_licenses_file.read().split()
else:
accepted_licenses = []
api_urls = ApiUrls(sys.argv[1])
download(api_urls, accepted_licenses)
# Store the licenses that the user accepted
if not os.path.exists(dest):
os.makedirs(dest)
with open(accepted_licenses_path, "w") as accepted_licenses_file:
accepted_licenses_file.write(" ".join(accepted_licenses))
|
