1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
#!/usr/bin/env python
import argparse
import atexit
import cStringIO
import os
import pycurl
import sys
import tempfile
import time
# Public artifacts BUILD-INFO.txt
build_info = 'Format-Version: 0.5\n\nFiles-Pattern: *\nLicense-Type: open\n'
class API_v1(object):
def __init__(self, server, build_info, api_key):
self.server = server
self.api_base = server
self.build_info = build_info
self.api_key = api_key
self.curl = pycurl.Curl()
def __del__(self):
self.curl.close()
def _upload_data(self, url, data, headers=None, retry_count=3):
response = cStringIO.StringIO()
self.curl.setopt(pycurl.URL, url)
self.curl.setopt(pycurl.HTTPPOST, data)
self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
if headers:
self.curl.setopt(pycurl.HTTPHEADER, headers)
try:
self.curl.perform()
except Exception as e:
if retry_count > 0:
# server could be reloading or something. give it a second and
# try again
print('Upload failed for %s, retrying in 2 seconds' % url)
time.sleep(2)
return self._upload_data(url, data, headers, retry_count - 1)
else:
return str(e)
code = self.curl.getinfo(pycurl.RESPONSE_CODE)
if code not in (200, 201):
return response.getvalue()
def upload_file(self, url, filename):
data = [
('file', (pycurl.FORM_FILE, filename)),
('key', (pycurl.FORM_CONTENTS, self.api_key)),
]
return self._upload_data(url, data)
def upload_transfer_queue(self, transfer_queue):
transfer_failures = []
for transfer_item in transfer_queue:
failure = self.upload_file(
transfer_item, transfer_queue[transfer_item])
if failure:
transfer_failures.append('%s: %s' % (transfer_item, failure))
return transfer_failures
def get_transfer_queue(self, src, dst):
if self.api_base[-1] != '/' and dst[0] != '/':
# one of these needs to be a slash to produce a url
dst = '/' + dst
transfer_queue = {}
src_dir = os.path.abspath(src)
if not os.path.isdir(src_dir):
transfer_queue[self.api_base + dst + "/" + os.path.basename(src)] = src_dir
return transfer_queue
for root, dirs, files in os.walk(src_dir):
dst_dir = dst
if not root.endswith(dst): # in sub directory
dst_dir = os.path.join(dst, root[len(src_dir) + 1:])
for f in files:
dst_file = '%s%s/%s' % (self.api_base, dst_dir, f)
transfer_queue[dst_file] = os.path.join(root, f)
build_info_file = os.path.join(root, 'BUILD-INFO.txt')
if not os.path.exists(build_info_file):
dst_file = '%s%s/%s' % (
self.api_base, dst_dir, 'BUILD-INFO.txt')
transfer_queue[dst_file] = self.build_info
return transfer_queue
def upload(self, src, dst):
transfer_queue = self.get_transfer_queue(src, dst)
if not transfer_queue:
print "Warning: no files to publish"
return self.upload_transfer_queue(transfer_queue)
class API_v2(API_v1):
def __init__(self, server, build_info, api_key):
super(API_v2, self).__init__(server, build_info, api_key)
self.api_base = server + '/api/v2/publish/'
def upload_file(self, url, filename):
headers = ['AuthToken: ' + self.api_key]
data = [('file', (pycurl.FORM_FILE, filename))]
return self._upload_data(url, data, headers)
def main():
parser = argparse.ArgumentParser(
description='Copy file(s) from source to destination')
parser.add_argument('-k', '--key', help='key used for the copy')
parser.add_argument('-a', '--api_version', choices=('1', '2'), default='1',
help='API version to use. default=%(default)s')
parser.add_argument('--server', default='http://snapshots.linaro.org/',
help='Publishing API server. default=%(default)s')
parser.add_argument('-b', '--build-info', help='Custom build-info file')
parser.add_argument('src', help='source directory with files to publish')
parser.add_argument('dst', help='destination to publish to')
arguments = parser.parse_args()
# Publish key is required. Fallback to PUBLISH_KEY environment
# variable when it isn't passed as an argument
if arguments.key:
key = arguments.key
else:
key = os.environ.get('PUBLISH_KEY')
if key is None:
key = os.environ.get('PUBLISH_TOKEN')
if key:
arguments.api_version = '2'
else:
sys.exit('Error: Key is not defined.')
if not arguments.build_info:
fd, arguments.build_info = tempfile.mkstemp(prefix='BUILD-INFO.txt')
atexit.register(os.unlink, arguments.build_info)
os.write(fd, build_info)
if arguments.api_version == '1':
api = API_v1(arguments.server, arguments.build_info, key)
else:
api = API_v2(arguments.server, arguments.build_info, key)
transfer_failures = api.upload(arguments.src, arguments.dst)
if len(transfer_failures) > 0:
sys.exit('Error: Failed to transfer:\n %s' % '\n '.join(transfer_failures))
if __name__ == '__main__':
main()
|
