1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
|
#!/usr/bin/env python
import argparse
import atexit
import cStringIO
import os
import pycurl
import sys
import tempfile
import time
import re
# Public artifacts BUILD-INFO.txt
build_info = 'Format-Version: 0.5\n\nFiles-Pattern: *\nLicense-Type: open\n'
class API_v1(object):
def __init__(self, server, build_info, api_key):
self.server = server
self.api_base = server
self.build_info = build_info
self.api_key = api_key
self.curl = pycurl.Curl()
def __del__(self):
self.curl.close()
def _upload_data(self, url, data, headers=None, retry_count=3):
response = cStringIO.StringIO()
self.curl.setopt(pycurl.URL, url)
self.curl.setopt(pycurl.HTTPPOST, data)
self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
if headers:
self.curl.setopt(pycurl.HTTPHEADER, headers)
try:
self.curl.perform()
except Exception as e:
if retry_count > 0:
# server could be reloading or something. give it a second and
# try again
print('Upload failed for %s, retrying in 2 seconds' % url)
time.sleep(2)
return self._upload_data(url, data, headers, retry_count - 1)
else:
return str(e)
code = self.curl.getinfo(pycurl.RESPONSE_CODE)
if code not in (200, 201):
return response.getvalue()
def upload_file(self, url, filename):
data = [
('file', (pycurl.FORM_FILE, filename)),
('key', (pycurl.FORM_CONTENTS, self.api_key)),
]
return self._upload_data(url, data)
def upload_transfer_queue(self, transfer_queue):
transfer_failures = []
for transfer_item in transfer_queue:
failure = self.upload_file(
transfer_item, transfer_queue[transfer_item])
if failure:
transfer_failures.append('%s: %s' % (transfer_item, failure))
return transfer_failures
def get_transfer_queue(self, src, dst, options):
if self.api_base[-1] != '/' and dst[0] != '/':
# one of these needs to be a slash to produce a url
dst = '/' + dst
transfer_queue = {}
src_dir = os.path.abspath(src)
if not os.path.isdir(src_dir):
transfer_queue[self.api_base + dst + "/" + os.path.basename(src)] = src_dir
return transfer_queue
manifest = None
if options.manifest:
manifest = tempfile.NamedTemporaryFile(prefix="MANIFEST", delete=False)
for root, dirs, files in os.walk(src_dir):
rel_dir = root[len(src_dir) + 1:]
for f in files:
rel_path = os.path.join(rel_dir, f)
# If there's at least one include pattern, skip unless
# some pattern matches
skip = len(options.include) > 0
for pat in options.include:
if re.search(pat, rel_path):
skip = False
break
if not skip:
dst_file = '%s%s/%s' % (self.api_base, dst, rel_path)
transfer_queue[dst_file] = os.path.join(root, f)
if manifest:
manifest.write(rel_path + "\n")
if not options.no_build_info:
build_info_file = os.path.join(root, 'BUILD-INFO.txt')
if not os.path.exists(build_info_file):
dst_file = '%s%s/%s/%s' % (
self.api_base, dst, rel_path, 'BUILD-INFO.txt')
transfer_queue[dst_file] = self.build_info
if manifest:
manifest.close()
dst_file = '%s%s/%s' % (
self.api_base, dst, 'MANIFEST')
transfer_queue[dst_file] = manifest.name
return transfer_queue
def upload(self, src, dst, options):
transfer_queue = self.get_transfer_queue(src, dst, options)
if not transfer_queue:
print "Warning: no files to publish"
#print transfer_queue
return self.upload_transfer_queue(transfer_queue)
class API_v2(API_v1):
def __init__(self, server, build_info, api_key):
super(API_v2, self).__init__(server, build_info, api_key)
self.api_base = server + '/api/v2/publish/'
def upload_file(self, url, filename):
headers = ['AuthToken: ' + self.api_key]
data = [('file', (pycurl.FORM_FILE, filename))]
return self._upload_data(url, data, headers)
def main():
parser = argparse.ArgumentParser(
description='Copy file(s) from source to destination')
parser.add_argument('-k', '--key', help='key used for the copy')
parser.add_argument('-a', '--api_version', choices=('1', '2'), default='1',
help='API version to use. default=%(default)s')
parser.add_argument('--server', default='http://snapshots.linaro.org/',
help='Publishing API server. default=%(default)s')
parser.add_argument('-i', '--include', action='append', default=[],
help='Include regex for files')
parser.add_argument('--no-build-info', action='store_true',
help="Don't auto-generate BUILD-INFO.txt")
parser.add_argument('-b', '--build-info', help='Custom build-info file')
parser.add_argument('--split-job-owner', action='store_true',
help='Split Jenkins job owner in dst (owner_job -> ~owner/job)')
parser.add_argument('--manifest', action='store_true',
help='Generate MANIFEST file with list of all files published')
parser.add_argument('src', help='source directory with files to publish')
parser.add_argument('dst', help='destination to publish to')
arguments = parser.parse_args()
# Publish key is required. Fallback to PUBLISH_KEY environment
# variable when it isn't passed as an argument
if arguments.key:
key = arguments.key
else:
key = os.environ.get('PUBLISH_KEY')
if key is None:
key = os.environ.get('PUBLISH_TOKEN')
if key:
arguments.api_version = '2'
else:
sys.exit('Error: Key is not defined.')
if not arguments.build_info:
fd, arguments.build_info = tempfile.mkstemp(prefix='BUILD-INFO.txt')
atexit.register(os.unlink, arguments.build_info)
os.write(fd, build_info)
if arguments.api_version == '1':
api = API_v1(arguments.server, arguments.build_info, key)
else:
api = API_v2(arguments.server, arguments.build_info, key)
if arguments.split_job_owner:
# Rewrite job path .../owner_jobname/123 -> .../~owner/jobname/123 ,
# as required for android-build publishing.
arguments.dst = re.sub(r"^(.+?)/([^/]+?)_([^/]+?)/([0-9]+)/?$", r"\1/~\2/\3/\4/", arguments.dst)
transfer_failures = api.upload(arguments.src, arguments.dst, arguments)
if len(transfer_failures) > 0:
sys.exit('Error: Failed to transfer:\n %s' % '\n '.join(transfer_failures))
if __name__ == '__main__':
main()
|
