summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndy Doan <andy.doan@linaro.org>2015-08-24 12:39:11 -0500
committerAndy Doan <andy.doan@linaro.org>2015-08-24 12:40:48 -0500
commitbe1eac5f2c052c3bc6eb0fd4c2ab43b8674cdb23 (patch)
tree553eddfb97a813fc702573f06a2e4dfdedaca009
parenta22bfb931095605cc202fdaf432cd01a3af43eac (diff)
add support for new v3 api
l-l-p has a new v3 API that takes advantage of Amazon S3. Specifically it generates temp urls (or pre-signed urls) that push the actual file to. Change-Id: I867c7ab7565db586f00e96c83417675036bcc56e
-rwxr-xr-xlinaro-cp.py63
1 files changed, 58 insertions, 5 deletions
diff --git a/linaro-cp.py b/linaro-cp.py
index ed5fc10..5819211 100755
--- a/linaro-cp.py
+++ b/linaro-cp.py
@@ -4,6 +4,7 @@ import argparse
import atexit
import cStringIO
import os
+import mimetypes
import pycurl
import sys
import tempfile
@@ -142,11 +143,65 @@ class API_v2(API_v1):
return self._upload_data(url, [('foo', 'bar')], headers)
+class API_v3(API_v1):
+ def __init__(self, server, build_info, api_key):
+ super(API_v3, self).__init__(server, build_info, api_key)
+ self.api_base = server + '/api/v3/publish/'
+
+ def _upload_data(self, url, data, headers=None, retry_count=3):
+ self.last_headers = cStringIO.StringIO()
+ self.curl.setopt(pycurl.HEADERFUNCTION, self.last_headers.write)
+ return super(API_v3, self)._upload_data(
+ url, data, headers, retry_count)
+
+ def _put_s3(self, url, filename, mtype):
+ response = cStringIO.StringIO()
+ size = os.path.getsize(filename)
+ headers = ['Content-Type: ' + mtype]
+ c = pycurl.Curl()
+ c.setopt(pycurl.URL, url)
+ c.setopt(pycurl.HTTPHEADER, headers)
+ c.setopt(pycurl.INFILESIZE, size)
+ c.setopt(pycurl.PUT, 1)
+ c.setopt(pycurl.WRITEFUNCTION, response.write)
+ with open(filename, 'rb') as f:
+ c.setopt(pycurl.INFILE, f)
+ c.perform()
+ code = c.getinfo(pycurl.RESPONSE_CODE)
+ if code not in (200, 201):
+ return response.getvalue()
+
+ def upload_file(self, url, filename):
+ # ask llp for an s3 tempurl:
+ mtype = mimetypes.guess_type(filename)[0]
+ if not mtype:
+ mtype = 'other'
+
+ headers = ['AuthToken: ' + self.api_key]
+ code = self._upload_data(url, [('Content-Type', mtype)], headers)
+ if code:
+ return code
+
+ # now find the tempurl and really publish
+ for header in self.last_headers.getvalue().split('\n'):
+ if header.startswith('Location:'):
+ location = header[9:].strip()
+ return self._put_s3(location, filename, mtype)
+ raise RuntimeError('l-l-p response missing s3 location')
+
+ def link_latest(self, dst):
+ headers = ['AuthToken: ' + self.api_key]
+ url = self.server + '/api/v3/link_latest/' + dst
+ # pycurl requires data to be passed, or it will do an
+ # HTTP GET even though we said to POST
+ return self._upload_data(url, [('foo', 'bar')], headers)
+
+
def main():
parser = argparse.ArgumentParser(
description='Copy file(s) from source to destination')
parser.add_argument('-k', '--key', help='key used for the copy')
- parser.add_argument('-a', '--api_version', choices=('1', '2'), default='1',
+ parser.add_argument('-a', '--api_version', choices=('1', '2', '3'), default='1',
help='API version to use. default=%(default)s')
parser.add_argument('--server', default='http://snapshots.linaro.org/',
help='Publishing API server. default=%(default)s')
@@ -191,10 +246,8 @@ def main():
atexit.register(os.unlink, arguments.build_info)
os.write(fd, build_info)
- if arguments.api_version == '1':
- api = API_v1(arguments.server, arguments.build_info, key)
- else:
- api = API_v2(arguments.server, arguments.build_info, key)
+ cls = globals()['API_v' + arguments.api_version]
+ api = cls(arguments.server, arguments.build_info, key)
if arguments.split_job_owner:
# Rewrite job path .../owner_jobname/123 -> .../~owner/jobname/123 ,