#!/usr/bin/env python import argparse import atexit import cStringIO import os import pycurl import sys import tempfile import time import re # Public artifacts BUILD-INFO.txt build_info = 'Format-Version: 0.5\n\nFiles-Pattern: *\nLicense-Type: open\n' class API_v1(object): def __init__(self, server, build_info, api_key): self.server = server self.api_base = server self.build_info = build_info self.api_key = api_key self.curl = pycurl.Curl() def __del__(self): self.curl.close() def _upload_data(self, url, data, headers=None, retry_count=3): response = cStringIO.StringIO() self.curl.setopt(pycurl.URL, url) self.curl.setopt(pycurl.HTTPPOST, data) self.curl.setopt(pycurl.WRITEFUNCTION, response.write) if headers: self.curl.setopt(pycurl.HTTPHEADER, headers) try: self.curl.perform() except Exception as e: if retry_count > 0: # server could be reloading or something. give it a second and # try again print('Upload failed for %s, retrying in 2 seconds' % url) time.sleep(2) return self._upload_data(url, data, headers, retry_count - 1) else: return str(e) code = self.curl.getinfo(pycurl.RESPONSE_CODE) if code not in (200, 201): return response.getvalue() def upload_file(self, url, filename): data = [ ('file', (pycurl.FORM_FILE, filename)), ('key', (pycurl.FORM_CONTENTS, self.api_key)), ] return self._upload_data(url, data) def upload_transfer_queue(self, transfer_queue): transfer_failures = [] for transfer_item in transfer_queue: failure = self.upload_file( transfer_item, transfer_queue[transfer_item]) if failure: transfer_failures.append('%s: %s' % (transfer_item, failure)) return transfer_failures def get_transfer_queue(self, src, dst, options): if self.api_base[-1] != '/' and dst[0] != '/': # one of these needs to be a slash to produce a url dst = '/' + dst transfer_queue = {} src_dir = os.path.abspath(src) if not os.path.isdir(src_dir): transfer_queue[self.api_base + dst + "/" + os.path.basename(src)] = src_dir return transfer_queue manifest = None if options.manifest: manifest = tempfile.NamedTemporaryFile(prefix="MANIFEST", delete=False) for root, dirs, files in os.walk(src_dir): rel_dir = root[len(src_dir) + 1:] for f in files: rel_path = os.path.join(rel_dir, f) # If there's at least one include pattern, skip unless # some pattern matches skip = len(options.include) > 0 for pat in options.include: if re.search(pat, rel_path): skip = False break if not skip: dst_file = '%s%s/%s' % (self.api_base, dst, rel_path) transfer_queue[dst_file] = os.path.join(root, f) if manifest: manifest.write(rel_path + "\n") if not options.no_build_info: build_info_file = os.path.join(root, 'BUILD-INFO.txt') if not os.path.exists(build_info_file): dst_file = '%s%s/%s/%s' % ( self.api_base, dst, rel_dir, 'BUILD-INFO.txt') transfer_queue[dst_file] = self.build_info if manifest: manifest.close() dst_file = '%s%s/%s' % ( self.api_base, dst, 'MANIFEST') transfer_queue[dst_file] = manifest.name return transfer_queue def upload(self, src, dst, options): transfer_queue = self.get_transfer_queue(src, dst, options) if not transfer_queue: print "Warning: no files to publish" if options.verbose: from pprint import pprint pprint(transfer_queue) if not options.dry_run: return self.upload_transfer_queue(transfer_queue) else: return [] class API_v2(API_v1): def __init__(self, server, build_info, api_key): super(API_v2, self).__init__(server, build_info, api_key) self.api_base = server + '/api/v2/publish/' def upload_file(self, url, filename): headers = ['AuthToken: ' + self.api_key] data = [('file', (pycurl.FORM_FILE, filename))] return self._upload_data(url, data, headers) def main(): parser = argparse.ArgumentParser( description='Copy file(s) from source to destination') parser.add_argument('-k', '--key', help='key used for the copy') parser.add_argument('-a', '--api_version', choices=('1', '2'), default='1', help='API version to use. default=%(default)s') parser.add_argument('--server', default='http://snapshots.linaro.org/', help='Publishing API server. default=%(default)s') parser.add_argument('-i', '--include', action='append', default=[], help='Include regex for files') parser.add_argument('--no-build-info', action='store_true', help="Don't auto-generate BUILD-INFO.txt") parser.add_argument('-b', '--build-info', help='Custom build-info file') parser.add_argument('--split-job-owner', action='store_true', help='Split Jenkins job owner in dst (owner_job -> ~owner/job)') parser.add_argument('--manifest', action='store_true', help='Generate MANIFEST file with list of all files published') parser.add_argument('--verbose', action='store_true', help='Verbose operation') parser.add_argument('--dry-run', action='store_true', help="Don't actually publish files") parser.add_argument('src', help='source directory with files to publish') parser.add_argument('dst', help='destination to publish to') arguments = parser.parse_args() # Publish key is required. Fallback to PUBLISH_KEY environment # variable when it isn't passed as an argument if arguments.key: key = arguments.key else: key = os.environ.get('PUBLISH_KEY') if key is None: key = os.environ.get('PUBLISH_TOKEN') if key: arguments.api_version = '2' else: sys.exit('Error: Key is not defined.') if not arguments.build_info: fd, arguments.build_info = tempfile.mkstemp(prefix='BUILD-INFO.txt') atexit.register(os.unlink, arguments.build_info) os.write(fd, build_info) if arguments.api_version == '1': api = API_v1(arguments.server, arguments.build_info, key) else: api = API_v2(arguments.server, arguments.build_info, key) if arguments.split_job_owner: # Rewrite job path .../owner_jobname/123 -> .../~owner/jobname/123 , # as required for android-build publishing. arguments.dst = re.sub(r"^(.+?)/([^/]+?)_([^/]+?)/([0-9]+)/?$", r"\1/~\2/\3/\4/", arguments.dst) transfer_failures = api.upload(arguments.src, arguments.dst, arguments) if len(transfer_failures) > 0: sys.exit('Error: Failed to transfer:\n %s' % '\n '.join(transfer_failures)) if __name__ == '__main__': main()