summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPaul Sokolovsky <pfalcon@users.sourceforge.net>2016-12-06 21:13:23 +0300
committerPaul Sokolovsky <pfalcon@users.sourceforge.net>2016-12-09 01:05:08 +0300
commit6ba065ab03883d5c989c57a05e37efd75387301e (patch)
tree6509a187704dfdc498c81aba29cc45d6f1a05271
parentf86ab2379a6a17cdfc932bcc217ada3c429db1f4 (diff)
downloadpublishing-api-6ba065ab03883d5c989c57a05e37efd75387301e.tar.gz
linaro-cp.py: Add thread pool support.
Based on Ben Copeland's experimental code. Also, consistently use exceptions for error propagation. Change-Id: I42993861d65b5c12fe6204cade8191c42b0d031f
-rwxr-xr-xlinaro-cp.py23
1 files changed, 14 insertions, 9 deletions
diff --git a/linaro-cp.py b/linaro-cp.py
index ac60b5b..54af9c1 100755
--- a/linaro-cp.py
+++ b/linaro-cp.py
@@ -9,6 +9,7 @@ import tempfile
import time
import re
import pprint
+from multiprocessing.pool import ThreadPool
import requests
@@ -42,15 +43,19 @@ class API_v1(object):
def upload_file(self, url, filename):
with open(filename, 'rb') as f:
- return self._upload_data(url, files={'file': f}, data={'key': self.api_key})
+ self._upload_data(url, files={'file': f}, data={'key': self.api_key})
def upload_transfer_queue(self, transfer_queue):
+ p = ThreadPool(10)
transfer_failures = []
- for transfer_item in transfer_queue:
- failure = self.upload_file(
- transfer_item, transfer_queue[transfer_item])
- if failure:
- transfer_failures.append('%s: %s' % (transfer_item, failure))
+
+ results = [p.apply_async(self.upload_file, (transfer_item, transfer_queue[transfer_item])) for transfer_item in transfer_queue]
+
+ for res in results:
+ try:
+ res.get()
+ except Exception as e:
+ transfer_failures.append(str(e))
return transfer_failures
def get_transfer_queue(self, src, dst, options):
@@ -119,7 +124,7 @@ class API_v2(API_v1):
def upload_file(self, url, filename):
with open(filename, 'rb') as f:
- return self._upload_data(url, files={'file': f}, headers={'AuthToken': self.api_key})
+ self._upload_data(url, files={'file': f}, headers={'AuthToken': self.api_key})
def link_latest(self, dst):
url = self.server + '/api/v2/link_latest/' + dst
@@ -148,7 +153,7 @@ class API_v3(API_v1):
time.sleep(2)
retry_count -= 1
- return "S3 upload failed"
+ raise Exception("S3 upload failed")
def upload_file(self, url, filename):
# ask llp for an s3 tempurl:
@@ -160,7 +165,7 @@ class API_v3(API_v1):
# now find the tempurl and really publish
location = resp.headers['location']
- return self._put_s3(location, filename, mtype)
+ self._put_s3(location, filename, mtype)
def link_latest(self, dst):
url = self.server + '/api/v3/link_latest/' + dst