bug 357: make upload more resilient to transient failures

While doing local testing, I noticed a whole publishing run failed
because my django server was reloading. This adds a small bit of
retry logic to the upload operation to try and handle things more
gracefully.

Change-Id: I96f208b94e4196ffed2bb23303fad96e960962f7

1 files changed, 13 insertions, 2 deletions

diff --git a/linaro-cp.py b/linaro-cp.py
index 2bd20d3..0e23bd6 100755
--- a/linaro-cp.py
+++ b/linaro-cp.py
@@ -5,6 +5,7 @@ import cStringIO
 import os
 import pycurl
 import sys
+import time
 
 # Public artifacts BUILD-INFO.txt
 build_info = 'Format-Version: 0.5\n\nFiles-Pattern: *\nLicense-Type: open\n'
@@ -24,7 +25,7 @@ def _get_transfer_queue(server_base, src, dst):
     return transfer_queue
 
 
-def _upload(curl, key, url, filename):
+def _upload(curl, key, url, filename, retry_count=3):
     response = cStringIO.StringIO()
     send = [
         ('file', (pycurl.FORM_FILE, filename)),
@@ -33,7 +34,17 @@ def _upload(curl, key, url, filename):
     curl.setopt(pycurl.URL, url)
     curl.setopt(pycurl.HTTPPOST, send)
     curl.setopt(pycurl.WRITEFUNCTION, response.write)
-    curl.perform()
+    try:
+        curl.perform()
+    except Exception as e:
+        if retry_count > 0:
+            # server could be reloading or something. give it a second and
+            # try again
+            print('Upload failed for %s, retrying in 2 seconds' % url)
+            time.sleep(2)
+            return _upload(curl, key, url, filename, retry_count - 1)
+        else:
+            return str(e)
 
     return response.getvalue()