s3_flatten.py: add retries for boto3 bucket operations

It turns out that this script can occasionally error out
if the AWS API doesn't respond with proper HTTP.  When
this happens, the error isn't caught and the script ends
prematurely and can leave large amounts of unnecessary
data in the bucket.

This wraps S3 bucket operations (specifically delete_keys
and list_objectS) in a try statement that will attempt
up to 3 retries after sleeping for 30 seconds.

Change-Id: I3f4c945937b20635166cad093630bfbca9026a78
Reviewed-on: https://review.linaro.org/c/infrastructure/linaro-license-protection/+/35063
Reviewed-by: Benjamin Copeland <ben.copeland@linaro.org>

1 files changed, 19 insertions, 2 deletions

diff --git a/license_protected_downloads/management/commands/s3_flatten.py b/license_protected_downloads/management/commands/s3_flatten.py
index 128b339..9e7651b 100644
--- a/license_protected_downloads/management/commands/s3_flatten.py
+++ b/license_protected_downloads/management/commands/s3_flatten.py
@@ -7,6 +7,7 @@ from fnmatch import fnmatch
 from boto.s3.connection import S3Connection
 from boto.s3 import deletemarker,key,prefix
 import sys
+import httplib
 
 logging.getLogger().setLevel(logging.INFO)
 
@@ -62,7 +63,15 @@ class Command(BaseCommand):
                     logging.info("deleting: %s" % (x))
 
         if not dryrun:
-            bucket.delete_keys(delete_list)
+            retries = 3
+            while retries > 0:
+                try:
+                    bucket.delete_keys(delete_list)
+                    retries = -1
+                except httplib.BadStatusLine as e:
+                    logging.error("httplib error in delete_keys():  %" % e)
+                    retries -= 1
+                    sleep(30)
         else:
             logging.info( "DRYRUN: delete_keys for %s keys" % len(delete_list) )
 
@@ -78,7 +87,15 @@ class Command(BaseCommand):
     def handle_bucket(self, *args, **options):
         logging.info( "--> %s" % options['prefix'])
 
-        bucket_keys = self.bucket.list_versions(options['prefix'], delimiter='/')
+        retries = 3
+        while retries > 0:
+            try:
+                bucket_keys = self.bucket.list_versions(options['prefix'], delimiter='/')
+                retries = -1
+            except httplib.BadStatusLine as e:
+                logging.error("httplib error in delete_keys():  %" % e)
+                retries -= 1
+                sleep(30)
 
         objs = {}
         delete_list = []