s3_flatten.py: wrap the entire handle_bucket() method in retry logic

It turns out that boto will attempt to lazily load details for objects
in a bucket.  As a result, it is not sufficient to simply wrap the
list_objects() calls.  In order to prevent even more complexity
by attempting to predict where try/excepts retries are needed,
let's just wrap the handle_bucket() method itself and use the
wrapper to safely clean up objects.

Change-Id: I257f9390dbb6f59e83511e50594824306ad9baf8
Reviewed-on: https://review.linaro.org/c/infrastructure/linaro-license-protection/+/35184
Reviewed-by: Kelley Spoon <kelley.spoon@linaro.org>

1 files changed, 13 insertions, 17 deletions

diff --git a/license_protected_downloads/management/commands/s3_flatten.py b/license_protected_downloads/management/commands/s3_flatten.py
index 9e7651b..5044775 100644
--- a/license_protected_downloads/management/commands/s3_flatten.py
+++ b/license_protected_downloads/management/commands/s3_flatten.py
@@ -63,15 +63,7 @@ class Command(BaseCommand):
                     logging.info("deleting: %s" % (x))
 
         if not dryrun:
-            retries = 3
-            while retries > 0:
-                try:
-                    bucket.delete_keys(delete_list)
-                    retries = -1
-                except httplib.BadStatusLine as e:
-                    logging.error("httplib error in delete_keys():  %" % e)
-                    retries -= 1
-                    sleep(30)
+            bucket.delete_keys(delete_list)
         else:
             logging.info( "DRYRUN: delete_keys for %s keys" % len(delete_list) )
 
@@ -82,21 +74,25 @@ class Command(BaseCommand):
         self.now_mark = self.x_days_ago(int(options['markdays']))
         self.now_delete = self.x_days_ago(int(options['deletedays']))
 
-        self.handle_bucket(*args, **options)
-
-    def handle_bucket(self, *args, **options):
-        logging.info( "--> %s" % options['prefix'])
+        self.handle_bucket_retry(*args, **options)
 
+    # wrap the handle_bucket() method in a try/except to allow us to
+    # retry if there's an httplib error.
+    def handle_bucket_retry(self, *args, **options):
         retries = 3
         while retries > 0:
             try:
-                bucket_keys = self.bucket.list_versions(options['prefix'], delimiter='/')
-                retries = -1
+                self.handle_bucket(*args, **options)
             except httplib.BadStatusLine as e:
-                logging.error("httplib error in delete_keys():  %" % e)
+                logging.error("httplib error handle_bucket():  %" % e)
                 retries -= 1
                 sleep(30)
 
+    def handle_bucket(self, *args, **options):
+        logging.info( "--> %s" % options['prefix'])
+
+        bucket_keys = self.bucket.list_versions(options['prefix'], delimiter='/')
+
         objs = {}
         delete_list = []
         subdirs = []
@@ -204,4 +200,4 @@ class Command(BaseCommand):
         for s in subdirs:
             new_opts = options
             new_opts['prefix'] = s
-            self.handle_bucket(*args, **new_opts)
+            self.handle_bucket_retry(*args, **new_opts)