aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKelley Spoon <kelley.spoon@linaro.org>2020-05-07 02:27:55 -0500
committerKelley Spoon <kelley.spoon@linaro.org>2020-05-14 12:25:04 +0000
commit8fe67d54224c04f5c522cc22a5e8f89e4420ef96 (patch)
tree393356ab73d73668e3ed94b22dd5a40844a9b885
parent76b6c1df753e776d9e3813f2ea8ed8d1295ff150 (diff)
downloadlinaro-license-protection-8fe67d54224c04f5c522cc22a5e8f89e4420ef96.tar.gz
s3_flatten.py: add retries for boto3 bucket operations
It turns out that this script can occasionally error out if the AWS API doesn't respond with proper HTTP. When this happens, the error isn't caught and the script ends prematurely and can leave large amounts of unnecessary data in the bucket. This wraps S3 bucket operations (specifically delete_keys and list_objectS) in a try statement that will attempt up to 3 retries after sleeping for 30 seconds. Change-Id: I3f4c945937b20635166cad093630bfbca9026a78 Reviewed-on: https://review.linaro.org/c/infrastructure/linaro-license-protection/+/35063 Reviewed-by: Benjamin Copeland <ben.copeland@linaro.org>
-rw-r--r--license_protected_downloads/management/commands/s3_flatten.py21
1 files changed, 19 insertions, 2 deletions
diff --git a/license_protected_downloads/management/commands/s3_flatten.py b/license_protected_downloads/management/commands/s3_flatten.py
index 128b339..9e7651b 100644
--- a/license_protected_downloads/management/commands/s3_flatten.py
+++ b/license_protected_downloads/management/commands/s3_flatten.py
@@ -7,6 +7,7 @@ from fnmatch import fnmatch
from boto.s3.connection import S3Connection
from boto.s3 import deletemarker,key,prefix
import sys
+import httplib
logging.getLogger().setLevel(logging.INFO)
@@ -62,7 +63,15 @@ class Command(BaseCommand):
logging.info("deleting: %s" % (x))
if not dryrun:
- bucket.delete_keys(delete_list)
+ retries = 3
+ while retries > 0:
+ try:
+ bucket.delete_keys(delete_list)
+ retries = -1
+ except httplib.BadStatusLine as e:
+ logging.error("httplib error in delete_keys(): %" % e)
+ retries -= 1
+ sleep(30)
else:
logging.info( "DRYRUN: delete_keys for %s keys" % len(delete_list) )
@@ -78,7 +87,15 @@ class Command(BaseCommand):
def handle_bucket(self, *args, **options):
logging.info( "--> %s" % options['prefix'])
- bucket_keys = self.bucket.list_versions(options['prefix'], delimiter='/')
+ retries = 3
+ while retries > 0:
+ try:
+ bucket_keys = self.bucket.list_versions(options['prefix'], delimiter='/')
+ retries = -1
+ except httplib.BadStatusLine as e:
+ logging.error("httplib error in delete_keys(): %" % e)
+ retries -= 1
+ sleep(30)
objs = {}
delete_list = []