S3: Script to delete files older than x days

This is a script to delete any prefix in S3 when the files are older
than X days.

Change-Id: I23b9ebcfc52a7107a0a6a9f8e3aa5674231d81cb
Reviewed-on: https://review.linaro.org/15072
Reviewed-by: Andy Doan <andy.doan@linaro.org>

1 files changed, 51 insertions, 0 deletions

diff --git a/license_protected_downloads/management/commands/s3_purge.py b/license_protected_downloads/management/commands/s3_purge.py
new file mode 100644
index 0000000..086580a
--- /dev/null
+++ b/license_protected_downloads/management/commands/s3_purge.py
@@ -0,0 +1,51 @@
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+import logging
+import datetime
+from boto.s3.connection import S3Connection
+
+logging.getLogger().setLevel(logging.INFO)
+EXCLUDE_FILES = (
+    'releases/',
+    '96boards/',
+    'snapshots/components/toolchain/infrastructure',
+    'snapshots/components/toolchain/gcc-linaro'
+)
+
+
+class Command(BaseCommand):
+
+    help = 'Delete any prefix in S3 when the files are older than X days'
+
+    @staticmethod
+    def add_arguments(parser):
+        parser.add_argument('--dryrun', action='store_true',
+                            help='Do not perform any actions, just report')
+        parser.add_argument('--days', default=90,
+                            help='Number of days to delete files')
+        parser.add_argument('--prefix', default='snapshots/',
+                            help='Custom prefix path')
+    @staticmethod
+    def x_days_ago(days):
+        date = datetime.datetime.now() - datetime.timedelta(days=days)
+        return date.isoformat()
+
+    def handle(self, *args, **options):
+        conn = S3Connection(settings.AWS_ACCESS_KEY_ID,
+                            settings.AWS_SECRET_ACCESS_KEY)
+        bucket = conn.get_bucket(settings.S3_BUCKET, validate=False)
+        bucket_key = bucket.list(options['prefix'])
+
+        for key in bucket_key:
+            if not any(map(key.name.startswith, EXCLUDE_FILES)):
+                if key.last_modified < self.x_days_ago(int(options['days'])):
+                    if options['dryrun']:
+                        logging.info('Will delete %s', key.name)
+                    else:
+                        try:
+                            logging.debug('Deleted %s', key.name)
+                            bucket.delete_key(key)
+                        except Exception:
+                            logging.exception('S3Connection error for %s',
+                                              key.name)