diff options
authorKelley Spoon <kelley.spoon@linaro.org>2020-11-18 06:12:28 -0600
committerKelley Spoon <kelley.spoon@linaro.org>2020-12-01 13:56:03 +0000
commite15d8aa7900f10f39e24bb5ff7c5abd09c1e6f40 (patch)
parent59536fd627f91d881980e19113159c1bfa6ba1cf (diff)
Add timestamps to the report CSV
Currently, our process_report script assumes that a Download occured on the day the process_report script was run. This isn't correct, so we need to keep track of the actual datetime when the download occurred in the CSV as an extra field. This attempts to handle it in a way that is backwards compatible with previous CSV files that don't include a timestamp column. Change-Id: I5249c434a2777f09fb1e2010b812a91255517b07 Reviewed-on: https://review.linaro.org/c/infrastructure/linaro-license-protection/+/36874 Reviewed-by: Kelley Spoon <kelley.spoon@linaro.org>
2 files changed, 18 insertions, 2 deletions
diff --git a/license_protected_downloads/management/commands/report_process.py b/license_protected_downloads/management/commands/report_process.py
index 288a6d0..806f34e 100644
--- a/license_protected_downloads/management/commands/report_process.py
+++ b/license_protected_downloads/management/commands/report_process.py
@@ -9,6 +9,7 @@ import logging
import csv
import fcntl
import sys
+from datetime import datetime
@@ -45,10 +46,20 @@ class Command(BaseCommand):
# This looks odd, but we sometimes get URLs with newlines
# in them and we need the real file name
download = row[1].replace('\n', '')
+ # Check and see if there's a timestamp column..
+ # if not, then we fail back to the old behavior
+ # of assuming that download time is when report was run
+ try:
+ download_timestamp = datetime.strptime(
+ row[4], "%Y-%m-%d %H:%M:%S.%f")
+ except IndexError as e:
+ download_timestamp = datetime.now()
ip=row[0], name=download, link=str2bool(row[2]),
- ref=row[3])
+ ref=row[3], timestamp=download_timestamp)
except DataError:
'Skipping invalid download entry: %s', row)
diff --git a/license_protected_downloads/models.py b/license_protected_downloads/models.py
index 07c8c8b..eeade98 100644
--- a/license_protected_downloads/models.py
+++ b/license_protected_downloads/models.py
@@ -140,6 +140,10 @@ class Download(models.Model):
region_isp = models.CharField(max_length=256, blank=True, null=True)
ref = models.CharField(max_length=4096, blank=True, null=True)
+ # this just notes the download request in a giant CSV file.
+ # We then run a report script via cron in order to fill out
+ # the missing details, such as geo ip and all that fun stuff.
def mark(request, artifact):
@@ -157,9 +161,10 @@ class Download(models.Model):
link = name != artifact.url()
http_ref = request.META.get('HTTP_REFERER', '/')
ref = http_ref.replace('\n', '').replace(',', '')
+ timestamp = datetime.datetime.now()
with open(settings.REPORT_CSV, "a") as f:
writer = csv.writer(f)
- writer.writerow([ip, name, link, ref[:4096]])
+ writer.writerow([ip, name, link, ref[:4096], timestamp])
logging.exception('unable to mark download')