fix s3 urlencoded paths

We have some publishing jobs that are now naming files with ~'s in them.
The problem is that the tilde has to be converted to %7E so that its a
safe url-encoded string for S3 to sign. This leaves us calling the file
2 different things which S3 won't allow when trying to upload to the
signed url.

The safe way to handle this is to store all S3 files with urlencoded
names. We then modify our presentation layer to decode that name so its
listed sanely.

Change-Id: I782de89b25776d55ed03afa882245e0d01c5216b
Reviewed-on: https://review.linaro.org/19091
Reviewed-by: Ben Copeland <ben.copeland@linaro.org>
Reviewed-by: Fathi Boudra <fathi.boudra@linaro.org>

3 files changed, 8 insertions, 3 deletions

diff --git a/license_protected_downloads/api/v3.py b/license_protected_downloads/api/v3.py
index b43f10b..9ef6470 100644
--- a/license_protected_downloads/api/v3.py
+++ b/license_protected_downloads/api/v3.py
@@ -1,4 +1,5 @@
 import os
+import urllib
 
 from django.conf import settings
 from django.http import HttpResponse
@@ -27,7 +28,8 @@ class PublishResource(v2.PublishResource):
         if not b:
             raise HttpResponseError('S3 is not enabled', 403)
 
-        k = b.new_key(settings.S3_PREFIX_PATH + self.path)
+        path = urllib.pathname2url(self.path)
+        k = b.new_key(settings.S3_PREFIX_PATH + path)
         if k.exists():
             APILog.mark(self.request, 'FILE_OVERWRITE_DENIED')
             raise HttpResponseError('File already exists', 403)
diff --git a/license_protected_downloads/artifact/s3.py b/license_protected_downloads/artifact/s3.py
index e7f8282..0ba0cca 100644
--- a/license_protected_downloads/artifact/s3.py
+++ b/license_protected_downloads/artifact/s3.py
@@ -1,6 +1,7 @@
 import datetime
 import mimetypes
 import os
+import urllib
 import time
 
 import boto
@@ -50,6 +51,7 @@ class S3Artifact(Artifact):
                 file_name = ''
             item.size = 0
             item.last_modified = '-'
+        file_name = urllib.url2pathname(file_name)
         self.bucket = bucket
         self.parent = parent
         if parent and hasattr(self.parent, 'children'):
@@ -128,7 +130,7 @@ class S3Artifact(Artifact):
         else:
             key += '/' + os.path.dirname(self.file_name) + fname
         try:
-            key = boto.s3.key.Key(self.bucket, key)
+            key = boto.s3.key.Key(self.bucket, urllib.pathname2url(key))
             return key.get_contents_as_string()
         except boto.exception.S3ResponseError:
             pass  # return None - its okay
diff --git a/license_protected_downloads/common.py b/license_protected_downloads/common.py
index 7a9aaff..fe4722d 100644
--- a/license_protected_downloads/common.py
+++ b/license_protected_downloads/common.py
@@ -1,5 +1,6 @@
 import fnmatch
 import os
+import urllib
 
 import boto
 
@@ -97,7 +98,7 @@ def _find_s3_artifact(request, path):
     if not b:
         return  # s3 isn't configured
 
-    prefix = settings.S3_PREFIX_PATH + path
+    prefix = settings.S3_PREFIX_PATH + urllib.pathname2url(path)
     if prefix[-1] == '/':
         # s3 listing give sub dir, we don't want that
         prefix = prefix[:-1]