patchwork-tools: upgrades to enable python3

This change includes updates to the codebase in order
to allow it to run in python3.  Most of the changes
relate to the str->utf-8 switch in python3 and the retirement
of urllib2 as urllib.parse and urllib.request absorbed its
functionality present in python2.

In order to address some python3 bugs in Django it is upgraded from 1.11
to 1.11.29 (the last LTS release in the 1.11 series).

The python used by the virtualenv for the unit-test.sh script has been
changed to python3.

Change-Id: I3f8016fb405e4d5f0a8a47729b42fdab772588d2

1 files changed, 41 insertions, 35 deletions

diff --git a/gitrepo.py b/gitrepo.py
index 2340d8b..7fed039 100644
--- a/gitrepo.py
+++ b/gitrepo.py
@@ -9,41 +9,42 @@ import dulwich.repo
 from django.conf import settings
 from patchwork.parser import parse_patch
 
-log = logging.getLogger('gitrepo')
+log = logging.getLogger("gitrepo")
 
 
-def croncmd(args, cwd='./', timeout=None, get_fail_logger=None):
+def croncmd(args, cwd="./", timeout=None, get_fail_logger=None):
     if timeout:
-        args = ['timeout', str(timeout)] + args
+        args = ["timeout", str(timeout)] + args
 
     with tempfile.SpooledTemporaryFile(max_size=4096) as f:
         try:
             subprocess.check_call(args, cwd=cwd, stdout=f, stderr=f)
             if log.level == logging.DEBUG:
-                log.debug('Results of cmd(%s): %r', cwd, args)
+                log.debug("Results of cmd(%s): %r", cwd, args)
                 f.seek()
-                log.debug('COMBINED_OUTPUT\n%s', f.read())
+                log.debug("COMBINED_OUTPUT\n%s", f.read())
         except subprocess.CalledProcessError as e:
             logger = log.error
             if get_fail_logger:
                 logger = get_fail_logger()
-            logger('Unable to run command(%s): %r', cwd, args)
+            logger("Unable to run command(%s): %r", cwd, args)
             if timeout and e.returncode == 124:
-                logger('Command timed out')
+                logger("Command timed out")
             f.seek(0)
             if e.output:
-                logger('STDOUT:\n%s', e.output)
-                logger('STDERR:\n%s', f.read())
+                logger("STDOUT:\n%s", e.output)
+                logger("STDERR:\n%s", f.read())
             else:
-                logger('COMBINED OUTPUT:\n%s', f.read())
+                logger("COMBINED OUTPUT:\n%s", f.read())
             raise
 
 
 class Repo(object):
-    '''Our patchwork deployments try and automatically update patches by
+    """Our patchwork deployments try and automatically update patches by
     looking at the change history on a repository. This class provides a
     simple interface to analyze new commits
-    '''
+    """
+
     def __init__(self, repo_dir, name, scm_url):
         self.path = os.path.join(repo_dir, name)
         self.scm_url = scm_url
@@ -59,24 +60,27 @@ class Repo(object):
         return self.repo[key]
 
     def _clone(self):
-        croncmd(['git', 'clone', '--mirror', self.scm_url, self.path])
+        croncmd(["git", "clone", "--mirror", self.scm_url, self.path])
 
     def _pull(self):
-        fail_file = os.path.join(self.path, 'failures')
+        fail_file = os.path.join(self.path, "failures")
 
         def get_fail_logger():
-            with open(fail_file, 'a+') as f:
-                f.write('failed at: %s\n' % str(datetime.datetime.now()))
+            with open(fail_file, "a+") as f:
+                f.write("failed at: %s\n" % str(datetime.datetime.now()))
                 f.seek(0)
                 for count, line in enumerate(f, 1):
                     if count > 3:
                         return log.error
             return log.info
 
-        timeout = str(getattr(settings, 'REPO_TIMEOUT', 120))
+        timeout = str(getattr(settings, "REPO_TIMEOUT", 120))
         croncmd(
-            ['git', 'remote', '-v', 'update', '--prune'], self.path, timeout,
-            get_fail_logger)
+            ["git", "remote", "-v", "update", "--prune"],
+            self.path,
+            timeout,
+            get_fail_logger,
+        )
         if os.path.exists(fail_file):
             # clean out old failures, now that we've succeeded
             os.unlink(fail_file)
@@ -94,47 +98,49 @@ class Repo(object):
                 pass
 
     def process_unchecked_commits(self, save_state=True):
-        last_commit = os.path.join(self.path, 'patchwork-last-commit')
+        last_commit = os.path.join(self.path, "patchwork-last-commit")
         if os.path.exists(last_commit):
-            with open(last_commit) as f:
+            with open(last_commit, "r") as f:
                 start_at = f.read().strip()
         else:
-            start_at = 'HEAD~100'
+            start_at = "HEAD~100"
 
-        log.debug('looking for commits since: %s', start_at)
-        args = ['git', 'rev-list', '--reverse', start_at + '..HEAD']
-        with open('/dev/null', 'w') as f:
+        log.debug("looking for commits since: %s", start_at)
+        args = ["git", "rev-list", "--reverse", start_at + "..HEAD"]
+        with open("/dev/null", "w") as f:
             rc = subprocess.call(
-                ['git', 'show', start_at], cwd=self.path, stdout=f, stderr=f)
+                ["git", "show", start_at], cwd=self.path, stdout=f, stderr=f
+            )
         if rc != 0:
             # we may have had a branch who's history was re-written
             # just try and get changes for past day
-            args = ['git', 'rev-list', '--reverse',
-                    '--since', '1 day ago', 'HEAD']
+            args = ["git", "rev-list", "--reverse", "--since",
+                    "1 day ago", "HEAD"]
 
         try:
-            for x in subprocess.check_output(args, cwd=self.path).split('\n'):
+            for x in subprocess.check_output(
+                    args, cwd=self.path, text=True).split("\n"):
                 if x:
-                    yield self.repo[x]
+                    yield self.repo[x.encode("utf-8")]
                     start_at = x
         finally:
             if save_state:
-                with open(last_commit, 'w') as f:
+                with open(last_commit, "w") as f:
                     f.write(start_at)
 
     def get_patch(self, commit):
-        args = ['git', 'show', '--format=format:%e', '-M', str(commit.id)]
+        args = ["git", "show", "--format=format:%e", "-M", commit.id.decode()]
         patch = subprocess.check_output(args, cwd=self.path)
         # the patchwork parser code operates character by character so we must
         # convert to unicode so it can be handled properly
-        patch = patch.decode('utf-8', errors='replace')
+        patch = patch.decode("utf-8", errors="replace")
         # Don't try and process >5Mb patches, they flood the server
         if len(patch) > 5000000:
-            raise MemoryError('patch too large to process: %d' % len(patch))
+            raise MemoryError("patch too large to process: %d" % len(patch))
         patch = parse_patch(patch)[0]
         if patch is None:
             # happens for binary only patches like:
             # https://git.linaro.org/uefi/OpenPlatformPkg.git/commit/ \
             # ?id=7ab4bb34b2464a2491868264bdf2931f2acd6452
-            patch = ''
+            patch = ""
         return patch