Move to Python3

Most of the changes have been made by 2to3. Some manual fiddling was
needed to make sure we're calling python3 instead of just python in the
tests. Two of the helpers in the tests were reworked into proper methods
rather than partials because we had to convert their returned value to
str and this seems like the most straightforward/easy-to-read way.

Change-Id: I74fdc1eaade3c026ee0c3e6bcdf26f8840f259b3

5 files changed, 114 insertions, 109 deletions

diff --git a/check.sh b/check.sh
index f5b65fa..1f4ac44 100755
--- a/check.sh
+++ b/check.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 echo "Running unit tests"
-python -m unittest discover -s tests/unittests
+python3 -m unittest discover -s tests/unittests
 
 echo "Running cli tests"
-python -m unittest discover -s tests/cli
+python3 -m unittest discover -s tests/cli
diff --git a/modules/llvm.py b/modules/llvm.py
index 8922c83..34cd02c 100644
--- a/modules/llvm.py
+++ b/modules/llvm.py
@@ -135,12 +135,12 @@ class LLVMSourceConfig(object):
         if remove is None:
             remove = []
 
-        for subproj in add.keys():
+        for subproj in list(add.keys()):
             if subproj in remove:
                 raise ValueError("Can't add and remove {} at the same time"
                                  .format(subproj))
 
-        for (subproj, repo) in add.items():
+        for (subproj, repo) in list(add.items()):
             self.__add_subproject(subproj, repo)
 
         for subproj in remove:
@@ -148,7 +148,7 @@ class LLVMSourceConfig(object):
 
     def __get_subproj_object(self, subprojName):
         """Get the LLVMSubproject object corresponding to subprojName."""
-        if not subprojName in self.subprojs.keys():
+        if not subprojName in list(self.subprojs.keys()):
             raise ValueError("Unknown llvm subproject {0}".format(subprojName))
         return self.subprojs[subprojName]
 
@@ -202,7 +202,7 @@ class LLVMSourceConfig(object):
 
         # Create a new worktree
         branch = self.llvmSourceTree.getbranch()
-        if subprojRepo.branchexists(branch):
+        if subprojRepo.branch_exists(branch):
             Worktree.create(self.proj, subprojRepo, path, branch)
         else:
             trackedBranch = "master"  # TODO: track proper branch
diff --git a/scripts/llvm.py b/scripts/llvm.py
index 762f9e7..706af11 100644
--- a/scripts/llvm.py
+++ b/scripts/llvm.py
@@ -12,7 +12,7 @@ from argparse import Action, ArgumentParser, RawTextHelpFormatter
 
 def die(message, config_to_dump=None):
     """Print an error message and exit."""
-    print message
+    print(message)
 
     if config_to_dump is not None:
         dump_config(config_to_dump)
@@ -35,13 +35,13 @@ llvm_worktree_root = os.environ["LLVM_SRC"]
 def dump_config(config):
     """Dump the list of projects that are enabled in the given config."""
 
-    print "Projects linked:"
+    print("Projects linked:")
     enabled = config.get_enabled_subprojects()
     if not enabled:
-        print "none"
+        print("none")
     else:
         for subproj in sorted(enabled):
-            print "  + {}".format(subproj)
+            print("  + {}".format(subproj))
 
 
 def projects(args):
@@ -74,7 +74,7 @@ def projects(args):
 
 # If we decide we want shorthands for the subprojects, we can append to this
 # list
-valid_subprojects = LLVMSubproject.get_all_subprojects().keys()
+valid_subprojects = list(LLVMSubproject.get_all_subprojects().keys())
 
 options = ArgumentParser(formatter_class=RawTextHelpFormatter)
 subcommands = options.add_subparsers()
diff --git a/tests/cli/testllvmprojects.py b/tests/cli/testllvmprojects.py
index c33ddb5..e8bb860 100644
--- a/tests/cli/testllvmprojects.py
+++ b/tests/cli/testllvmprojects.py
@@ -10,7 +10,6 @@ import os
 import subprocess
 import unittest
 
-from functools import partial
 from tempfile import mkdtemp
 from uuid import uuid4
 
@@ -29,13 +28,14 @@ def debug(test):
         try:
             test(*args, **kwargs)
         except subprocess.CalledProcessError as exc:
-            print "Error in {}:".format(test.__name__)
-            print "Command {} exited with error code {}:\n{}".format(
-                exc.cmd, exc.returncode, exc.output)
+            print("Error in {}:".format(test.__name__))
+            print("Command {} exited with error code {}:\n{}".format(
+                exc.cmd, exc.returncode, exc.output))
     return wrapper
 
 
 class Testllvmprojs(unittest.TestCase):
+    python = "python3"
     script = os.path.join("scripts", "llvm.py")
 
     @classmethod
@@ -64,6 +64,22 @@ class Testllvmprojs(unittest.TestCase):
     def __get_subproj_repo(cls, subproj):
         return os.path.join(cls.repos, subproj)
 
+    @staticmethod
+    def run_with_output(*args, **kwargs):
+        """Helper for running a command and capturing stdout and stderr"""
+        kwargs["stderr"] = subprocess.STDOUT
+        return str(subprocess.check_output(*args, **kwargs), 'utf-8')
+
+    @staticmethod
+    def run_quietly(*args, **kwargs):
+        """
+        Helper for running a command and ignoring stdout and stderr. Exceptions
+        are still thrown if something goes wrong
+        """
+        kwargs["stdout"] = subprocess.DEVNULL
+        kwargs["stderr"] = subprocess.DEVNULL
+        return subprocess.check_call(*args, **kwargs)
+
     @classmethod
     def setUpClass(cls):
         """Create the file structure and environment that llvmprojs expects"""
@@ -73,17 +89,6 @@ class Testllvmprojs(unittest.TestCase):
         cls.all_repos = ("llvm", "clang", "compiler-rt", "lld", "lldb",
                          "libcxx", "libcxxabi", "libunwind", "test-suite")
 
-        # Set up helper functions for running commands (this needs to be done
-        # before creating the repos)
-        # FIXME: In newer versions of Python (3.3+) we should be able to bind
-        # the stdout to DEVNULL, as below:
-        # cls.run_quietly = partial(subprocess.call, stdout=subprocess.DEVNULL)
-        # This doesn't work in Python 2, so we'll just have to ignore the output
-        # for now...
-        cls.run_with_output = partial(subprocess.check_output,
-                                      stderr=subprocess.STDOUT)
-        cls.run_quietly = cls.run_with_output
-
         # Create dummy repos
         for reponame in cls.all_repos:
             cls.__create_dummy_repo(cls.__get_subproj_repo(reponame))
@@ -128,38 +133,38 @@ class Testllvmprojs(unittest.TestCase):
         Test that we're correctly dumping an empty configuration (i.e. no
         projects linked) when running llvmprojs without arguments.
         """
-        output = self.run_with_output(["python", self.script, "projects"])
-        self.assertRegexpMatches(output, "Projects linked:.*\n.*none.*")
+        output = self.run_with_output([self.python, self.script, "projects"])
+        self.assertRegex(output, "Projects linked:.*\n.*none.*")
 
     def test_add_remove_subprojects(self):
         """
         Test that we can add and remove one or several subprojects.
         """
-        output = self.run_with_output(["python", self.script, "projects",
+        output = self.run_with_output([self.python, self.script, "projects",
                                        "--add", "clang"])
-        self.assertRegexpMatches(output, "Projects linked:.*\n.*clang.*")
+        self.assertRegex(output, "Projects linked:.*\n.*clang.*")
 
-        output = self.run_with_output(["python", self.script, "projects",
+        output = self.run_with_output([self.python, self.script, "projects",
                                        "--add", "libcxx", "lldb"])
-        self.assertRegexpMatches(
+        self.assertRegex(
             output,
             "Projects linked:.*\n" +
             ".*clang.*\n" +
             ".*libcxx.*\n" +
             ".*lldb.*")
 
-        output = self.run_with_output(["python", self.script, "projects",
+        output = self.run_with_output([self.python, self.script, "projects",
                                        "--remove", "libcxx"])
-        self.assertRegexpMatches(output,
-                                 "Projects linked:.*\n" +
-                                 ".*clang.*\n" +
-                                 ".*lldb.*")
+        self.assertRegex(output,
+                         "Projects linked:.*\n" +
+                         ".*clang.*\n" +
+                         ".*lldb.*")
 
-        output = self.run_with_output(["python", self.script, "projects",
+        output = self.run_with_output([self.python, self.script, "projects",
                                        "--remove", "clang", "lldb"])
-        self.assertRegexpMatches(output,
-                                 "Projects linked:.*\n" +
-                                 ".*none.*")
+        self.assertRegex(output,
+                         "Projects linked:.*\n" +
+                         ".*none.*")
 
     def test_add_remove_invalid_subprojects(self):
         """
@@ -167,19 +172,19 @@ class Testllvmprojs(unittest.TestCase):
         subprojects.
         """
         with self.assertRaises(subprocess.CalledProcessError) as context:
-            self.run_quietly(["python", self.script, "projects",
-                              "--add", "inventedsubproject"])
+            self.run_with_output([self.python, self.script, "projects",
+                                  "--add", "inventedsubproject"])
 
-        self.assertRegexpMatches(
-            context.exception.output,
+        self.assertRegex(
+            str(context.exception.output),
             "(.*\n)*.*invalid choice:.*inventedsubproject(.*\n)*")
 
         with self.assertRaises(subprocess.CalledProcessError) as context:
-            self.run_quietly(["python", self.script, "projects",
-                              "--remove", "inventedsubproject"])
+            self.run_with_output([self.python, self.script, "projects",
+                                  "--remove", "inventedsubproject"])
 
-        self.assertRegexpMatches(
-            context.exception.output,
+        self.assertRegex(
+            str(context.exception.output),
             "(.*\n)*.*invalid choice:.*inventedsubproject(.*\n)*")
 
     def test_duplicate_add_remove(self):
@@ -187,18 +192,18 @@ class Testllvmprojs(unittest.TestCase):
         Test that we don't crash when trying to add / remove the same subproject
         twice with the same command.
         """
-        output = self.run_with_output(["python", self.script, "projects",
+        output = self.run_with_output([self.python, self.script, "projects",
                                        "--add", "clang", "lld", "clang"])
-        self.assertRegexpMatches(output,
-                                 "Projects linked:.*\n" +
-                                 ".*clang.*\n" +
-                                 ".*lld.*")
+        self.assertRegex(output,
+                         "Projects linked:.*\n" +
+                         ".*clang.*\n" +
+                         ".*lld.*")
 
-        output = self.run_with_output(["python", self.script, "projects",
+        output = self.run_with_output([self.python, self.script, "projects",
                                        "--remove", "lld", "lld", "clang"])
-        self.assertRegexpMatches(output,
-                                 "Projects linked:.*\n" +
-                                 ".*none.*")
+        self.assertRegex(output,
+                         "Projects linked:.*\n" +
+                         ".*none.*")
 
     def test_redundant_add_remove(self):
         """
@@ -214,24 +219,24 @@ class Testllvmprojs(unittest.TestCase):
             os.path.join(self.llvm_src, "projects", "compiler-rt"),
             self.branch)
 
-        output = self.run_with_output(["python", self.script, "projects",
+        output = self.run_with_output([self.python, self.script, "projects",
                                        "--add", "clang"])
-        self.assertRegexpMatches(output,
-                                 "Projects linked:.*\n" +
-                                 ".*clang.*")
+        self.assertRegex(output,
+                         "Projects linked:.*\n" +
+                         ".*clang.*")
 
-        output = self.run_with_output(["python", self.script, "projects",
+        output = self.run_with_output([self.python, self.script, "projects",
                                        "--add", "compiler-rt", "lld"])
-        self.assertRegexpMatches(
+        self.assertRegex(
             output,
             "Projects linked:.*\n" +
             ".*clang.*\n" +
             ".*compiler-rt.*\n" +
             ".*lld.*")
 
-        output = self.run_with_output(["python", self.script, "projects",
+        output = self.run_with_output([self.python, self.script, "projects",
                                        "--remove", "lldb", "libcxx", "lld"])
-        self.assertRegexpMatches(
+        self.assertRegex(
             output,
             "Projects linked:.*\n" +
             ".*clang.*\n" +
@@ -244,11 +249,11 @@ class Testllvmprojs(unittest.TestCase):
         """
         # Try the really basic case and make sure we're not touching anything
         with self.assertRaises(subprocess.CalledProcessError) as context:
-            self.run_quietly(["python", self.script, "projects",
-                              "--add", "clang", "--remove", "clang"])
+            self.run_with_output([self.python, self.script, "projects",
+                                  "--add", "clang", "--remove", "clang"])
 
-        self.assertRegexpMatches(
-            context.exception.output,
+        self.assertRegex(
+            str(context.exception.output),
             "(.*\n)*.*Can't add and remove clang at the same time(.*\n)*")
 
         self.assertFalse(
@@ -263,11 +268,11 @@ class Testllvmprojs(unittest.TestCase):
             self.branch)
 
         with self.assertRaises(subprocess.CalledProcessError) as context:
-            self.run_quietly(["python", self.script, "projects",
-                              "--add", "clang", "lld", "libcxx",
-                              "--remove", "lld", "libcxx"])
-        self.assertRegexpMatches(
-            context.exception.output,
+            self.run_with_output([self.python, self.script, "projects",
+                                  "--add", "clang", "lld", "libcxx",
+                                  "--remove", "lld", "libcxx"])
+        self.assertRegex(
+            str(context.exception.output),
             "(.*\n)*" +
             ".*Can't add and remove (lld|libcxx) at the same time(.*\n)*")
 
@@ -287,32 +292,32 @@ class Testllvmprojs(unittest.TestCase):
         Test that we can have multiple --add and --remove options in the same
         command and that only the last one of each kind matters.
         """
-        output = self.run_with_output(["python", self.script, "projects",
+        output = self.run_with_output([self.python, self.script, "projects",
                                        "--add", "libcxxabi",
                                        "--remove", "lld", "lldb",
                                        "--add", "clang", "libcxx",
                                        "--remove", "libunwind"])
-        self.assertRegexpMatches(output,
-                                 "Projects linked:.*\n" +
-                                 ".*clang.*\n" +
-                                 ".*libcxx.*\n")
+        self.assertRegex(output,
+                         "Projects linked:.*\n" +
+                         ".*clang.*\n" +
+                         ".*libcxx.*\n")
 
-        output = self.run_with_output(["python", self.script, "projects",
+        output = self.run_with_output([self.python, self.script, "projects",
                                        "--add", "libunwind", "libcxxabi",
                                        "--remove", "clang", "libcxx",
                                        "--add", "compiler-rt",
                                        "--remove", "libcxxabi"])
-        self.assertRegexpMatches(output,
-                                 "Projects linked:.*\n" +
-                                 ".*clang.*\n" +
-                                 ".*compiler-rt.*\n" +
-                                 ".*libcxx.*\n")
+        self.assertRegex(output,
+                         "Projects linked:.*\n" +
+                         ".*clang.*\n" +
+                         ".*compiler-rt.*\n" +
+                         ".*libcxx.*\n")
 
-        output = self.run_with_output(["python", self.script, "projects",
+        output = self.run_with_output([self.python, self.script, "projects",
                                        "--add", "libcxx", "--remove", "lld",
                                        "--add", "lld", "--remove", "libcxx"])
-        self.assertRegexpMatches(output,
-                                 "Projects linked:.*\n" +
-                                 ".*clang.*\n" +
-                                 ".*compiler-rt.*\n" +
-                                 ".*lld.*\n")
+        self.assertRegex(output,
+                         "Projects linked:.*\n" +
+                         ".*clang.*\n" +
+                         ".*compiler-rt.*\n" +
+                         ".*lld.*\n")
diff --git a/tests/unittests/testllvmsourceconfig.py b/tests/unittests/testllvmsourceconfig.py
index 1add13a..3ad93b1 100644
--- a/tests/unittests/testllvmsourceconfig.py
+++ b/tests/unittests/testllvmsourceconfig.py
@@ -45,7 +45,7 @@ class TestLLVMSourceConfig(unittest.TestCase):
         self.originalLLVM = Clone(self.proj, path)
 
         subprojs = LLVMSubproject.get_all_subprojects()
-        for subproj in subprojs.keys():
+        for subproj in list(subprojs.keys()):
             repo = self.__get_subproj_repo_path(subproj)
             self.__create_dummy_repo(repo)
 
@@ -170,8 +170,8 @@ class TestLLVMSourceConfig(unittest.TestCase):
         with self.assertRaises(ValueError) as context:
             config.update({subproj : Clone(self.proj, subprojPath)})
 
-        self.assertRegexpMatches(str(context.exception),
-                                 "Unknown llvm subproject %s" % subproj)
+        self.assertRegex(str(context.exception),
+                         "Unknown llvm subproject %s" % subproj)
 
     def test_add_each_subproject(self):
         sourcePath = self.temporaryLLVM.repodir
@@ -283,9 +283,9 @@ class TestLLVMSourceConfig(unittest.TestCase):
         with self.assertRaises(EnvironmentError) as context:
             config.update({ "lldb" : self.__get_subproj_repo("lldb")})
 
-        self.assertRegexpMatches(str(context.exception),
-                "{} already exists but is not a valid subproject directory.*"
-                .format(existingPath))
+        self.assertRegex(str(context.exception),
+                         "{} already exists but is not a valid subproject directory.*"
+                         .format(existingPath))
 
         # If we got this far, we're probably ok, but let's be pedantic and check
         # that the subproject is still there
@@ -313,9 +313,9 @@ class TestLLVMSourceConfig(unittest.TestCase):
         with self.assertRaises(EnvironmentError) as context:
             config.update({ "lldb" : self.__get_subproj_repo("lldb")})
 
-        self.assertRegexpMatches(str(context.exception),
-                "{} already exists but is not a valid subproject directory.*"
-                .format(existingPath))
+        self.assertRegex(str(context.exception),
+                         "{} already exists but is not a valid subproject directory.*"
+                         .format(existingPath))
 
         # If we got this far, we're probably ok, but let's be pedantic and check
         # that the subproject is still there
@@ -378,7 +378,7 @@ class TestLLVMSourceConfig(unittest.TestCase):
 
         subprojs = LLVMSubproject.get_all_subprojects()
 
-        for subproj in subprojs.keys():
+        for subproj in list(subprojs.keys()):
             path = subprojs[subproj].get_cmake_path(sourcePath)
             worktree = Worktree.create(
                 self.proj,
@@ -388,9 +388,9 @@ class TestLLVMSourceConfig(unittest.TestCase):
             self.assertTrue(os.path.isdir(path), "Failed to create worktree")
 
         config = LLVMSourceConfig(self.proj, sourcePath)
-        config.update(remove=subprojs.keys())
+        config.update(remove=list(subprojs.keys()))
 
-        for subproj in subprojs.keys():
+        for subproj in list(subprojs.keys()):
             path = subprojs[subproj].get_cmake_path(sourcePath)
             self.assertFalse(
                 os.path.isdir(path),
@@ -401,7 +401,7 @@ class TestLLVMSourceConfig(unittest.TestCase):
 
         subprojs = LLVMSubproject.get_all_subprojects()
 
-        for subproj in subprojs.keys():
+        for subproj in list(subprojs.keys()):
             path = subprojs[subproj].get_cmake_path(sourcePath)
             worktree = Worktree.create(
                 self.proj,
@@ -412,10 +412,10 @@ class TestLLVMSourceConfig(unittest.TestCase):
 
         config = LLVMSourceConfig(self.proj, sourcePath)
 
-        for subproj in subprojs.keys():
+        for subproj in list(subprojs.keys()):
             config.update(remove=[subproj])
 
-        for subproj in subprojs.keys():
+        for subproj in list(subprojs.keys()):
             path = subprojs[subproj].get_cmake_path(sourcePath)
             self.assertFalse(
                 os.path.isdir(path),
@@ -456,8 +456,8 @@ class TestLLVMSourceConfig(unittest.TestCase):
         with self.assertRaises(ValueError) as context:
             config.update(remove=[subproj])
 
-        self.assertRegexpMatches(str(context.exception),
-                                 "Unknown llvm subproject %s" % subproj)
+        self.assertRegex(str(context.exception),
+                         "Unknown llvm subproject %s" % subproj)
 
     def test_remove_inexistent_subproject(self):
         sourcePath = self.temporaryLLVM.repodir