tests/cli/testllvmprojects.py

"""Command line interface tests for llvmprojs.py

Note that although this uses the unittest framework, it does *not* contain unit
tests.

"""

import shutil
import os
import subprocess
import unittest

from tempfile import mkdtemp
from uuid import uuid4

from linaropy.cd import cd


# TODO: move this somewhere more public (maybe linaropy?)
def debug(test):
    """
    Decorator that dumps the output of any subprocess.CalledProcessError
    exception. Use this to decorate a test function when you can't tell what the
    problem is.
    """
    def wrapper(*args, **kwargs):
        # Catch any exceptions so we can dump all the output
        try:
            test(*args, **kwargs)
        except subprocess.CalledProcessError as exc:
            print("Error in {}:".format(test.__name__))
            print("Command {} exited with error code {}:\n{}".format(
                exc.cmd, exc.returncode, exc.output))
    return wrapper


class Testllvmprojs(unittest.TestCase):
    python = "python3"
    script = os.path.join("scripts", "llvm.py")

    @classmethod
    def __create_dummy_commit(cls):
        filename = "filethatshouldntexist"
        cls.run_quietly(["touch", filename])
        cls.run_quietly(["git", "add", filename])
        cls.run_quietly(["git", "commit", "-m", "Dummy commit"])

    @classmethod
    def __create_dummy_repo(cls, repopath):
        if not os.path.isdir(repopath):
            os.makedirs(repopath)

        with cd(repopath):
            cls.run_quietly(["git", "init"])
            cls.__create_dummy_commit()

    @classmethod
    def __add_worktree(cls, repopath, worktreepath, branch):
        with cd(repopath):
            cls.run_quietly(["git", "worktree", "add", worktreepath,
                             "-b", branch])

    @classmethod
    def __get_subproj_repo(cls, subproj):
        return os.path.join(cls.repos, subproj)

    @staticmethod
    def run_with_output(*args, **kwargs):
        """Helper for running a command and capturing stdout and stderr"""
        kwargs["stderr"] = subprocess.STDOUT
        return str(subprocess.check_output(*args, **kwargs), 'utf-8')

    @staticmethod
    def run_quietly(*args, **kwargs):
        """
        Helper for running a command and ignoring stdout and stderr. Exceptions
        are still thrown if something goes wrong
        """
        kwargs["stdout"] = subprocess.DEVNULL
        kwargs["stderr"] = subprocess.DEVNULL
        return subprocess.check_call(*args, **kwargs)

    @classmethod
    def llvm_projects(cls, *args, **kwargs):
        """
        Build a list representing a llvm projects subcommand with the given
        args. Unless otherwise specified in kwargs, this uses the values for
        repos and env that it finds in cls.
        """
        command = [cls.python, cls.script]

        repos = cls.repos
        if "repos" in kwargs:
            repos = kwargs["repos"]
        if repos:
            command.append("--repos")
            command.append(repos)

        env = cls.env
        if "env" in kwargs:
            env = kwargs["env"]
        if env:
            command.append("--env")
            command.append(env)

        command.append("projects")

        if len(args):
            command.extend(args)

        return command

    @classmethod
    def setUpClass(cls):
        """Create the file structure and environment that llvmprojs expects"""
        cls.llvm_root = mkdtemp()
        cls.repos = os.path.join(cls.llvm_root, "repos")

        cls.all_repos = ("llvm", "clang", "compiler-rt", "lld", "lldb",
                         "libcxx", "libcxxabi", "libunwind", "test-suite")

        # Create dummy repos
        for reponame in cls.all_repos:
            cls.__create_dummy_repo(cls.__get_subproj_repo(reponame))

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.llvm_root)

    @classmethod
    def setUp(cls):
        cls.env = os.path.join(cls.llvm_root, "env" + str(uuid4()))
        cls.llvm_src = os.path.join(cls.env, "llvm")

        # Create LLVM worktree
        cls.branch = "br" + str(uuid4())
        cls.__add_worktree(cls.__get_subproj_repo("llvm"), cls.llvm_src,
                           cls.branch)

    @classmethod
    def tearDown(cls):
        # Clean up the directories where we might have added subprojects.
        # This isn't 100% clean, because we don't clean up the repos between
        # tests (so any branches will remain), but it's good enough for the
        # current tests.
        for subprojdir in (os.path.join(cls.llvm_src, "projects"),
                           os.path.join(cls.llvm_src, "tools")):
            if os.path.isdir(subprojdir):
                shutil.rmtree(subprojdir)
                os.makedirs(subprojdir)

        # Run prune on the original repos, to remove any dangling worktrees.
        for reponame in cls.all_repos:
            repopath = cls.__get_subproj_repo(reponame)
            with cd(repopath):
                cls.run_quietly(["git", "worktree", "prune"])

    def test_repos_arg_is_compulsory(self):
        """
        Test that we must pass in the repos for various combinations of input
        args.
        """
        with self.assertRaises(subprocess.CalledProcessError) as context:
            self.run_with_output(self.llvm_projects(repos=None))

        self.assertRegex(
            str(context.exception.output),
            "(.*\n)*.*the following arguments are required:(.*)--repos(.*\n)*")

        with self.assertRaises(subprocess.CalledProcessError) as context:
            self.run_with_output(
                self.llvm_projects(
                    "--add", "clang", repos=None))

        self.assertRegex(
            str(context.exception.output),
            "(.*\n)*.*the following arguments are required:(.*)--repos(.*\n)*")

        with self.assertRaises(subprocess.CalledProcessError) as context:
            self.run_with_output(
                self.llvm_projects(
                    "--remove",
                    "clang",
                    repos=None))

        self.assertRegex(
            str(context.exception.output),
            "(.*\n)*.*the following arguments are required:(.*)--repos(.*\n)*")

    def test_env_arg_is_compulsory(self):
        """
        Test that we must pass in the environment for various combinations of
        input args.
        """
        with self.assertRaises(subprocess.CalledProcessError) as context:
            self.run_with_output(self.llvm_projects(env=None))

        self.assertRegex(
            str(context.exception.output),
            "(.*\n)*.*the following arguments are required:(.*)--env(.*\n)*")

        with self.assertRaises(subprocess.CalledProcessError) as context:
            self.run_with_output(
                self.llvm_projects(
                    "--add", "clang", env=None))

        self.assertRegex(
            str(context.exception.output),
            "(.*\n)*.*the following arguments are required:(.*)--env(.*\n)*")

        with self.assertRaises(subprocess.CalledProcessError) as context:
            self.run_with_output(
                self.llvm_projects(
                    "--remove", "clang", env=None))

        self.assertRegex(
            str(context.exception.output),
            "(.*\n)*.*the following arguments are required:(.*)--env(.*\n)*")

    def test_dump_empty_config(self):
        """
        Test that we're correctly dumping an empty configuration (i.e. no
        projects linked) when running llvmprojs without arguments.
        """
        output = self.run_with_output(self.llvm_projects())
        self.assertRegex(output, "Projects linked:.*\n.*none.*")

    def test_add_remove_subprojects(self):
        """
        Test that we can add and remove one or several subprojects.
        """
        output = self.run_with_output(self.llvm_projects("--add", "clang"))
        self.assertRegex(output, "Projects linked:.*\n.*clang.*")

        output = self.run_with_output(
            self.llvm_projects(
                "--add", "libcxx", "lldb"))
        self.assertRegex(
            output,
            "Projects linked:.*\n" +
            ".*clang.*\n" +
            ".*libcxx.*\n" +
            ".*lldb.*")

        output = self.run_with_output(self.llvm_projects("--remove", "libcxx"))
        self.assertRegex(output,
                         "Projects linked:.*\n" +
                         ".*clang.*\n" +
                         ".*lldb.*")

        output = self.run_with_output(
            self.llvm_projects(
                "--remove", "clang", "lldb"))
        self.assertRegex(output,
                         "Projects linked:.*\n" +
                         ".*none.*")

    def test_add_remove_invalid_subprojects(self):
        """
        Test that we error out nicely when trying to add/remove invalid
        subprojects.
        """
        with self.assertRaises(subprocess.CalledProcessError) as context:
            self.run_with_output(
                self.llvm_projects(
                    "--add", "inventedsubproject"))

        self.assertRegex(
            str(context.exception.output),
            "(.*\n)*.*invalid choice:.*inventedsubproject(.*\n)*")

        with self.assertRaises(subprocess.CalledProcessError) as context:
            self.run_with_output(
                self.llvm_projects(
                    "--remove",
                    "inventedsubproject"))

        self.assertRegex(
            str(context.exception.output),
            "(.*\n)*.*invalid choice:.*inventedsubproject(.*\n)*")

    def test_duplicate_add_remove(self):
        """
        Test that we don't crash when trying to add / remove the same subproject
        twice with the same command.
        """
        output = self.run_with_output(
            self.llvm_projects(
                "--add", "clang", "lld", "clang"))
        self.assertRegex(output,
                         "Projects linked:.*\n" +
                         ".*clang.*\n" +
                         ".*lld.*")

        output = self.run_with_output(
            self.llvm_projects(
                "--remove", "lld", "lld", "clang"))
        self.assertRegex(output,
                         "Projects linked:.*\n" +
                         ".*none.*")

    def test_redundant_add_remove(self):
        """
        Test that we can add a subproject that already exists (either because it
        was added by our script or manually) or remove a subproject that doesn't
        exist.
        """
        self.__add_worktree(self.__get_subproj_repo("clang"),
                            os.path.join(self.llvm_src, "tools", "clang"),
                            self.branch)
        self.__add_worktree(
            self.__get_subproj_repo("compiler-rt"),
            os.path.join(self.llvm_src, "projects", "compiler-rt"),
            self.branch)

        output = self.run_with_output(self.llvm_projects("--add", "clang"))
        self.assertRegex(output,
                         "Projects linked:.*\n" +
                         ".*clang.*")

        output = self.run_with_output(
            self.llvm_projects(
                "--add", "compiler-rt", "lld"))
        self.assertRegex(
            output,
            "Projects linked:.*\n" +
            ".*clang.*\n" +
            ".*compiler-rt.*\n" +
            ".*lld.*")

        output = self.run_with_output(
            self.llvm_projects(
                "--remove", "lldb", "libcxx", "lld"))
        self.assertRegex(
            output,
            "Projects linked:.*\n" +
            ".*clang.*\n" +
            ".*compiler-rt.*")

    def test_simultaneous_add_remove(self):
        """
        Test that we error out when someone is trying to add and remove the same
        project with the same command.
        """
        # Try the really basic case and make sure we're not touching anything
        with self.assertRaises(subprocess.CalledProcessError) as context:
            self.run_with_output(
                self.llvm_projects(
                    "--add",
                    "clang",
                    "--remove",
                    "clang"))

        self.assertRegex(
            str(context.exception.output),
            "(.*\n)*.*Can't add and remove clang at the same time(.*\n)*")

        self.assertFalse(
            os.path.exists(
                os.path.join(self.llvm_src, "tools", "clang")))

        # Try something a bit more complicated and make sure we're not touching
        # anything
        self.__add_worktree(
            self.__get_subproj_repo("lld"),
            os.path.join(self.llvm_src, "tools", "lld"),
            self.branch)

        with self.assertRaises(subprocess.CalledProcessError) as context:
            self.run_with_output(
                self.llvm_projects(
                    "--add",
                    "clang",
                    "lld",
                    "libcxx",
                    "--remove",
                    "lld",
                    "libcxx"))
        self.assertRegex(
            str(context.exception.output),
            "(.*\n)*" +
            ".*Can't add and remove (lld|libcxx) at the same time(.*\n)*")

        # Make sure we didn't touch anything
        self.assertFalse(
            os.path.exists(
                os.path.join(self.llvm_src, "tools", "clang")))
        self.assertTrue(
            os.path.exists(
                os.path.join(self.llvm_src, "tools", "lld")))
        self.assertFalse(
            os.path.exists(
                os.path.join(self.llvm_src, "projects", "libcxx")))

    def test_multiple_adds_removes(self):
        """
        Test that we can have multiple --add and --remove options in the same
        command and that only the last one of each kind matters.
        """
        output = self.run_with_output(
            self.llvm_projects(
                "--add",
                "libcxxabi",
                "--remove",
                "lld",
                "lldb",
                "--add",
                "clang",
                "libcxx",
                "--remove",
                "libunwind"))
        self.assertRegex(output,
                         "Projects linked:.*\n" +
                         ".*clang.*\n" +
                         ".*libcxx.*\n")

        output = self.run_with_output(
            self.llvm_projects(
                "--add",
                "libunwind",
                "libcxxabi",
                "--remove",
                "clang",
                "libcxx",
                "--add",
                "compiler-rt",
                "--remove",
                "libcxxabi"))
        self.assertRegex(output,
                         "Projects linked:.*\n" +
                         ".*clang.*\n" +
                         ".*compiler-rt.*\n" +
                         ".*libcxx.*\n")

        output = self.run_with_output(
            self.llvm_projects(
                "--add",
                "libcxx",
                "--remove",
                "lld",
                "--add",
                "lld",
                "--remove",
                "libcxx"))
        self.assertRegex(output,
                         "Projects linked:.*\n" +
                         ".*clang.*\n" +
                         ".*compiler-rt.*\n" +
                         ".*lld.*\n")

    def test_different_env(self):
        """
        Test that we can have different environments in completely different
        paths and they don't interfere when we try to add/remove projects.
        """
        # Create a separate environment
        new_env = mkdtemp()
        new_branch = "br" + str(uuid4())
        self.__add_worktree(self.__get_subproj_repo("llvm"),
                            os.path.join(new_env, "llvm"), new_branch)

        # Check that we start with a clean slate in both the new environment and
        # the one that's already set up
        output = self.run_with_output(self.llvm_projects())
        self.assertRegex(output, "Projects linked:.*\n.*none.*")

        output = self.run_with_output(self.llvm_projects(env=new_env))
        self.assertRegex(output, "Projects linked:.*\n.*none.*")

        # Make sure that adding projects works
        output = self.run_with_output(
            self.llvm_projects(
                "--add", "clang", "lld"))
        self.assertRegex(output, "Projects linked:.*\n.*clang.*\n.*lld.*\n")

        output = self.run_with_output(
            self.llvm_projects(
                "--add",
                "libcxx",
                "libcxxabi",
                env=new_env))
        self.assertRegex(output,
                         "Projects linked:.*\n.*libcxx.*\n.*libcxxabi.*\n.*")

        output = self.run_with_output(self.llvm_projects())
        self.assertRegex(output, "Projects linked:.*\n.*clang.*\n.*lld.*\n")

        # Make sure that removing projects works
        output = self.run_with_output(self.llvm_projects("--remove", "lld"))
        self.assertRegex(output, "Projects linked:.*\n.*clang.*\n")

        output = self.run_with_output(self.llvm_projects("--remove", "libcxx",
                                                         env=new_env))
        self.assertRegex(output,
                         "Projects linked:.*\n.*libcxxabi.*\n.*")

        output = self.run_with_output(self.llvm_projects())
        self.assertRegex(output, "Projects linked:.*\n.*clang.*\n")

        shutil.rmtree(new_env)