Add support for building LLVM

Add a llvm.py subcommand for building LLVM in a given build directory.
The build directory must have already been configured with CMake. The
build command that is run will be either a 'ninja' command or a 'make'
command, depending on whether the build directory contains a
'build.ninja' or a 'Makefile'. If it contains neither of those, an error
is generated.

I initially wanted to put the implementation for this in the
LLVMBuildConfig, but it doesn't really need anything from there. For now
I left it as a free-standing function. It doesn't have any knowledge of
LLVM whatsoever, so an argument could be made for moving it into the
"utils" package rather than the "llvm" package. In the future, we may
want to remove the LLVMBuildConfig class and see if there's a better way
to organize the code for configuring and building.

One disadvantage with using python here is that we lose the nice
progress bars that we could see when running 'ninja check' from the bash
scripts. I'm not sure how to fix that, suggestions welcome.

Change-Id: I48cf464a6412238a26eb5bcfb4723946983c86f2

5 files changed, 215 insertions, 14 deletions

diff --git a/helpers/llvm-build b/helpers/llvm-build
index 64c15ea..773c16a 100755
--- a/helpers/llvm-build
+++ b/helpers/llvm-build
@@ -1,10 +1,6 @@
 #!/usr/bin/env bash
 
-# This script helps you build LLVM. It can update the repositories first,
-# run the check-all after and even install it in your system. Since the
-# LLVM tree changes constantly, it's recommended that you run this script
-# with update+check at least once a week, and most importantly, before you
-# begin a big change, to make sure the repos are current and stable.
+# This script helps you build LLVM.
 
 . llvm-common
 
@@ -24,10 +20,6 @@ shared=
 targets=
 prog=`basename $0`
 syntax="Syntax: $prog [-jN] [targets]"
-update=false
-check=false
-master=false
-inst=false
 minimal_targets="--cmake-def LLVM_TARGETS_TO_BUILD='ARM;X86;AArch64'"
 link_jobs=
 
@@ -37,10 +29,8 @@ if [ "$1" = "-h" ]; then
 fi
 
 ## Choose between make and ninja
-make=make
 generator="Unix Makefiles"
 if which ninja > /dev/null 2>&1; then
-  make=ninja
   generator="Ninja"
 fi
 
@@ -54,7 +44,7 @@ fi
 # Building on ARM?
 if grep -q "ARM.* Processor" /proc/cpuinfo; then
   targets=$minimal_targets
-  if [ "$make" = "ninja" ]; then
+  if [ "$generator" = "Ninja" ]; then
     link=$(free -g | awk '/Mem/ {print $2}')
     if [ "$link" -gt "$CPUS" ]; then
       link=$CPUS
@@ -95,11 +85,12 @@ fi
 ## Build
 if [ "$1" == "" ]; then
   echo " + Building LLVM"
-  safe_run $make $PARALLEL
+  safe_run python3 $llvmtool build --build-dir $build_dir --build-flag=$PARALLEL
 else
   for target in "$@"; do
     echo " + Running $target"
-    safe_run $make $PARALLEL $target
+    safe_run python3 $llvmtool build --build-dir $build_dir \
+                                     --build-flag=$PARALLEL --build-flag $target
   done
 fi
 
diff --git a/modules/llvm.py b/modules/llvm.py
index 76b317e..0d453ff 100644
--- a/modules/llvm.py
+++ b/modules/llvm.py
@@ -324,3 +324,40 @@ class LLVMBuildConfig(object):
                                                    cmakeSubprojFlags))
 
         return cmakeSubprojFlags
+
+
+def build_llvm(commandConsumer, buildDir, flags=[]):
+    """
+    Generate a build command and pass it to the 'commandConsumer'.
+
+    The build command that is generated will depend on what 'buildDir' contains.
+    If it contains a 'build.ninja', a 'ninja' command will be generated.
+    Otherwise, if it contains a 'Makefile', a 'make' command will be generated.
+    Otherwise, a RuntimeError will be thrown.
+
+    The 'commandConsumer' should have a 'consume' method taking two parameters:
+    the command to be consumed (in the form of a list) and the directory where
+    the command should be run. Any exceptions that may be raised by that method
+    should be handled by the calling code.
+
+    The 'buildDir' needs to be the path to an already-configured build
+    directory.
+
+    The 'flags' should be a list of flags to be passed to the build command,
+    e.g. ["-j8", "llvm-check-codegen-aarch64"].
+    """
+    useNinja = os.path.exists(os.path.join(buildDir, 'build.ninja'))
+    useMake = os.path.exists(os.path.join(buildDir, 'Makefile'))
+
+    if useNinja:
+        buildTool = "ninja"
+    elif useMake:
+        buildTool = "make"
+    else:
+        raise RuntimeError(
+            "Couldn't identify build system to use for {}. "
+            "It does not contain 'build.ninja' or 'Makefile'. "
+            "Are you sure it was configured?".format(buildDir))
+
+    command = [buildTool] + flags
+    commandConsumer.consume(command, buildDir)
diff --git a/scripts/llvm.py b/scripts/llvm.py
index d1bf29d..ff8ecb1 100644
--- a/scripts/llvm.py
+++ b/scripts/llvm.py
@@ -4,6 +4,7 @@ import os
 from sys import argv
 from sys import exit
 
+from modules.llvm import build_llvm
 from modules.llvm import LLVMBuildConfig
 from modules.llvm import LLVMSubproject
 from modules.llvm import LLVMSourceConfig
@@ -116,6 +117,21 @@ def configure_build(args):
         die("Failed to configure {} because:\n{}".format(args.build, str(exc)))
 
 
+def run_build(args):
+    """Run a build command in a given directory."""
+    build_dir = args.build
+
+    if args.dry:
+        consumer = CommandPrinter()
+    else:
+        consumer = CommandRunner()
+
+    try:
+        build_llvm(consumer, args.build, args.flags)
+    except RuntimeError as exc:
+        die("Failed to build {} because:\n{}".format(args.build, str(exc)))
+
+
 ##########################################################################
 # Command line parsing                                                   #
 ##########################################################################
@@ -215,6 +231,36 @@ configure.add_argument(
     help="Print the CMake command instead of executing it.")
 configure.set_defaults(run_command=configure_build)
 
+# Subcommand for building a target
+build = subcommands.add_parser(
+    'build',
+    help="Run a build command in the given directory."
+    "The build command can be either a 'ninja' or a 'make' command, depending "
+    "on what the build directory contains. First, we look for a 'build.ninja' "
+    "file. If that is not found, we look for a 'Makefile'. If that is not "
+    "found either, the script fails.")
+build.add_argument(
+    '--build-dir',
+    dest='build',
+    required=True,
+    help="Path to the build directory. It must have already been configured.")
+build.add_argument(
+    '-n', '--dry-run',
+    dest='dry',
+    action='store_true',
+    default=False,
+    help="Print the build command instead of executing it.")
+build.add_argument(
+    '--build-flag',
+    dest='flags',
+    metavar='FLAG',
+    default=[],
+    action='append',
+    help="Additional flags for the build command (e.g. targets to build). "
+    "May be passed several times. If your flag starts with a '-', use "
+    "'--build-flag=-FLAG' to pass it.")
+build.set_defaults(run_command=run_build)
+
 args = options.parse_args()
 if args.subcommand == "projects" and args.add and not args.repos:
     projs.error(
diff --git a/tests/cli/testllvmbuild.py b/tests/cli/testllvmbuild.py
new file mode 100644
index 0000000..a395d0f
--- /dev/null
+++ b/tests/cli/testllvmbuild.py
@@ -0,0 +1,61 @@
+"""Command line interface tests for llvm.py build.
+
+Note that although this uses the unittest framework, it does *not* contain unit
+tests.
+
+"""
+import os
+
+from shutil import rmtree
+from subprocess import CalledProcessError
+from tempfile import mkdtemp
+
+from llvmtestcase import LLVMTestCase, debug
+
+
+def create_empty_file(path):
+    open(path, "wt").close()
+
+
+class Testllvmbuild(LLVMTestCase):
+
+    @classmethod
+    def llvm_build(cls, *args, **kwargs):
+        return cls.command_with_defaults("build", *args, **kwargs)
+
+    def setUp(self):
+        self.buildDir = mkdtemp()
+
+    def tearDown(self):
+        rmtree(self.buildDir)
+
+    def test_build_dir_is_compulsory(self):
+        """Test that we get an error if we don't pass the build dir."""
+        with self.assertRaises(CalledProcessError) as context:
+            self.run_with_output(self.llvm_build())
+
+        self.assertRegex(
+            str(context.exception.output),
+            "(.*\n)*the following arguments are required: --build-dir(.*\n)*")
+
+    def test_dry_run(self):
+        """
+        Test that we at least dump the expected command. It's difficult to check
+        whether we would actually run it correctly without depending too much on
+        external factors.
+        """
+        create_empty_file(os.path.join(self.buildDir, "build.ninja"))
+
+        output = self.run_with_output(
+            self.llvm_build(
+                "--dry-run",
+                "--build-dir",
+                self.buildDir,
+                "--build-flag=-j8",
+                "--build-flag",
+                "llc",
+                "--build-flag",
+                "llvm-mc"))
+
+        self.assertEqual(output,
+                         "{}$ ninja -j8 llc llvm-mc\n".format(self.buildDir))
diff --git a/tests/unittests/testbuildllvm.py b/tests/unittests/testbuildllvm.py
new file mode 100644
index 0000000..5a31eb3
--- /dev/null
+++ b/tests/unittests/testbuildllvm.py
@@ -0,0 +1,66 @@
+from modules.llvm import build_llvm
+
+import os
+
+from shutil import rmtree
+from tempfile import mkdtemp
+
+from unittest import TestCase
+from unittest.mock import MagicMock
+
+
+def create_empty_dir():
+    return mkdtemp()
+
+
+def create_dir_with_empty_file(filename):
+    dir = create_empty_dir()
+    open(os.path.join(dir, filename), "wt").close()
+    return dir
+
+
+class TestBuildLLVM(TestCase):
+
+    def tearDown(self):
+        rmtree(self.buildDir)
+
+    def test_invalid_build_dir(self):
+        self.buildDir = create_empty_dir()
+
+        with self.assertRaises(RuntimeError) as context:
+            build_llvm(None, self.buildDir)
+
+        self.assertRegex(
+            str(context.exception),
+            "(.*\n)*Couldn't identify build system to use for {}(.*\n)*".format(
+                self.buildDir))
+
+    def test_ninja_build_dir(self):
+        self.buildDir = create_dir_with_empty_file("build.ninja")
+        consumer = MagicMock()
+
+        build_llvm(consumer, self.buildDir)
+        command, directory = consumer.consume.call_args[0]
+
+        self.assertEqual(command, ["ninja"])
+        self.assertEqual(directory, self.buildDir)
+
+    def test_make_build_dir(self):
+        self.buildDir = create_dir_with_empty_file("Makefile")
+        consumer = MagicMock()
+
+        build_llvm(consumer, self.buildDir)
+        command, directory = consumer.consume.call_args[0]
+
+        self.assertEqual(command, ["make"])
+        self.assertEqual(directory, self.buildDir)
+
+    def test_flags(self):
+        self.buildDir = create_dir_with_empty_file("build.ninja")
+        consumer = MagicMock()
+
+        build_llvm(consumer, self.buildDir, ["-t", "targets"])
+        command, directory = consumer.consume.call_args[0]
+
+        self.assertEqual(command, ["ninja", "-t", "targets"])
+        self.assertEqual(directory, self.buildDir)