aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDiana Picus <diana.picus@linaro.org>2017-11-24 16:19:41 +0100
committerDiana Picus <diana.picus@linaro.org>2017-12-15 13:08:13 +0100
commit052b7d37cefe8975f699110a5c9508b2501a97e0 (patch)
tree8d78edb8da899bfdfbab43abd6a36db816a3fd4e
parent5fec8b75ddf572779af5dc2a639c8eb15c6bfc9f (diff)
downloadlinaro-scripts-052b7d37cefe8975f699110a5c9508b2501a97e0.tar.gz
Add llvm.py configure
Add a subcommand that runs CMake in a given build directory, with a custom generator and custom CMake definitions. Also update llvm-build to use it instead of calling CMake directly, and add calls to it in llvm-projs as well to make sure we update the build directories whenever we enable or disable a project. One known issue with the current code is that the output of the CMake command is not printed out live, but rather after the command has finished execution. This is going to be more important for llvm.py build than for llvm.py configure, so we can fix it in a future commit. Change-Id: I263b2d47c2083a1778608253bbd437149375c539
-rwxr-xr-xhelpers/llvm-build27
-rwxr-xr-xhelpers/llvm-projs10
-rw-r--r--modules/llvm.py63
-rw-r--r--modules/utils.py29
-rw-r--r--scripts/llvm.py62
-rw-r--r--tests/unittests/testllvmbuildconfig.py89
-rw-r--r--tests/unittests/testllvmsourceconfig.py5
7 files changed, 274 insertions, 11 deletions
diff --git a/helpers/llvm-build b/helpers/llvm-build
index 0556b6e..d40ef40 100755
--- a/helpers/llvm-build
+++ b/helpers/llvm-build
@@ -8,7 +8,11 @@
. llvm-common
+progdir=$(dirname $0)
+llvmtool=$progdir/../scripts/llvm.py
+
safe_run verify_env
+env=$(dirname $LLVM_SRC)
## CMD line options and defaults
CPUS=$(grep -c proc /proc/cpuinfo)
@@ -24,7 +28,7 @@ update=false
check=false
master=false
inst=false
-minimal_targets="-DLLVM_TARGETS_TO_BUILD='ARM;X86;AArch64'"
+minimal_targets="--cmake-def LLVM_TARGETS_TO_BUILD='ARM;X86;AArch64'"
link_jobs=
if [ "$1" = "-h" ]; then
@@ -43,7 +47,7 @@ fi
## Debug mode, make it lighter
if [ "$LLVM_DEBUG" = true ]; then
build_type=Debug
- shared=-DBUILD_SHARED_LIBS=True
+ shared="--cmake-def BUILD_SHARED_LIBS=True"
targets=$minimal_targets
fi
@@ -57,7 +61,7 @@ if grep -q "ARM.* Processor" /proc/cpuinfo; then
else
link=$((link+1))
fi
- link_jobs="-DLLVM_PARALLEL_LINK_JOBS=$link"
+ link_jobs="--cmake-def LLVM_PARALLEL_LINK_JOBS=$link"
fi
fi
@@ -75,14 +79,15 @@ fi
## Re-run CMake if files are damaged / not there
if [ ! -f build.ninja ] && [ ! -f Makefile ]; then
- echo " + Configuring Build"
- safe_run cmake -G "$generator" $LLVM_SRC \
- -DCMAKE_BUILD_TYPE=$build_type \
- -DLLVM_BUILD_TESTS=True \
- -DLLVM_ENABLE_ASSERTIONS=True \
- -DPYTHON_EXECUTABLE=/usr/bin/python2 \
- -DCMAKE_INSTALL_PREFIX=$install_dir \
- -DLLVM_LIT_ARGS="-sv $PARALLEL" \
+ echo " + Configuring Build"
+ safe_run python3 $llvmtool --env $env configure --build-dir $build_dir \
+ --cmake-generator "$generator" \
+ --cmake-def CMAKE_BUILD_TYPE=$build_type \
+ --cmake-def LLVM_BUILD_TESTS=True \
+ --cmake-def LLVM_ENABLE_ASSERTIONS=True \
+ --cmake-def PYTHON_EXECUTABLE=/usr/bin/python2 \
+ --cmake-def CMAKE_INSTALL_PREFIX=$install_dir \
+ --cmake-def LLVM_LIT_ARGS="-sv $PARALLEL" \
$LLVM_CMAKE_FLAGS $targets $shared $link_jobs
fi
diff --git a/helpers/llvm-projs b/helpers/llvm-projs
index 21b8bea..f4d5df7 100755
--- a/helpers/llvm-projs
+++ b/helpers/llvm-projs
@@ -197,3 +197,13 @@ if [ "$remove" != "" ]; then
fi
safe_run python3 $llvmtool --env $env projects --repos $repos $add $remove
+
+if [ -d $env/build ]; then
+ echo "Updating $env/build"
+ safe_run python3 $llvmtool --env $env configure --build-dir $env/build
+fi
+
+if [ -d $env/debug ]; then
+ echo "Updating $env/debug"
+ safe_run python3 $llvmtool --env $env configure --build-dir $env/debug
+fi
diff --git a/modules/llvm.py b/modules/llvm.py
index 8d549fd..2625038 100644
--- a/modules/llvm.py
+++ b/modules/llvm.py
@@ -1,6 +1,8 @@
import os
import re
+from functools import partial
+
from linaropy.git.worktree import Worktree
@@ -98,6 +100,10 @@ class LLVMSourceConfig(object):
self.llvmSourceTree = Worktree(proj, sourcePath)
self.subprojs = subprojs
+ def get_path(self):
+ """Get the path corresponding to this source config."""
+ return self.llvmSourceTree.repodir
+
def get_enabled_subprojects(self):
"""Get a list of the subprojects enabled in this configuration."""
enabled = []
@@ -263,6 +269,63 @@ class LLVMSourceConfig(object):
worktree.clean(False)
+class LLVMBuildConfig(object):
+ """Class for managing an LLVM build directory.
+
+ It should know how to configure a build directory (with CMake) and how to
+ run a build command. The directory must already exist, but it may be empty.
+ """
+
+ def __init__(self, sourceConfig, buildDir=None):
+ """Create an LLVMBuildConfig."""
+ self.sourceConfig = sourceConfig
+ self.buildDir = buildDir
+
+ def cmake(self, commandConsumer, cmakeFlags, generator):
+ """
+ Generate the CMake command needed for configuring the build directory
+ with the given flags and generator, and pass it to the 'commandConsumer'.
+
+ The command will always explicitly enable or disable the build of
+ specific subprojects to mirror the source config. This is important
+ because projects can always be added or removed from the source config,
+ and CMake doesn't by default pick up the new situation (so we might end
+ up trying to build subprojects that were removed, or not build
+ subprojects that were added).
+
+ The 'commandConsumer' should have a 'consume' method taking two
+ parameters: the command to be consumed (in the form of a list) and the
+ directory where the command should be run. Any exceptions that may be
+ raised by that method should be handled by the calling code.
+ """
+ cmakeSubprojFlags = self.__get_subproj_flags()
+ command = ["cmake", "-G", generator] + cmakeSubprojFlags + \
+ cmakeFlags + [self.sourceConfig.get_path()]
+ commandConsumer.consume(command, self.buildDir)
+
+ def __get_subproj_flags(self):
+ """
+ Get the CMake flags needed to explicitly enable or disable the build of
+ each specific subproject, depending on whether it is enabled or disabled
+ in the source config.
+ """
+ def append_cmake_var(cmakeVars, subproj, enabled):
+ if subproj.cmake_var is None:
+ return
+
+ if enabled:
+ status = "ON"
+ else:
+ status = "OFF"
+ cmakeVars.append("-D{}={}".format(subproj.cmake_var, status))
+
+ cmakeSubprojFlags = []
+ self.sourceConfig.for_each_subproj(partial(append_cmake_var,
+ cmakeSubprojFlags))
+
+ return cmakeSubprojFlags
+
+
# FIXME: repo.pushToBranch doesn't work, because it doesn't handle remote
# branches properly. Furthermore, there's no support for getting the remote URL,
# so we need to resort to raw git commands. We may also consider moving the
diff --git a/modules/utils.py b/modules/utils.py
new file mode 100644
index 0000000..9e6f29a
--- /dev/null
+++ b/modules/utils.py
@@ -0,0 +1,29 @@
+from subprocess import CalledProcessError
+from subprocess import check_output
+from subprocess import STDOUT
+
+
+class CommandPrinter(object):
+ """Command consumer that just prints the commands that it receives."""
+
+ def consume(self, command, directory):
+ print("{}$ {}".format(directory, ' '.join(command)))
+
+
+class CommandRunner(object):
+ """Command consumer that runs the commands that it receives."""
+
+ def consume(self, command, directory):
+ """
+ Run the given command in the given directory and print the stdout and
+ stderr. If an exception is thrown while running the command, it will be
+ rethrown as a RuntimeError.
+ """
+ # FIXME: This prints the results after the command has finished running.
+ # For long-running commands (e.g. an LLVM build) we'll want live
+ # output.
+ try:
+ print(str(check_output(command, stderr=STDOUT, cwd=directory), 'utf-8'))
+ except CalledProcessError as exc:
+ raise RuntimeError(
+ "Error while running command\n{}".format(str(exc.output, 'utf-8'))) from exc
diff --git a/scripts/llvm.py b/scripts/llvm.py
index 27b8cb7..2489171 100644
--- a/scripts/llvm.py
+++ b/scripts/llvm.py
@@ -4,11 +4,15 @@ import os
from sys import argv
from sys import exit
+from modules.llvm import LLVMBuildConfig
from modules.llvm import LLVMSubproject
from modules.llvm import LLVMSourceConfig
from modules.llvm import get_remote_branch
from modules.llvm import push_branch
+from modules.utils import CommandPrinter
+from modules.utils import CommandRunner
+from linaropy.cd import cd
from linaropy.git.clone import Clone
from linaropy.proj import Proj
@@ -91,6 +95,32 @@ def push_current_branch(args):
die("Failed to push branch because: " + str(exc) + str(exc.__cause__))
+def configure_build(args):
+ """Configure a given build directory."""
+
+ proj = Proj()
+
+ llvm_worktree_root = get_worktree_root(args.env)
+ sourceConfig = LLVMSourceConfig(proj, llvm_worktree_root)
+
+ buildConfig = LLVMBuildConfig(sourceConfig, args.build)
+
+ if args.defs:
+ args.defs = ["-D{}".format(v) for v in args.defs]
+
+ if args.dry:
+ consumer = CommandPrinter()
+ else:
+ if not os.path.exists(args.build):
+ os.makedirs(args.build)
+ consumer = CommandRunner()
+
+ try:
+ buildConfig.cmake(consumer, args.defs, args.generator)
+ except RuntimeError as exc:
+ die("Failed to configure {} because:\n{}".format(args.build, str(exc)))
+
+
##########################################################################
# Command line parsing #
##########################################################################
@@ -147,6 +177,38 @@ push = subcommands.add_parser(
"for all enabled subprojects.")
push.set_defaults(run_command=push_current_branch)
+# Subcommand for configuring a build directory
+configure = subcommands.add_parser(
+ 'configure',
+ help="Run CMake in the given build directory.")
+configure.add_argument(
+ '--build-dir',
+ dest='build',
+ required=True,
+ help="Path to the build directory. It will be created if it does not exist")
+configure.add_argument(
+ '--cmake-generator',
+ dest='generator',
+ default='Ninja',
+ help="CMake generator to use (default is Ninja).")
+configure.add_argument(
+ '--cmake-def',
+ dest='defs',
+ metavar='VAR=VALUE',
+ default=[],
+ action='append',
+ # We add the -D in front of the variable ourselves because the argument
+ # parsing gets confused otherwise (and quoting doesn't help).
+ help="Additional CMake definitions, e.g. CMAKE_BUILD_TYPE=Release."
+ "May be passed several times. The -D is added automatically.")
+configure.add_argument(
+ '-n', '--dry-run',
+ dest='dry',
+ action='store_true',
+ default=False,
+ help="Print the CMake command instead of executing it.")
+configure.set_defaults(run_command=configure_build)
+
args = options.parse_args()
if args.subcommand == "projects" and args.add and not args.repos:
projs.error(
diff --git a/tests/unittests/testllvmbuildconfig.py b/tests/unittests/testllvmbuildconfig.py
new file mode 100644
index 0000000..509e8a4
--- /dev/null
+++ b/tests/unittests/testllvmbuildconfig.py
@@ -0,0 +1,89 @@
+from modules.llvm import LLVMBuildConfig
+
+from os import makedirs
+from shutil import rmtree
+
+from unittest import TestCase
+from unittest.mock import MagicMock
+from uuid import uuid4
+
+
+class TestLLVMBuildConfig(TestCase):
+ testdirprefix = "BuildConfigUT"
+
+ def setUp(self):
+ self.sourcePath = "llvm" + str(uuid4())
+ self.buildPath = "build" + str(uuid4())
+
+ sourceConfigAttrs = {"get_path.return_value": self.sourcePath}
+ self.sourceConfig = MagicMock(**sourceConfigAttrs)
+
+ makedirs(self.buildPath)
+
+ def tearDown(self):
+ rmtree(self.buildPath)
+
+ def test_configure_generator(self):
+ """Test that we can use a custom generator for our CMake command."""
+ buildConfig = LLVMBuildConfig(self.sourceConfig, self.buildPath)
+
+ consumer = MagicMock()
+ buildConfig.cmake(consumer, [], "Unix Makefiles")
+ command, directory = consumer.consume.call_args[0]
+
+ self.assertEqual(directory, self.buildPath)
+
+ self.assertEqual(command[0], "cmake")
+ self.assertIn(self.sourcePath, command)
+ self.assertEqual(
+ command.index("-G") + 1,
+ command.index("Unix Makefiles"))
+
+ def test_configure_definitions(self):
+ """Test that we can define custom CMake variables."""
+ buildConfig = LLVMBuildConfig(self.sourceConfig, self.buildPath)
+ flags = ["-DCMAKE_BUILD_TYPE=Release",
+ "-DCMAKE_CXX_FLAGS=\"-Oz -g\""]
+
+ consumer = MagicMock()
+ buildConfig.cmake(consumer, flags, "Ninja")
+ command, directory = consumer.consume.call_args[0]
+
+ self.assertEqual(directory, self.buildPath)
+
+ self.assertEqual(command[0], "cmake")
+ self.assertEqual(command.index("-G") + 1, command.index("Ninja"))
+ self.assertIn(self.sourcePath, command)
+ self.assertEqual(command.index(flags[0]) + 1, command.index(flags[1]))
+
+ def test_update_projects(self):
+ """
+ Test that we explicitly enable/disable subprojects based on what is
+ enabled in the source config.
+ """
+
+ def for_each_subproj(action):
+ # Pretend that clang is enabled and lld isn't.
+ clang_subproj = MagicMock()
+ clang_subproj.cmake_var = "BUILD_CLANG"
+ action(clang_subproj, True)
+
+ lld_subproj = MagicMock()
+ lld_subproj.cmake_var = "BUILD_LLD"
+ action(lld_subproj, False)
+
+ self.sourceConfig.for_each_subproj.side_effect = for_each_subproj
+
+ buildConfig = LLVMBuildConfig(self.sourceConfig, self.buildPath)
+
+ consumer = MagicMock()
+ buildConfig.cmake(consumer, [], "Ninja")
+ command, directory = consumer.consume.call_args[0]
+
+ self.assertEqual(directory, self.buildPath)
+
+ self.assertEqual(command[0], "cmake")
+ self.assertEqual(command.index("-G") + 1, command.index("Ninja"))
+ self.assertIn(self.sourcePath, command)
+ self.assertIn("-DBUILD_CLANG=ON", command)
+ self.assertIn("-DBUILD_LLD=OFF", command)
diff --git a/tests/unittests/testllvmsourceconfig.py b/tests/unittests/testllvmsourceconfig.py
index 2c69fa3..0baf7e3 100644
--- a/tests/unittests/testllvmsourceconfig.py
+++ b/tests/unittests/testllvmsourceconfig.py
@@ -58,6 +58,11 @@ class TestLLVMSourceConfig(unittest.TestCase):
def tearDown(self):
self.proj.cleanup()
+ def test_get_path(self):
+ sourcePath = self.temporaryLLVM.repodir
+ config = LLVMSourceConfig(self.proj, sourcePath)
+ self.assertEqual(config.get_path(), sourcePath)
+
def test_detect_enabled_all(self):
subprojs = LLVMSubproject.get_all_subprojects()
sourcePath = self.temporaryLLVM.repodir