From 052b7d37cefe8975f699110a5c9508b2501a97e0 Mon Sep 17 00:00:00 2001 From: Diana Picus Date: Fri, 24 Nov 2017 16:19:41 +0100 Subject: Add llvm.py configure Add a subcommand that runs CMake in a given build directory, with a custom generator and custom CMake definitions. Also update llvm-build to use it instead of calling CMake directly, and add calls to it in llvm-projs as well to make sure we update the build directories whenever we enable or disable a project. One known issue with the current code is that the output of the CMake command is not printed out live, but rather after the command has finished execution. This is going to be more important for llvm.py build than for llvm.py configure, so we can fix it in a future commit. Change-Id: I263b2d47c2083a1778608253bbd437149375c539 --- helpers/llvm-build | 27 ++++++---- helpers/llvm-projs | 10 ++++ modules/llvm.py | 63 +++++++++++++++++++++++ modules/utils.py | 29 +++++++++++ scripts/llvm.py | 62 +++++++++++++++++++++++ tests/unittests/testllvmbuildconfig.py | 89 +++++++++++++++++++++++++++++++++ tests/unittests/testllvmsourceconfig.py | 5 ++ 7 files changed, 274 insertions(+), 11 deletions(-) create mode 100644 modules/utils.py create mode 100644 tests/unittests/testllvmbuildconfig.py diff --git a/helpers/llvm-build b/helpers/llvm-build index 0556b6e..d40ef40 100755 --- a/helpers/llvm-build +++ b/helpers/llvm-build @@ -8,7 +8,11 @@ . llvm-common +progdir=$(dirname $0) +llvmtool=$progdir/../scripts/llvm.py + safe_run verify_env +env=$(dirname $LLVM_SRC) ## CMD line options and defaults CPUS=$(grep -c proc /proc/cpuinfo) @@ -24,7 +28,7 @@ update=false check=false master=false inst=false -minimal_targets="-DLLVM_TARGETS_TO_BUILD='ARM;X86;AArch64'" +minimal_targets="--cmake-def LLVM_TARGETS_TO_BUILD='ARM;X86;AArch64'" link_jobs= if [ "$1" = "-h" ]; then @@ -43,7 +47,7 @@ fi ## Debug mode, make it lighter if [ "$LLVM_DEBUG" = true ]; then build_type=Debug - shared=-DBUILD_SHARED_LIBS=True + shared="--cmake-def BUILD_SHARED_LIBS=True" targets=$minimal_targets fi @@ -57,7 +61,7 @@ if grep -q "ARM.* Processor" /proc/cpuinfo; then else link=$((link+1)) fi - link_jobs="-DLLVM_PARALLEL_LINK_JOBS=$link" + link_jobs="--cmake-def LLVM_PARALLEL_LINK_JOBS=$link" fi fi @@ -75,14 +79,15 @@ fi ## Re-run CMake if files are damaged / not there if [ ! -f build.ninja ] && [ ! -f Makefile ]; then - echo " + Configuring Build" - safe_run cmake -G "$generator" $LLVM_SRC \ - -DCMAKE_BUILD_TYPE=$build_type \ - -DLLVM_BUILD_TESTS=True \ - -DLLVM_ENABLE_ASSERTIONS=True \ - -DPYTHON_EXECUTABLE=/usr/bin/python2 \ - -DCMAKE_INSTALL_PREFIX=$install_dir \ - -DLLVM_LIT_ARGS="-sv $PARALLEL" \ + echo " + Configuring Build" + safe_run python3 $llvmtool --env $env configure --build-dir $build_dir \ + --cmake-generator "$generator" \ + --cmake-def CMAKE_BUILD_TYPE=$build_type \ + --cmake-def LLVM_BUILD_TESTS=True \ + --cmake-def LLVM_ENABLE_ASSERTIONS=True \ + --cmake-def PYTHON_EXECUTABLE=/usr/bin/python2 \ + --cmake-def CMAKE_INSTALL_PREFIX=$install_dir \ + --cmake-def LLVM_LIT_ARGS="-sv $PARALLEL" \ $LLVM_CMAKE_FLAGS $targets $shared $link_jobs fi diff --git a/helpers/llvm-projs b/helpers/llvm-projs index 21b8bea..f4d5df7 100755 --- a/helpers/llvm-projs +++ b/helpers/llvm-projs @@ -197,3 +197,13 @@ if [ "$remove" != "" ]; then fi safe_run python3 $llvmtool --env $env projects --repos $repos $add $remove + +if [ -d $env/build ]; then + echo "Updating $env/build" + safe_run python3 $llvmtool --env $env configure --build-dir $env/build +fi + +if [ -d $env/debug ]; then + echo "Updating $env/debug" + safe_run python3 $llvmtool --env $env configure --build-dir $env/debug +fi diff --git a/modules/llvm.py b/modules/llvm.py index 8d549fd..2625038 100644 --- a/modules/llvm.py +++ b/modules/llvm.py @@ -1,6 +1,8 @@ import os import re +from functools import partial + from linaropy.git.worktree import Worktree @@ -98,6 +100,10 @@ class LLVMSourceConfig(object): self.llvmSourceTree = Worktree(proj, sourcePath) self.subprojs = subprojs + def get_path(self): + """Get the path corresponding to this source config.""" + return self.llvmSourceTree.repodir + def get_enabled_subprojects(self): """Get a list of the subprojects enabled in this configuration.""" enabled = [] @@ -263,6 +269,63 @@ class LLVMSourceConfig(object): worktree.clean(False) +class LLVMBuildConfig(object): + """Class for managing an LLVM build directory. + + It should know how to configure a build directory (with CMake) and how to + run a build command. The directory must already exist, but it may be empty. + """ + + def __init__(self, sourceConfig, buildDir=None): + """Create an LLVMBuildConfig.""" + self.sourceConfig = sourceConfig + self.buildDir = buildDir + + def cmake(self, commandConsumer, cmakeFlags, generator): + """ + Generate the CMake command needed for configuring the build directory + with the given flags and generator, and pass it to the 'commandConsumer'. + + The command will always explicitly enable or disable the build of + specific subprojects to mirror the source config. This is important + because projects can always be added or removed from the source config, + and CMake doesn't by default pick up the new situation (so we might end + up trying to build subprojects that were removed, or not build + subprojects that were added). + + The 'commandConsumer' should have a 'consume' method taking two + parameters: the command to be consumed (in the form of a list) and the + directory where the command should be run. Any exceptions that may be + raised by that method should be handled by the calling code. + """ + cmakeSubprojFlags = self.__get_subproj_flags() + command = ["cmake", "-G", generator] + cmakeSubprojFlags + \ + cmakeFlags + [self.sourceConfig.get_path()] + commandConsumer.consume(command, self.buildDir) + + def __get_subproj_flags(self): + """ + Get the CMake flags needed to explicitly enable or disable the build of + each specific subproject, depending on whether it is enabled or disabled + in the source config. + """ + def append_cmake_var(cmakeVars, subproj, enabled): + if subproj.cmake_var is None: + return + + if enabled: + status = "ON" + else: + status = "OFF" + cmakeVars.append("-D{}={}".format(subproj.cmake_var, status)) + + cmakeSubprojFlags = [] + self.sourceConfig.for_each_subproj(partial(append_cmake_var, + cmakeSubprojFlags)) + + return cmakeSubprojFlags + + # FIXME: repo.pushToBranch doesn't work, because it doesn't handle remote # branches properly. Furthermore, there's no support for getting the remote URL, # so we need to resort to raw git commands. We may also consider moving the diff --git a/modules/utils.py b/modules/utils.py new file mode 100644 index 0000000..9e6f29a --- /dev/null +++ b/modules/utils.py @@ -0,0 +1,29 @@ +from subprocess import CalledProcessError +from subprocess import check_output +from subprocess import STDOUT + + +class CommandPrinter(object): + """Command consumer that just prints the commands that it receives.""" + + def consume(self, command, directory): + print("{}$ {}".format(directory, ' '.join(command))) + + +class CommandRunner(object): + """Command consumer that runs the commands that it receives.""" + + def consume(self, command, directory): + """ + Run the given command in the given directory and print the stdout and + stderr. If an exception is thrown while running the command, it will be + rethrown as a RuntimeError. + """ + # FIXME: This prints the results after the command has finished running. + # For long-running commands (e.g. an LLVM build) we'll want live + # output. + try: + print(str(check_output(command, stderr=STDOUT, cwd=directory), 'utf-8')) + except CalledProcessError as exc: + raise RuntimeError( + "Error while running command\n{}".format(str(exc.output, 'utf-8'))) from exc diff --git a/scripts/llvm.py b/scripts/llvm.py index 27b8cb7..2489171 100644 --- a/scripts/llvm.py +++ b/scripts/llvm.py @@ -4,11 +4,15 @@ import os from sys import argv from sys import exit +from modules.llvm import LLVMBuildConfig from modules.llvm import LLVMSubproject from modules.llvm import LLVMSourceConfig from modules.llvm import get_remote_branch from modules.llvm import push_branch +from modules.utils import CommandPrinter +from modules.utils import CommandRunner +from linaropy.cd import cd from linaropy.git.clone import Clone from linaropy.proj import Proj @@ -91,6 +95,32 @@ def push_current_branch(args): die("Failed to push branch because: " + str(exc) + str(exc.__cause__)) +def configure_build(args): + """Configure a given build directory.""" + + proj = Proj() + + llvm_worktree_root = get_worktree_root(args.env) + sourceConfig = LLVMSourceConfig(proj, llvm_worktree_root) + + buildConfig = LLVMBuildConfig(sourceConfig, args.build) + + if args.defs: + args.defs = ["-D{}".format(v) for v in args.defs] + + if args.dry: + consumer = CommandPrinter() + else: + if not os.path.exists(args.build): + os.makedirs(args.build) + consumer = CommandRunner() + + try: + buildConfig.cmake(consumer, args.defs, args.generator) + except RuntimeError as exc: + die("Failed to configure {} because:\n{}".format(args.build, str(exc))) + + ########################################################################## # Command line parsing # ########################################################################## @@ -147,6 +177,38 @@ push = subcommands.add_parser( "for all enabled subprojects.") push.set_defaults(run_command=push_current_branch) +# Subcommand for configuring a build directory +configure = subcommands.add_parser( + 'configure', + help="Run CMake in the given build directory.") +configure.add_argument( + '--build-dir', + dest='build', + required=True, + help="Path to the build directory. It will be created if it does not exist") +configure.add_argument( + '--cmake-generator', + dest='generator', + default='Ninja', + help="CMake generator to use (default is Ninja).") +configure.add_argument( + '--cmake-def', + dest='defs', + metavar='VAR=VALUE', + default=[], + action='append', + # We add the -D in front of the variable ourselves because the argument + # parsing gets confused otherwise (and quoting doesn't help). + help="Additional CMake definitions, e.g. CMAKE_BUILD_TYPE=Release." + "May be passed several times. The -D is added automatically.") +configure.add_argument( + '-n', '--dry-run', + dest='dry', + action='store_true', + default=False, + help="Print the CMake command instead of executing it.") +configure.set_defaults(run_command=configure_build) + args = options.parse_args() if args.subcommand == "projects" and args.add and not args.repos: projs.error( diff --git a/tests/unittests/testllvmbuildconfig.py b/tests/unittests/testllvmbuildconfig.py new file mode 100644 index 0000000..509e8a4 --- /dev/null +++ b/tests/unittests/testllvmbuildconfig.py @@ -0,0 +1,89 @@ +from modules.llvm import LLVMBuildConfig + +from os import makedirs +from shutil import rmtree + +from unittest import TestCase +from unittest.mock import MagicMock +from uuid import uuid4 + + +class TestLLVMBuildConfig(TestCase): + testdirprefix = "BuildConfigUT" + + def setUp(self): + self.sourcePath = "llvm" + str(uuid4()) + self.buildPath = "build" + str(uuid4()) + + sourceConfigAttrs = {"get_path.return_value": self.sourcePath} + self.sourceConfig = MagicMock(**sourceConfigAttrs) + + makedirs(self.buildPath) + + def tearDown(self): + rmtree(self.buildPath) + + def test_configure_generator(self): + """Test that we can use a custom generator for our CMake command.""" + buildConfig = LLVMBuildConfig(self.sourceConfig, self.buildPath) + + consumer = MagicMock() + buildConfig.cmake(consumer, [], "Unix Makefiles") + command, directory = consumer.consume.call_args[0] + + self.assertEqual(directory, self.buildPath) + + self.assertEqual(command[0], "cmake") + self.assertIn(self.sourcePath, command) + self.assertEqual( + command.index("-G") + 1, + command.index("Unix Makefiles")) + + def test_configure_definitions(self): + """Test that we can define custom CMake variables.""" + buildConfig = LLVMBuildConfig(self.sourceConfig, self.buildPath) + flags = ["-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CXX_FLAGS=\"-Oz -g\""] + + consumer = MagicMock() + buildConfig.cmake(consumer, flags, "Ninja") + command, directory = consumer.consume.call_args[0] + + self.assertEqual(directory, self.buildPath) + + self.assertEqual(command[0], "cmake") + self.assertEqual(command.index("-G") + 1, command.index("Ninja")) + self.assertIn(self.sourcePath, command) + self.assertEqual(command.index(flags[0]) + 1, command.index(flags[1])) + + def test_update_projects(self): + """ + Test that we explicitly enable/disable subprojects based on what is + enabled in the source config. + """ + + def for_each_subproj(action): + # Pretend that clang is enabled and lld isn't. + clang_subproj = MagicMock() + clang_subproj.cmake_var = "BUILD_CLANG" + action(clang_subproj, True) + + lld_subproj = MagicMock() + lld_subproj.cmake_var = "BUILD_LLD" + action(lld_subproj, False) + + self.sourceConfig.for_each_subproj.side_effect = for_each_subproj + + buildConfig = LLVMBuildConfig(self.sourceConfig, self.buildPath) + + consumer = MagicMock() + buildConfig.cmake(consumer, [], "Ninja") + command, directory = consumer.consume.call_args[0] + + self.assertEqual(directory, self.buildPath) + + self.assertEqual(command[0], "cmake") + self.assertEqual(command.index("-G") + 1, command.index("Ninja")) + self.assertIn(self.sourcePath, command) + self.assertIn("-DBUILD_CLANG=ON", command) + self.assertIn("-DBUILD_LLD=OFF", command) diff --git a/tests/unittests/testllvmsourceconfig.py b/tests/unittests/testllvmsourceconfig.py index 2c69fa3..0baf7e3 100644 --- a/tests/unittests/testllvmsourceconfig.py +++ b/tests/unittests/testllvmsourceconfig.py @@ -58,6 +58,11 @@ class TestLLVMSourceConfig(unittest.TestCase): def tearDown(self): self.proj.cleanup() + def test_get_path(self): + sourcePath = self.temporaryLLVM.repodir + config = LLVMSourceConfig(self.proj, sourcePath) + self.assertEqual(config.get_path(), sourcePath) + def test_detect_enabled_all(self): subprojs = LLVMSubproject.get_all_subprojects() sourcePath = self.temporaryLLVM.repodir -- cgit v1.2.3