scripts/llvm.py

"""This is the main tool for handling llvm builds, bisects etc."""

import os
from sys import argv
from sys import exit

from modules.llvm import LLVMBuildConfig
from modules.llvm import LLVMSubproject
from modules.llvm import LLVMSourceConfig
from modules.llvm import run_test_suite
from modules.llvm import setup_test_suite
from modules.utils import CommandPrinter
from modules.utils import CommandRunner
from modules.utils import get_remote_branch
from modules.utils import push_branch

from linaropy.cd import cd
from linaropy.git.clone import Clone
from linaropy.proj import Proj

from argparse import Action, ArgumentParser, RawTextHelpFormatter
from functools import partial


def die(message, config_to_dump=None):
    """Print an error message and exit."""
    print(message)

    if config_to_dump is not None:
        dump_config(config_to_dump)

    exit(1)


def dump_config(config):
    """Dump the list of projects that are enabled in the given config."""

    print("Projects linked:")
    enabled = config.get_enabled_subprojects()
    if not enabled:
        print("none")
    else:
        for subproj in sorted(enabled):
            print("  + {}".format(subproj))


def subproj_to_repo_map(subprojs, proj, reposRoot, dry=False):
    """Get a dictionary mapping each subproject in subprojs to its repo."""
    subprojsToRepos = {}
    repo = None
    for subproj in subprojs:
        if not dry:
            repo = Clone(proj, os.path.join(reposRoot, subproj))
        subprojsToRepos[subproj] = repo

    return subprojsToRepos


def projects(args):
    """Add/remove subprojects based on the values in args."""

    proj = Proj()

    llvm_worktree_root = args.sources
    llvm_repos_root = args.repos
    config = LLVMSourceConfig(proj, llvm_worktree_root, dry=False)

    if not args.add and not args.remove:
        # Nothing to change, just print the current configuration
        dump_config(config)
        exit(0)

    to_add = {}
    if args.add:
        to_add = subproj_to_repo_map(args.add, proj, llvm_repos_root)

    try:
        config.update(to_add, args.remove)
    except (EnvironmentError, ValueError) as exc:
        die("Failed to update subprojects because:\n{}".format(str(exc)))

    dump_config(config)


def push_current_branch(args):
    """Push current branch to origin."""

    proj = Proj()

    llvm_worktree_root = args.sources
    config = LLVMSourceConfig(proj, llvm_worktree_root, dry=False)

    llvm_worktree = Clone(proj, llvm_worktree_root)
    local_branch = llvm_worktree.getbranch()

    try:
        remote_branch = get_remote_branch(llvm_worktree, local_branch)
        config.for_each_enabled(partial(push_branch, proj, local_branch,
                                        remote_branch))
        print("Pushed to {}".format(remote_branch))
    except (EnvironmentError, RuntimeError) as exc:
        die("Failed to push branch because: " + str(exc) + str(exc.__cause__))


def cmake_flags_from_args(defs):
    """
    Get a list of valid CMake flags from the input VAR=VALUE list.
    This boils down to just adding -D in front of each of them.
    """
    return ["-D{}".format(v) for v in defs]


def configure_build(args):
    """Configure a given build directory."""

    proj = Proj()

    llvm_worktree_root = args.sources
    sourceConfig = LLVMSourceConfig(proj, llvm_worktree_root, args.dry)

    if args.dry:
        consumer = CommandPrinter()
    else:
        if not os.path.exists(args.build):
            os.makedirs(args.build)
        consumer = CommandRunner()

    buildConfig = LLVMBuildConfig(sourceConfig, args.build, consumer)

    if args.defs:
        args.defs = cmake_flags_from_args(args.defs)

    try:
        buildConfig.cmake(args.defs, args.generator)
    except RuntimeError as exc:
        die("Failed to configure {} because:\n{}".format(args.build, str(exc)))


def run_build(args):
    """Run a build command in a given directory."""
    build_dir = args.build

    if args.dry:
        consumer = CommandPrinter()
    else:
        consumer = CommandRunner()

    try:
        LLVMBuildConfig(None, args.build, consumer).build(args.flags)
    except RuntimeError as exc:
        die("Failed to build {} because:\n{}".format(args.build, str(exc)))


def setup_the_test_suite(args):
    """Setup a sandbox for the test-suite."""
    if args.dry:
        consumer = CommandPrinter()
    else:
        consumer = CommandRunner()

    try:
        setup_test_suite(consumer, args.sandbox, args.lnt)
    except RuntimeError as exc:
        die("Failed to setup the test-suite because:\n{}".format(str(exc)))


def run_the_test_suite(args):
    """Run the test-suite in a given sandbox."""
    if args.dry:
        consumer = CommandPrinter()
    else:
        consumer = CommandRunner()

    compilers = ["--cc={}".format(args.cc)]
    if args.cxx:
        compilers.append("--cxx={}".format(args.cxx))

    try:
        run_test_suite(consumer, args.sandbox, args.testsuite, args.lit,
                       compilers + args.flags)
    except RuntimeError as exc:
        die("Failed to run the test-suite because:\n{}".format(str(exc)))


def build_and_test(args):
    """
    Build a set of LLVM subprojects, in one or two stages, with or without a
    test-suite run.
    """

    proj = Proj()

    dryRun = args.dry
    llvmRepos = args.repos
    llvmWorktreeRoot = args.sources

    stage1BuildDir = args.stage1
    stage1Subprojs = args.stage1Subprojs
    stage1Defs = cmake_flags_from_args(args.stage1Defs)
    stage1BuildFlags = args.stage1BuildFlags

    stage2BuildDir = args.stage2
    stage2Subprojs = args.stage2Subprojs
    stage2Defs = cmake_flags_from_args(args.stage2Defs)
    stage2BuildFlags = args.stage2BuildFlags

    enableTestSuite = args.enableTestSuite
    sandboxDir = args.sandbox

    if dryRun:
        consumer = CommandPrinter()
    else:
        consumer = CommandRunner()

    try:
        sourceConfig = LLVMSourceConfig(proj, llvmWorktreeRoot, args.dry)
        if stage1Subprojs:
            # FIXME: Decide whether or not we want to remove anything that isn't
            # in stage1Subprojs (in case there are already some enabled
            # subprojects in the source config).
            sourceConfig.update(
                subproj_to_repo_map(stage1Subprojs, proj, llvmRepos,
                                    args.dry),
                [])

        if not dryRun and not os.path.exists(stage1BuildDir):
            os.makedirs(stage1BuildDir)

        buildConfig1 = LLVMBuildConfig(sourceConfig, stage1BuildDir, consumer)
        buildConfig1.cmake(stage1Defs, "Ninja")
        buildConfig1.build(stage1BuildFlags)
        testedBuildDir = stage1BuildDir

        if stage2BuildDir is not None:
            if stage2Subprojs:
                toAdd = list(set(stage2Subprojs) - set(stage1Subprojs))
                toRemove = list(set(stage1Subprojs) - set(stage2Subprojs))
                sourceConfig.update(
                    subproj_to_repo_map(toAdd, proj, llvmRepos, args.dry),
                    toRemove)

            if not dryRun and not os.path.exists(stage2BuildDir):
                os.makedirs(stage2BuildDir)

            buildConfig2 = LLVMBuildConfig(sourceConfig, stage2BuildDir,
                                           consumer)

            # TODO: Make sure clang is actually built in this config (preferably
            # before reaching this point)
            buildConfig2.cmake(
                stage2Defs + [
                    "-DCMAKE_C_COMPILER={}/bin/clang".format(stage1BuildDir),
                    "-DCMAKE_CXX_COMPILER={}/bin/clang++".format(stage1BuildDir)],
                "Ninja")
            buildConfig2.build(stage2BuildFlags)
            testedBuildDir = stage2BuildDir

        if enableTestSuite:
            testSuiteDir = os.path.join(llvmRepos, "test-suite")
            lntDir = os.path.join(llvmRepos, "lnt")

            setup_test_suite(consumer, sandboxDir, lntDir)

            # TODO: Make sure clang is actually built in this config (preferably
            # before reaching this point)
            lit = os.path.join(testedBuildDir, "bin", "llvm-lit")
            clang = os.path.join(testedBuildDir, "bin", "clang")
            run_test_suite(consumer, sandboxDir, testSuiteDir, lit,
                           ["--cc={}".format(clang)])

    except RuntimeError as exc:
        die("Failed because:\n{}".format(str(exc)))


##########################################################################
# Command line parsing                                                   #
##########################################################################

# If we decide we want shorthands for the subprojects, we can append to this
# list
valid_subprojects = list(LLVMSubproject.get_all_subprojects().keys())

options = ArgumentParser(formatter_class=RawTextHelpFormatter)
subcommands = options.add_subparsers(dest="subcommand")

# Subcommand for adding / removing subprojects
projs = subcommands.add_parser(
    "projects", help="Add/remove LLVM subprojects.\n"
                     "Adding a subproject will create a worktree for it "
                     "somewhere in the LLVM source tree, on the same git "
                     "branch as LLVM itself.\n"
                     "Removing a subproject will remove the worktree, but "
                     "not the underlying git branch.")
projs.set_defaults(run_command=projects)

# TODO: Overwriting previous values is not necessarily what users expect (so for
# instance --add S1 S2 --remove S3 --add S4 would lead to adding only S4). We
# can do better by using action='append', which would create a list (of lists?
# or of lists and scalars?) that we can flatten to obtain all the values passed
# by the user.
projs.add_argument(
    '-a', '--add',
    nargs='+',
    choices=valid_subprojects,
    metavar='subproject',
    help="Enable given subprojects. Valid values are:\n\t{}\n".format(
         "\n\t".join(valid_subprojects)))
projs.add_argument(
    '-r', '--remove',
    nargs='+',
    choices=valid_subprojects,
    metavar='subproject',
    help="Disable given subprojects.")
projs.add_argument(
    '--repos',
    help="Path to the directory containing the repositories for all LLVM "
         "subprojects.")
projs.add_argument(
    '--source-dir',
    dest='sources',
    required=True,
    help="Path to the directory containing the LLVM worktree that we're adding "
         "or removing subprojects from.")

# Subcommand for pushing the current branch to origin
push = subcommands.add_parser(
    "push",
    help="Push current branch to origin linaro-local/<user>/<branch>, "
         "for all enabled subprojects.")
push.set_defaults(run_command=push_current_branch)
push.add_argument(
    '--source-dir',
    dest='sources',
    required=True,
    help="Path to the directory containing the LLVM worktree.")

# Subcommand for configuring a build directory
configure = subcommands.add_parser(
    'configure',
    help="Run CMake in the given build directory.")
configure.add_argument(
    '--source-dir',
    dest='sources',
    required=True,
    help="Path to the sources directory. It should contain an LLVM worktree.")
configure.add_argument(
    '--build-dir',
    dest='build',
    required=True,
    help="Path to the build directory. It will be created if it does not exist")
configure.add_argument(
    '--cmake-generator',
    dest='generator',
    default='Ninja',
    help="CMake generator to use (default is Ninja).")
configure.add_argument(
    '--cmake-def',
    dest='defs',
    metavar='VAR=VALUE',
    default=[],
    action='append',
    # We add the -D in front of the variable ourselves because the argument
    # parsing gets confused otherwise (and quoting doesn't help).
    help="Additional CMake definitions, e.g. CMAKE_BUILD_TYPE=Release."
    "May be passed several times. The -D is added automatically.")
configure.add_argument(
    '-n', '--dry-run',
    dest='dry',
    action='store_true',
    default=False,
    help="Print the CMake command instead of executing it.")
configure.set_defaults(run_command=configure_build)

# Subcommand for building a target
build = subcommands.add_parser(
    'build',
    help="Run a build command in the given directory."
    "The build command can be either a 'ninja' or a 'make' command, depending "
    "on what the build directory contains. First, we look for a 'build.ninja' "
    "file. If that is not found, we look for a 'Makefile'. If that is not "
    "found either, the script fails.")
build.add_argument(
    '--build-dir',
    dest='build',
    required=True,
    help="Path to the build directory. It must have already been configured.")
build.add_argument(
    '-n', '--dry-run',
    dest='dry',
    action='store_true',
    default=False,
    help="Print the build command instead of executing it.")
build.add_argument(
    '--build-flag',
    dest='flags',
    metavar='FLAG',
    default=[],
    action='append',
    help="Additional flags for the build command (e.g. targets to build). "
    "May be passed several times. If your flag starts with a '-', use "
    "'--build-flag=-FLAG' to pass it.")
build.set_defaults(run_command=run_build)

# Subcommand for setting up the test-suite
setupTestSuite = subcommands.add_parser(
    'setup-test-suite',
    help="Prepare a sandbox for running the test-suite.")
setupTestSuite.add_argument(
    '--sandbox',
    required=True,
    help="Path where we should setup the sandbox.")
setupTestSuite.add_argument(
    '--lnt',
    required=True,
    help="Path to the LNT sources.")
setupTestSuite.add_argument(
    '-n', '--dry-run',
    dest='dry',
    action='store_true',
    default=False,
    help="Print the commands instead of executing them.")
setupTestSuite.set_defaults(run_command=setup_the_test_suite)

# Subcommand for running the test-suite
runTestSuite = subcommands.add_parser(
    'run-test-suite',
    help="Run the test-suite in the given sandbox.")
runTestSuite.add_argument(
    '--sandbox',
    required=True,
    help="Path to the sandbox. It must point to a virtualenv with a LNT setup.")
runTestSuite.add_argument(
    '--test-suite',
    dest="testsuite",
    required=True,
    help="Path to the test-suite repo.")
runTestSuite.add_argument(
    '--use-lit',
    dest="lit",
    required=True,
    help="Path to llvm-lit.")
runTestSuite.add_argument(
    '--lnt-flag',
    dest='flags',
    metavar='FLAG',
    default=[],
    action='append',
    help="Additional flags to be passed to LNT when running the test-suite."
    "May be passed several times. If your flag starts with a '-', use "
    "'--lnt-flag=-FLAG' to pass it.")
runTestSuite.add_argument(
    # We can pass --cc through the --lnt-flag interface, but we generally won't
    # want to test the system compiler, so force the user to be specific.
    '--cc',
    required=True,
    help="The path to the C compiler that we're testing.")
runTestSuite.add_argument(
    # For symmetry, we also provide a --cxx argument, but this one isn't
    # required since LNT tries to guess it based on the value of --cc.
    '--cxx',
    required=False,
    help="The path to the C++ compiler that we're testing.")
runTestSuite.add_argument(
    '-n', '--dry-run',
    dest='dry',
    action='store_true',
    default=False,
    help="Print the commands instead of executing them.")
runTestSuite.set_defaults(run_command=run_the_test_suite)

buildAndTest = subcommands.add_parser(
    'build-and-test',  # TODO: This really needs a better name...
    help="Run complex build scenarios with one or two stages of clang and "
         "optionally a test-suite run. This should be flexible enough to allow "
         "us to reproduce any buildbot configuration, but it can obviously be "
         "used for other purposes as well.")
buildAndTest.set_defaults(run_command=build_and_test)
buildAndTest.add_argument(
    '--repos-dir',
    dest='repos',
    required=True,
    help="Path to the root directory containing the repositories for LLVM and "
         "the other subprojects.")
buildAndTest.add_argument(
    '--source-dir',
    dest='sources',
    required=True,
    help="Path to the directory containing the LLVM worktree that we're going "
         "to build from.")
buildAndTest.add_argument(
    '--stage1-build-dir',
    dest='stage1',
    required=True,
    help="Path to the build directory for stage 1.")
buildAndTest.add_argument(
    '--stage1-subproject',
    dest='stage1Subprojs',
    metavar='SUBPROJ',
    choices=valid_subprojects,
    default=[],
    action='append',
    help="Subprojects to enable for stage 1 of the build. Can be passed "
         "multiple times. Valid values for the subproject are: {}. "
         "If this is a 2-stage build, the same subprojects will be used for "
         "both stages unless other subprojects are explicitly requested for "
         "stage 2.".format(" ".join(valid_subprojects)))
buildAndTest.add_argument(
    '--stage1-cmake-def',
    dest='stage1Defs',
    metavar='VAR=VALUE',
    default=[],
    action='append',
    help="Additional CMake definitions for stage 1, e.g. "
         "CMAKE_BUILD_TYPE=Release. Can be passed multiple times. "
         "The -D is added automatically. Does not affect stage 2.")
buildAndTest.add_argument(
    '--stage1-build-flag',
    dest='stage1BuildFlags',
    metavar='FLAG',
    default=[],
    action='append',
    help="Additional flags for the stage 1 build command (e.g. targets to "
         "build). Can be passed multiple times. If your flag starts with "
         "a '-', use '--stage1-build-flag=-FLAG' to pass it. "
         "Does not affect stage 2.")
buildAndTest.add_argument(
    '--stage2-build-dir',
    dest='stage2',
    help="Path to the build directory for stage 2.")
buildAndTest.add_argument(
    '--stage2-subproject',
    dest='stage2Subprojs',
    metavar='SUBPROJ',
    choices=valid_subprojects,
    default=[],
    action='append',
    help="Subprojects to enable for stage 2 of the build. Can be passed "
         "multiple times. Valid values for the subproject are: {}. "
         "If this is a 2-stage build, the same subprojects will be used for "
         "both stages unless other subprojects are explicitly requested for "
         "stage 2.".format(" ".join(valid_subprojects)))
buildAndTest.add_argument(
    '--stage2-cmake-def',
    dest='stage2Defs',
    metavar='VAR=VALUE',
    default=[],
    action='append',
    help="Additional CMake definitions for stage 2, e.g. "
         "CMAKE_BUILD_TYPE=Release. Can be passed multiple times. "
         "The -D is added automatically.")
buildAndTest.add_argument(
    '--stage2-build-flag',
    dest='stage2BuildFlags',
    metavar='FLAG',
    default=[],
    action='append',
    help="Additional flags for the stage 2 build command (e.g. targets to "
         "build). Can be passed multiple times. If your flag starts with "
         "a '-', use '--stage2-build-flag=-FLAG' to pass it.")
buildAndTest.add_argument(
    "--enable-test-suite",
    dest='enableTestSuite',
    action='store_true',
    default=False,
    help="Whether or not to run the test-suite with the last compiler built.")
buildAndTest.add_argument(
    "--sandbox",
    help="Path to the sandbox where the test-suite should be setup.")
buildAndTest.add_argument(
    '-n', '--dry-run',
    dest='dry',
    action='store_true',
    default=False,
    help="Print the commands instead of executing them.")

args = options.parse_args()
if args.subcommand == "projects" and args.add and not args.repos:
    projs.error(
        "When adding a subproject you must also pass the --repos argument")
args.run_command(args)