scripts/llvm.py

"""This is the main tool for handling llvm builds, bisects etc."""

import os
from sys import argv
from sys import exit

from modules.llvm import build_llvm
from modules.llvm import LLVMBuildConfig
from modules.llvm import LLVMSubproject
from modules.llvm import LLVMSourceConfig
from modules.llvm import run_test_suite
from modules.llvm import setup_test_suite
from modules.utils import CommandPrinter
from modules.utils import CommandRunner
from modules.utils import get_remote_branch
from modules.utils import push_branch

from linaropy.cd import cd
from linaropy.git.clone import Clone
from linaropy.proj import Proj

from argparse import Action, ArgumentParser, RawTextHelpFormatter
from functools import partial


def die(message, config_to_dump=None):
    """Print an error message and exit."""
    print(message)

    if config_to_dump is not None:
        dump_config(config_to_dump)

    exit(1)


def dump_config(config):
    """Dump the list of projects that are enabled in the given config."""

    print("Projects linked:")
    enabled = config.get_enabled_subprojects()
    if not enabled:
        print("none")
    else:
        for subproj in sorted(enabled):
            print("  + {}".format(subproj))


def projects(args):
    """Add/remove subprojects based on the values in args."""

    proj = Proj()

    llvm_worktree_root = args.sources
    llvm_repos_root = args.repos
    config = LLVMSourceConfig(proj, llvm_worktree_root, dry=False)

    if not args.add and not args.remove:
        # Nothing to change, just print the current configuration
        dump_config(config)
        exit(0)

    to_add = {}
    if args.add:
        for subproj in args.add:
            repo = Clone(proj, os.path.join(llvm_repos_root, subproj))
            to_add[subproj] = repo

    try:
        config.update(to_add, args.remove)
    except (EnvironmentError, ValueError) as exc:
        die("Failed to update subprojects because:\n{}".format(str(exc)))

    dump_config(config)


def push_current_branch(args):
    """Push current branch to origin."""

    proj = Proj()

    llvm_worktree_root = args.sources
    config = LLVMSourceConfig(proj, llvm_worktree_root, dry=False)

    llvm_worktree = Clone(proj, llvm_worktree_root)
    local_branch = llvm_worktree.getbranch()

    try:
        remote_branch = get_remote_branch(llvm_worktree, local_branch)
        config.for_each_enabled(partial(push_branch, proj, local_branch,
                                        remote_branch))
        print("Pushed to {}".format(remote_branch))
    except (EnvironmentError, RuntimeError) as exc:
        die("Failed to push branch because: " + str(exc) + str(exc.__cause__))


def configure_build(args):
    """Configure a given build directory."""

    proj = Proj()

    llvm_worktree_root = args.sources
    sourceConfig = LLVMSourceConfig(proj, llvm_worktree_root, args.dry)

    buildConfig = LLVMBuildConfig(sourceConfig, args.build)

    if args.defs:
        args.defs = ["-D{}".format(v) for v in args.defs]

    if args.dry:
        consumer = CommandPrinter()
    else:
        if not os.path.exists(args.build):
            os.makedirs(args.build)
        consumer = CommandRunner()

    try:
        buildConfig.cmake(consumer, args.defs, args.generator)
    except RuntimeError as exc:
        die("Failed to configure {} because:\n{}".format(args.build, str(exc)))


def run_build(args):
    """Run a build command in a given directory."""
    build_dir = args.build

    if args.dry:
        consumer = CommandPrinter()
    else:
        consumer = CommandRunner()

    try:
        build_llvm(consumer, args.build, args.flags)
    except RuntimeError as exc:
        die("Failed to build {} because:\n{}".format(args.build, str(exc)))


def setup_the_test_suite(args):
    """Setup a sandbox for the test-suite."""
    if args.dry:
        consumer = CommandPrinter()
    else:
        consumer = CommandRunner()

    try:
        setup_test_suite(consumer, args.sandbox, args.lnt)
    except RuntimeError as exc:
        die("Failed to setup the test-suite because:\n{}".format(str(exc)))


def run_the_test_suite(args):
    """Run the test-suite in a given sandbox."""
    if args.dry:
        consumer = CommandPrinter()
    else:
        consumer = CommandRunner()

    compilers = ["--cc={}".format(args.cc)]
    if args.cxx:
        compilers.append("--cxx={}".format(args.cxx))

    try:
        run_test_suite(consumer, args.sandbox, args.testsuite, args.lit,
                       compilers + args.flags)
    except RuntimeError as exc:
        die("Failed to run the test-suite because:\n{}".format(str(exc)))

##########################################################################
# Command line parsing                                                   #
##########################################################################

# If we decide we want shorthands for the subprojects, we can append to this
# list
valid_subprojects = list(LLVMSubproject.get_all_subprojects().keys())

options = ArgumentParser(formatter_class=RawTextHelpFormatter)
subcommands = options.add_subparsers(dest="subcommand")

# Subcommand for adding / removing subprojects
projs = subcommands.add_parser(
    "projects", help="Add/remove LLVM subprojects.\n"
                     "Adding a subproject will create a worktree for it "
                     "somewhere in the LLVM source tree, on the same git "
                     "branch as LLVM itself.\n"
                     "Removing a subproject will remove the worktree, but "
                     "not the underlying git branch.")
projs.set_defaults(run_command=projects)

# TODO: Overwriting previous values is not necessarily what users expect (so for
# instance --add S1 S2 --remove S3 --add S4 would lead to adding only S4). We
# can do better by using action='append', which would create a list (of lists?
# or of lists and scalars?) that we can flatten to obtain all the values passed
# by the user.
projs.add_argument(
    '-a', '--add',
    nargs='+',
    choices=valid_subprojects,
    metavar='subproject',
    help="Enable given subprojects. Valid values are:\n\t{}\n".format(
         "\n\t".join(valid_subprojects)))
projs.add_argument(
    '-r', '--remove',
    nargs='+',
    choices=valid_subprojects,
    metavar='subproject',
    help="Disable given subprojects.")
projs.add_argument(
    '--repos',
    help="Path to the directory containing the repositories for all LLVM "
         "subprojects.")
projs.add_argument(
    '--source-dir',
    dest='sources',
    required=True,
    help="Path to the directory containing the LLVM worktree that we're adding "
         "or removing subprojects from.")

# Subcommand for pushing the current branch to origin
push = subcommands.add_parser(
    "push",
    help="Push current branch to origin linaro-local/<user>/<branch>, "
         "for all enabled subprojects.")
push.set_defaults(run_command=push_current_branch)
push.add_argument(
    '--source-dir',
    dest='sources',
    required=True,
    help="Path to the directory containing the LLVM worktree.")

# Subcommand for configuring a build directory
configure = subcommands.add_parser(
    'configure',
    help="Run CMake in the given build directory.")
configure.add_argument(
    '--source-dir',
    dest='sources',
    required=True,
    help="Path to the sources directory. It should contain an LLVM worktree.")
configure.add_argument(
    '--build-dir',
    dest='build',
    required=True,
    help="Path to the build directory. It will be created if it does not exist")
configure.add_argument(
    '--cmake-generator',
    dest='generator',
    default='Ninja',
    help="CMake generator to use (default is Ninja).")
configure.add_argument(
    '--cmake-def',
    dest='defs',
    metavar='VAR=VALUE',
    default=[],
    action='append',
    # We add the -D in front of the variable ourselves because the argument
    # parsing gets confused otherwise (and quoting doesn't help).
    help="Additional CMake definitions, e.g. CMAKE_BUILD_TYPE=Release."
    "May be passed several times. The -D is added automatically.")
configure.add_argument(
    '-n', '--dry-run',
    dest='dry',
    action='store_true',
    default=False,
    help="Print the CMake command instead of executing it.")
configure.set_defaults(run_command=configure_build)

# Subcommand for building a target
build = subcommands.add_parser(
    'build',
    help="Run a build command in the given directory."
    "The build command can be either a 'ninja' or a 'make' command, depending "
    "on what the build directory contains. First, we look for a 'build.ninja' "
    "file. If that is not found, we look for a 'Makefile'. If that is not "
    "found either, the script fails.")
build.add_argument(
    '--build-dir',
    dest='build',
    required=True,
    help="Path to the build directory. It must have already been configured.")
build.add_argument(
    '-n', '--dry-run',
    dest='dry',
    action='store_true',
    default=False,
    help="Print the build command instead of executing it.")
build.add_argument(
    '--build-flag',
    dest='flags',
    metavar='FLAG',
    default=[],
    action='append',
    help="Additional flags for the build command (e.g. targets to build). "
    "May be passed several times. If your flag starts with a '-', use "
    "'--build-flag=-FLAG' to pass it.")
build.set_defaults(run_command=run_build)

# Subcommand for setting up the test-suite
setupTestSuite = subcommands.add_parser(
    'setup-test-suite',
    help="Prepare a sandbox for running the test-suite.")
setupTestSuite.add_argument(
    '--sandbox',
    required=True,
    help="Path where we should setup the sandbox.")
setupTestSuite.add_argument(
    '--lnt',
    required=True,
    help="Path to the LNT sources.")
setupTestSuite.add_argument(
    '-n', '--dry-run',
    dest='dry',
    action='store_true',
    default=False,
    help="Print the commands instead of executing them.")
setupTestSuite.set_defaults(run_command=setup_the_test_suite)

# Subcommand for running the test-suite
runTestSuite = subcommands.add_parser(
    'run-test-suite',
    help="Run the test-suite in the given sandbox.")
runTestSuite.add_argument(
    '--sandbox',
    required=True,
    help="Path to the sandbox. It must point to a virtualenv with a LNT setup.")
runTestSuite.add_argument(
    '--test-suite',
    dest="testsuite",
    required=True,
    help="Path to the test-suite repo.")
runTestSuite.add_argument(
    '--use-lit',
    dest="lit",
    required=True,
    help="Path to llvm-lit.")
runTestSuite.add_argument(
    '--lnt-flag',
    dest='flags',
    metavar='FLAG',
    default=[],
    action='append',
    help="Additional flags to be passed to LNT when running the test-suite."
    "May be passed several times. If your flag starts with a '-', use "
    "'--lnt-flag=-FLAG' to pass it.")
runTestSuite.add_argument(
    # We can pass --cc through the --lnt-flag interface, but we generally won't
    # want to test the system compiler, so force the user to be specific.
    '--cc',
    required=True,
    help="The path to the C compiler that we're testing.")
runTestSuite.add_argument(
    # For symmetry, we also provide a --cxx argument, but this one isn't
    # required since LNT tries to guess it based on the value of --cc.
    '--cxx',
    required=False,
    help="The path to the C++ compiler that we're testing.")
runTestSuite.add_argument(
    '-n', '--dry-run',
    dest='dry',
    action='store_true',
    default=False,
    help="Print the commands instead of executing them.")
runTestSuite.set_defaults(run_command=run_the_test_suite)

args = options.parse_args()
if args.subcommand == "projects" and args.add and not args.repos:
    projs.error(
        "When adding a subproject you must also pass the --repos argument")
args.run_command(args)