tests/cli/testbuildandtest.py

"""Command line interface tests for llvm.py build-and-test.

Note that although this uses the unittest framework, it does *not* contain unit
tests.

"""
import os
from subprocess import CalledProcessError
from tempfile import NamedTemporaryFile

from llvmtestcase import LLVMTestCase, require_command_arg, debug


class Testllvmbuildandtest(LLVMTestCase):

    @classmethod
    def llvm_build_and_test(cls, *args, **kwargs):
        return cls.command_with_defaults("build-and-test", *args, **kwargs)

    @classmethod
    def get_temp_file_with_content(cls, content):
        """Return a temporary file with the given content."""
        theFile = NamedTemporaryFile(mode='wt')
        print(content, file=theFile, flush=True)
        return theFile

    @require_command_arg("--repos")
    def test_repos_dir_is_compulsory(self):
        """Test that we get an error if we don't pass the path to the repos."""
        self.run_with_output(
            self.llvm_build_and_test("--source-dir=somewhere",
                                     "--stage1-build-dir=elsewhere"))

    @require_command_arg("--source-dir")
    def test_source_dir_is_compulsory(self):
        """Test that we get an error if we don't pass the path to the sources."""
        self.run_with_output(
            self.llvm_build_and_test("--repos-dir=somewhere",
                                     "--stage1-build-dir=elsewhere"))

    @require_command_arg("--stage1-build-dir")
    def test_stage1_build_dir_is_compulsory(self):
        """
        Test that we get an error if we don't pass the path to the stage 1
        build directory.
        """
        self.run_with_output(
            self.llvm_build_and_test("--repos-dir=somewhere",
                                     "--source-dir=elsewhere"))

    def test_default_stage1(self):
        """
        Test that we dump the correct commands for a single stage build of LLVM.
        """
        reposDir = "path-to-repos"
        sourceDir = "path-to-sources"
        buildDir = "path-to-stage1"

        output = self.run_with_output(
            self.llvm_build_and_test(
                "--dry-run",
                "--repos-dir", reposDir,
                "--source-dir", sourceDir,
                "--stage1-build-dir", buildDir))

        commands = output.splitlines()

        self.assertRegex(commands[0],
                         "{build}\$ cmake -G Ninja .* {sources}".format(
                             build=buildDir, sources=sourceDir))

        self.assertRegex(commands[1],
                         "{build}\$ ninja".format(build=buildDir))

    def test_custom_stage1(self):
        """Test that we can customize the first stage of the build."""
        reposDir = "path-to-repos"
        sourceDir = "path-to-sources"
        buildDir = "path-to-stage1"

        output = self.run_with_output(
            self.llvm_build_and_test(
                "--dry-run",
                "--repos-dir", reposDir,
                "--source-dir", sourceDir,
                "--stage1-build-dir", buildDir,
                "--stage1-subproject", "clang",
                "--stage1-subproject", "compiler-rt",
                "--stage1-cmake-def", "CMAKE_CXX_FLAGS=-marm",
                "--stage1-cmake-def", "LLVM_ENABLE_ASSERTIONS=True",
                "--stage1-build-flag=-j8",
                "--stage1-build-flag", "check-all"))

        commands = output.splitlines()

        self.assertRegex(commands[0],
                         "{build}\$ cmake -G Ninja .* {sources}".format(
                             build=buildDir, sources=sourceDir))

        self.assertIn("-DLLVM_TOOL_CLANG_BUILD=ON", commands[0])
        self.assertIn("-DLLVM_TOOL_COMPILER_RT_BUILD=ON", commands[0])
        self.assertIn("-DLLVM_TOOL_LIBCXX_BUILD=OFF", commands[0])
        self.assertIn("-DLLVM_TOOL_LLDB_BUILD=OFF", commands[0])

        self.assertIn("-DCMAKE_CXX_FLAGS=-marm", commands[0])
        self.assertIn("-DLLVM_ENABLE_ASSERTIONS=True", commands[0])

        self.assertRegex(commands[1],
                         "{build}\$ ninja -j8 check-all".format(build=buildDir))


    def test_stage1_and_default_testsuite(self):
        """
        Test that we dump the correct commands for a single stage build of LLVM
        and a run of the test-suite with the resulting compiler.
        """
        reposDir = "path-to-repos"
        sourceDir = "path-to-sources"
        buildDir = "path-to-stage1"
        sandboxDir = "path-to-sandbox"

        testSuiteDir = os.path.join(reposDir, "test-suite")
        lntDir = os.path.join(reposDir, "lnt")

        output = self.run_with_output(
            self.llvm_build_and_test(
                "--dry-run",
                "--repos-dir", reposDir,
                "--source-dir", sourceDir,
                "--stage1-build-dir", buildDir,
                "--stage1-subproject", "clang",
                "--enable-test-suite",
                "--sandbox", sandboxDir))

        commands = output.splitlines()

        self.assertRegex(commands[0],
                         "{build}\$ cmake -G Ninja .* {sources}".format(
                             build=buildDir, sources=sourceDir))

        self.assertRegex(commands[1],
                         "{build}\$ ninja".format(build=buildDir))

        self.assertRegex(
            commands[2], ".*\$ virtualenv {sandbox}".format(sandbox=sandboxDir))

        self.assertRegex(
            commands[3],
            ".*\$ {sandbox}/bin/python {lnt}/setup.py develop".format(
                sandbox=sandboxDir,
                lnt=lntDir))

        self.assertRegex(
            commands[4],
            ".*\$ {sandbox}/bin/python {sandbox}/bin/lnt runtest test-suite "
            "--sandbox={sandbox} --test-suite={testsuite} "
            "--use-lit={build}/bin/llvm-lit --cc={build}/bin/clang".format(
                sandbox=sandboxDir, testsuite=testSuiteDir, build=buildDir))

    def test_stage1_and_custom_testsuite(self):
        """Test that we can add custom flags to our test-suite run."""
        reposDir = "path-to-repos"
        sourceDir = "path-to-sources"
        buildDir = "path-to-stage1"
        sandboxDir = "path-to-sandbox"

        testSuiteDir = os.path.join(reposDir, "test-suite")
        lntDir = os.path.join(reposDir, "lnt")

        output = self.run_with_output(
            self.llvm_build_and_test(
                "--dry-run",
                "--repos-dir", reposDir,
                "--source-dir", sourceDir,
                "--stage1-build-dir", buildDir,
                "--stage1-subproject", "clang",
                "--enable-test-suite",
                "--sandbox", sandboxDir,
                "--lnt-flag=--threads=4",
                "--lnt-flag=--cppflags",
                "--lnt-flag", "'-mcpu=cortex-a15 -marm'"))

        commands = output.splitlines()

        self.assertRegex(commands[0],
                         "{build}\$ cmake -G Ninja .* {sources}".format(
                             build=buildDir, sources=sourceDir))

        self.assertRegex(commands[1],
                         "{build}\$ ninja".format(build=buildDir))

        self.assertRegex(
            commands[2], ".*\$ virtualenv {sandbox}".format(sandbox=sandboxDir))

        self.assertRegex(
            commands[3],
            ".*\$ {sandbox}/bin/python {lnt}/setup.py develop".format(
                sandbox=sandboxDir,
                lnt=lntDir))

        self.assertRegex(
            commands[4],
            ".*\$ {sandbox}/bin/python {sandbox}/bin/lnt runtest test-suite "
            "--sandbox={sandbox} --test-suite={testsuite} "
            "--use-lit={build}/bin/llvm-lit --cc={build}/bin/clang "
            "--threads=4 --cppflags '-mcpu=cortex-a15 -marm'".format(
                sandbox=sandboxDir, testsuite=testSuiteDir, build=buildDir))

    def test_stage1_needs_clang_if_testsuite_is_enabled(self):
        """
        Test that we error out early on if we enable the test-suite but don't
        add clang to stage 1.
        """
        reposDir = "path-to-repos"
        sourceDir = "path-to-sources"
        buildDir = "path-to-stage1"
        sandboxDir = "path-to-sandbox"

        testSuiteDir = os.path.join(reposDir, "test-suite")
        lntDir = os.path.join(reposDir, "lnt")

        with self.assertRaises(CalledProcessError) as context:
            output = self.run_with_output(
                self.llvm_build_and_test(
                    "--dry-run",
                    "--repos-dir", reposDir,
                    "--source-dir", sourceDir,
                    "--stage1-build-dir", buildDir,
                    "--enable-test-suite",
                    "--sandbox", sandboxDir))

        self.assertIn(
            "Can't enable the test-suite if stage 1 doesn't build clang",
            str(context.exception.output))

    def test_default_stage2(self):
        """
        Test that we dump the correct commands for a 2-stage build of LLVM.
        """
        reposDir = "path-to-repos"
        sourceDir = "path-to-sources"
        buildDir1 = "path-to-stage1"
        buildDir2 = "path-to-stage2"

        output = self.run_with_output(
            self.llvm_build_and_test(
                "--dry-run",
                "--repos-dir", reposDir,
                "--source-dir", sourceDir,
                "--stage1-build-dir", buildDir1,
                "--stage1-subproject", "clang",
                "--stage2-build-dir", buildDir2))

        commands = output.splitlines()

        self.assertRegex(
            commands[0],
            "{stage1}\$ cmake -G Ninja .* {sources}".format(
                stage1=buildDir1, sources=sourceDir))

        self.assertRegex(
            commands[1],
            "{stage1}\$ ninja".format(stage1=buildDir1))

        self.assertRegex(
            commands[2], "{stage2}\$ cmake -G Ninja .* "
            "-DCMAKE_C_COMPILER={stage1}/bin/clang "
            "-DCMAKE_CXX_COMPILER={stage1}/bin/clang\+\+ {sources}".format(
                stage1=buildDir1, stage2=buildDir2, sources=sourceDir))

        self.assertRegex(
            commands[3],
            "{stage2}\$ ninja".format(stage2=buildDir2))

    def test_custom_stage1_default_stage2(self):
        """
        Test that we preserve the subprojects, but not the CMake or build flags.
        """
        reposDir = "path-to-repos"
        sourceDir = "path-to-sources"
        buildDir1 = "path-to-stage1"
        buildDir2 = "path-to-stage2"

        output = self.run_with_output(
            self.llvm_build_and_test(
                "--dry-run",
                "--repos-dir", reposDir,
                "--source-dir", sourceDir,
                "--stage1-build-dir", buildDir1,
                "--stage1-subproject", "clang",
                "--stage1-subproject", "compiler-rt",
                "--stage1-cmake-def", "CMAKE_CXX_FLAGS=-marm",
                "--stage1-cmake-def", "LLVM_ENABLE_ASSERTIONS=True",
                "--stage1-build-flag=-j8",
                "--stage1-build-flag", "check-all",
                "--stage2-build-dir", buildDir2))

        commands = output.splitlines()

        self.assertRegex(commands[0],
                         "{build}\$ cmake -G Ninja .* {sources}".format(
                             build=buildDir1, sources=sourceDir))

        self.assertIn("-DLLVM_TOOL_CLANG_BUILD=ON", commands[0])
        self.assertIn("-DLLVM_TOOL_COMPILER_RT_BUILD=ON", commands[0])
        self.assertIn("-DLLVM_TOOL_LIBCXX_BUILD=OFF", commands[0])
        self.assertIn("-DLLVM_TOOL_LLDB_BUILD=OFF", commands[0])

        self.assertIn("-DCMAKE_CXX_FLAGS=-marm", commands[0])
        self.assertIn("-DLLVM_ENABLE_ASSERTIONS=True", commands[0])

        self.assertRegex(commands[1],
                         "{build}\$ ninja -j8 check-all".format(build=buildDir1))

        self.assertRegex(
            commands[2], "{stage2}\$ cmake -G Ninja .* "
            "-DCMAKE_C_COMPILER={stage1}/bin/clang "
            "-DCMAKE_CXX_COMPILER={stage1}/bin/clang\+\+ {sources}".format(
                stage1=buildDir1, stage2=buildDir2, sources=sourceDir))

        self.assertIn("-DLLVM_TOOL_CLANG_BUILD=ON", commands[2])
        self.assertIn("-DLLVM_TOOL_COMPILER_RT_BUILD=ON", commands[2])
        self.assertIn("-DLLVM_TOOL_LIBCXX_BUILD=OFF", commands[2])
        self.assertIn("-DLLVM_TOOL_LLDB_BUILD=OFF", commands[2])

        self.assertNotIn("-DCMAKE_CXX_FLAGS=-marm", commands[2])
        self.assertNotIn("-DLLVM_ENABLE_ASSERTIONS=True", commands[2])

        self.assertRegex(
            commands[3],
            "{stage2}\$ ninja".format(stage2=buildDir2))

    def test_custom_both_stages(self):
        """
        Test that we get the correct commands when trying to customize both
        stage 1 and stage 2.
        """
        reposDir = "path-to-repos"
        sourceDir = "path-to-sources"
        buildDir1 = "path-to-stage1"
        buildDir2 = "path-to-stage2"

        output = self.run_with_output(
            self.llvm_build_and_test(
                "--dry-run",
                "--repos-dir", reposDir,
                "--source-dir", sourceDir,
                "--stage1-build-dir", buildDir1,
                "--stage1-subproject", "clang",
                "--stage1-subproject", "compiler-rt",
                "--stage1-cmake-def", "CMAKE_CXX_FLAGS=-marm",
                "--stage1-cmake-def", "LLVM_ENABLE_ASSERTIONS=True",
                "--stage1-build-flag=-j8",
                "--stage2-build-dir", buildDir2,
                "--stage2-subproject", "clang",
                "--stage2-subproject", "libcxx",
                "--stage2-cmake-def", "CMAKE_BUILD_TYPE=MinSizeRel",
                "--stage2-build-flag", "check-all"))

        commands = output.splitlines()

        self.assertRegex(commands[0],
                         "{build}\$ cmake -G Ninja .* {sources}".format(
                             build=buildDir1, sources=sourceDir))

        self.assertIn("-DLLVM_TOOL_CLANG_BUILD=ON", commands[0])
        self.assertIn("-DLLVM_TOOL_COMPILER_RT_BUILD=ON", commands[0])
        self.assertIn("-DLLVM_TOOL_LIBCXX_BUILD=OFF", commands[0])
        self.assertIn("-DLLVM_TOOL_LLDB_BUILD=OFF", commands[0])

        self.assertIn("-DCMAKE_CXX_FLAGS=-marm", commands[0])
        self.assertIn("-DLLVM_ENABLE_ASSERTIONS=True", commands[0])
        self.assertNotIn("-DCMAKE_BUILD_TYPE=MinSizeRel", commands[0])

        self.assertRegex(commands[1],
                         "{build}\$ ninja -j8".format(build=buildDir1))

        self.assertRegex(
            commands[2], "{stage2}\$ cmake -G Ninja .* "
            "-DCMAKE_C_COMPILER={stage1}/bin/clang "
            "-DCMAKE_CXX_COMPILER={stage1}/bin/clang\+\+ {sources}".format(
                stage1=buildDir1, stage2=buildDir2, sources=sourceDir))

        self.assertIn("-DLLVM_TOOL_CLANG_BUILD=ON", commands[2])
        self.assertIn("-DLLVM_TOOL_COMPILER_RT_BUILD=OFF", commands[2])
        self.assertIn("-DLLVM_TOOL_LIBCXX_BUILD=ON", commands[2])
        self.assertIn("-DLLVM_TOOL_LLDB_BUILD=OFF", commands[2])

        self.assertNotIn("-DCMAKE_CXX_FLAGS=-marm", commands[2])
        self.assertNotIn("-DLLVM_ENABLE_ASSERTIONS=True", commands[2])
        self.assertIn("-DCMAKE_BUILD_TYPE=MinSizeRel", commands[2])

        self.assertRegex(
            commands[3],
            "{stage2}\$ ninja check-all".format(stage2=buildDir2))

    def test_stage1_needs_clang_if_stage2_is_enabled(self):
        """
        Test that we can't enable stage 2 unless stage 1 builds clang.
        """
        reposDir = "path-to-repos"
        sourceDir = "path-to-sources"
        buildDir1 = "path-to-stage1"
        buildDir2 = "path-to-stage2"

        with self.assertRaises(CalledProcessError) as context:
            output = self.run_with_output(
                self.llvm_build_and_test(
                    "--dry-run",
                    "--repos-dir", reposDir,
                    "--source-dir", sourceDir,
                    "--stage1-build-dir", buildDir1,
                    "--stage2-build-dir", buildDir2))

        self.assertIn(
            "Can't enable stage 2 if stage 1 doesn't build clang",
            str(context.exception.output))

    def test_stage2_and_testsuite(self):
        """
        Test that we dump the correct commands for a 2-stage build of LLVM and a
        run of the test-suite with the resulting compiler.
        """
        reposDir = "path-to-repos"
        sourceDir = "path-to-sources"
        buildDir1 = "path-to-stage1"
        buildDir2 = "path-to-stage2"
        sandboxDir = "path-to-sandbox"

        testSuiteDir = os.path.join(reposDir, "test-suite")
        lntDir = os.path.join(reposDir, "lnt")

        output = self.run_with_output(
            self.llvm_build_and_test(
                "--dry-run",
                "--repos-dir", reposDir,
                "--source-dir", sourceDir,
                "--stage1-build-dir", buildDir1,
                "--stage1-subproject", "clang",
                "--stage2-build-dir", buildDir2,
                "--enable-test-suite",
                "--sandbox", sandboxDir))

        commands = output.splitlines()

        self.assertRegex(
            commands[0],
            "{stage1}\$ cmake -G Ninja .* {sources}".format(
                stage1=buildDir1,
                sources=sourceDir))

        self.assertRegex(
            commands[1],
            "{stage1}\$ ninja".format(stage1=buildDir1))

        self.assertRegex(
            commands[2], "{stage2}\$ cmake -G Ninja .* "
            "-DCMAKE_C_COMPILER={stage1}/bin/clang "
            "-DCMAKE_CXX_COMPILER={stage1}/bin/clang\+\+ {sources}".format(
                stage1=buildDir1, stage2=buildDir2, sources=sourceDir))

        self.assertRegex(
            commands[3],
            "{stage2}\$ ninja".format(stage2=buildDir2))

        self.assertRegex(
            commands[4],
            ".*\$ virtualenv {sandbox}".format(sandbox=sandboxDir))

        self.assertRegex(
            commands[5],
            ".*\$ {sandbox}/bin/python {lnt}/setup.py develop".format(
                sandbox=sandboxDir,
                lnt=lntDir))

        self.assertRegex(
            commands[6],
            ".*\$ {sandbox}/bin/python {sandbox}/bin/lnt runtest test-suite "
            "--sandbox={sandbox} --test-suite={testsuite} "
            "--use-lit={build}/bin/llvm-lit --cc={build}/bin/clang".format(
                sandbox=sandboxDir, testsuite=testSuiteDir, build=buildDir2))

    def test_read_flags_from_file(self):
        """Test that we can read our flags from a configuration file."""
        reposDir = "path-to-repos"
        sourceDir = "path-to-sources"
        buildDir = "path-to-stage1"

        flagsInFile = ("--dry-run\n"
                       "--repos-dir\n"
                       "{repos}\n"
                       "--source-dir\n"
                       "{sources}\n"
                       "--stage1-build-dir\n"
                       "{build}").format(
            repos=reposDir, sources=sourceDir, build=buildDir)

        with self.get_temp_file_with_content(flagsInFile) as configFile:
            output = self.run_with_output(
                self.llvm_build_and_test("@{}".format(configFile.name)))

        commands = output.splitlines()

        self.assertRegex(commands[0],
                         "{build}\$ cmake -G Ninja .* {sources}".format(
                             build=buildDir, sources=sourceDir))

        self.assertRegex(commands[1],
                         "{build}\$ ninja".format(build=buildDir))

    def test_override_flags_from_file(self):
        """
        Test that we can combine flags from a configuration file and command
        line flags. The command line flags that come before the name of the
        configuration file should be overriden by those from the file, whereas
        command line flags that come after the name of the configuration file
        should override the values found in the file. For arguments that can be
        passed multiple times, all the values are collected (both from the
        command line and from the config file). They should however appear in
        the order that they were given.
        """
        reposDir = "path-to-repos"
        sourceDir = "path-to-sources"
        buildDir = "path-to-stage1"

        overridenSourceDir = "overriden-sources"
        overridenBuildDir = "overriden-build"

        flagsInFile = (
            "--dry-run\n"
            "--repos-dir\n"
            "{repos}\n"
            "--source-dir\n"
            "{sources}\n"
            "--stage1-build-dir\n"
            "{build}\n"
            "--stage1-subproject\n"
            "clang\n"
            "--stage1-cmake-def\n"
            "CMAKE_CXX_FLAGS=-mthumb").format(repos=reposDir, sources=sourceDir,
                                              build=overridenBuildDir)

        with self.get_temp_file_with_content(flagsInFile) as configFile:
            output = self.run_with_output(
                self.llvm_build_and_test(
                    # This should be overriden by the value in the config file.
                    "--source-dir", overridenSourceDir,
                    # This should be appended to the value in the config file.
                    "--stage1-subproject", "lld",
                    # The config file.
                    "@{}".format(configFile.name),
                    # This should override the value in the config file.
                    "--stage1-build-dir", buildDir,
                    # These should be appended to the values in the config file.
                    "--stage1-subproject", "compiler-rt",
                    "--stage1-cmake-def", "CMAKE_CXX_FLAGS=-marm",
                    "--stage1-cmake-def", "LLVM_ENABLE_ASSERTIONS=True"))

        commands = output.splitlines()

        self.assertRegex(commands[0],
                         "{build}\$ cmake -G Ninja .* {sources}".format(
                             build=buildDir, sources=sourceDir))

        self.assertIn("-DLLVM_TOOL_CLANG_BUILD=ON", commands[0])
        self.assertIn("-DLLVM_TOOL_LLD_BUILD=ON", commands[0])
        self.assertIn("-DLLVM_TOOL_COMPILER_RT_BUILD=ON", commands[0])
        self.assertIn("-DLLVM_TOOL_LIBCXX_BUILD=OFF", commands[0])
        self.assertIn("-DLLVM_TOOL_LLDB_BUILD=OFF", commands[0])

        self.assertIn("-DCMAKE_CXX_FLAGS=-marm", commands[0])
        self.assertIn("-DCMAKE_CXX_FLAGS=-mthumb", commands[0])
        self.assertLess(commands[0].find("-DCMAKE_CXX_FLAGS=-mthumb"),
                        commands[0].find("-DCMAKE_CXX_FLAGS=-marm"))
        self.assertIn("-DLLVM_ENABLE_ASSERTIONS=True", commands[0])

        self.assertRegex(commands[1],
                         "{build}\$ ninja".format(build=buildDir))