aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDiana Picus <diana.picus@linaro.org>2018-01-23 16:41:59 +0100
committerDiana Picus <diana.picus@linaro.org>2018-01-31 14:45:59 +0000
commitb368cb6384312a8dd26172202d6e33bea6b472b9 (patch)
treecfaea9ecf5689a86c9b6acf2dea9dba777915269
parent9276b78dd111f2ee91eca495d52c4a5c4f6893cb (diff)
downloadlinaro-scripts-b368cb6384312a8dd26172202d6e33bea6b472b9.tar.gz
Add support for running the test-suite
Add a llvm.py subcommand 'run-test-suite' which runs the test-suite in a given sandbox. This uses the 'test-suite' producer (as opposed to the old 'nt' one) since this is what the buildbots use these days. We don't update the helpers in the 'test-suite' directory because they are obsolete (and using the 'nt' producer) and hopefully nobody has been using them lately. Instead we add a really simple helper 'llvm-test-suite' which runs the test-suite with the compiler from $LLVM_BLD. We keep this helper simple for the time being, but in the future we may want to add more features here (e.g. a benchmarking mode, core/number of threads detection etc). We don't currently setup the sandbox. That will be a different commit. Change-Id: I2bc661aa483f1ba40cb9bdc01f8d297cd1fb99d0
-rwxr-xr-xhelpers/llvm-test-suite27
-rw-r--r--modules/llvm.py29
-rw-r--r--scripts/llvm.py65
-rw-r--r--tests/cli/testllvmtestsuite.py129
-rw-r--r--tests/unittests/testruntestsuite.py63
5 files changed, 313 insertions, 0 deletions
diff --git a/helpers/llvm-test-suite b/helpers/llvm-test-suite
new file mode 100755
index 0000000..283e52f
--- /dev/null
+++ b/helpers/llvm-test-suite
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# This script helps you run the test-suite with the compiler from $LLVM_BLD. It
+# assumes virtualenv is already installed.
+# Any flags passed to the script will be passed down to llvm.py run-test-suite.
+
+. llvm-common
+
+progdir=$(dirname $0)
+llvmtool=$progdir/../scripts/llvm.py
+
+safe_run verify_env
+build_dir=$LLVM_BLD
+env=$LLVM_BLD/../
+
+sandbox=$env/sandbox
+testsuite=$LLVM_ROOT/repos/test-suite
+lnt=$LLVM_ROOT/repos/lnt
+lit=$LLVM_BLD/bin/llvm-lit
+clang=$LLVM_BLD/bin/clang
+
+# TODO: There should be a llvm.py subcommand for setting up the sandbox
+safe_run virtualenv $sandbox
+safe_run $sandbox/bin/python $lnt/setup.py develop
+
+safe_run python3 $llvmtool run-test-suite --sandbox $sandbox \
+ --test-suite $testsuite --use-lit $lit --cc $clang --cxx $clang++ "$@"
diff --git a/modules/llvm.py b/modules/llvm.py
index 0d453ff..e70e411 100644
--- a/modules/llvm.py
+++ b/modules/llvm.py
@@ -361,3 +361,32 @@ def build_llvm(commandConsumer, buildDir, flags=[]):
command = [buildTool] + flags
commandConsumer.consume(command, buildDir)
+
+
+def run_test_suite(commandConsumer, sandbox, testsuite, lit, flags=[]):
+ """
+ Generate the command needed for running the test-suite with the given
+ parameters and pass it to the 'commandConsumer'.
+
+ The 'commandConsumer' should have a 'consume' method taking two
+ parameters: the command to be consumed (in the form of a list) and the
+ directory where the command should be run. Any exceptions that may be
+ raised by that method should be handled by the calling code.
+
+ The 'sandbox' should be the path to an already-setup virtualenv for running
+ the test-suite. The 'testsuite' and 'lit' should be the paths to the
+ test-suite repository and to the 'llvm-lit' binary (usually found in the
+ LLVM build directory under 'bin'), respectively.
+ """
+
+ python = os.path.join(sandbox, "bin", "python")
+ lnt = os.path.join(sandbox, "bin", "lnt")
+
+ command = [python, lnt, "runtest", "test-suite",
+ "--sandbox={}".format(sandbox),
+ "--test-suite={}".format(testsuite),
+ "--use-lit={}".format(lit)] + flags
+
+ # This doesn't need to run in any particular directory, just use the current
+ # working directory.
+ commandConsumer.consume(command, None)
diff --git a/scripts/llvm.py b/scripts/llvm.py
index ff8ecb1..be4628f 100644
--- a/scripts/llvm.py
+++ b/scripts/llvm.py
@@ -8,6 +8,7 @@ from modules.llvm import build_llvm
from modules.llvm import LLVMBuildConfig
from modules.llvm import LLVMSubproject
from modules.llvm import LLVMSourceConfig
+from modules.llvm import run_test_suite
from modules.utils import CommandPrinter
from modules.utils import CommandRunner
from modules.utils import get_remote_branch
@@ -132,6 +133,23 @@ def run_build(args):
die("Failed to build {} because:\n{}".format(args.build, str(exc)))
+def run_the_test_suite(args):
+ """Run the test-suite in a given sandbox."""
+ if args.dry:
+ consumer = CommandPrinter()
+ else:
+ consumer = CommandRunner()
+
+ compilers = ["--cc={}".format(args.cc)]
+ if args.cxx:
+ compilers.append("--cxx={}".format(args.cxx))
+
+ try:
+ run_test_suite(consumer, args.sandbox, args.testsuite, args.lit,
+ compilers + args.flags)
+ except RuntimeError as exc:
+ die("Failed to run the test-suite because:\n{}".format(str(exc)))
+
##########################################################################
# Command line parsing #
##########################################################################
@@ -261,6 +279,53 @@ build.add_argument(
"'--build-flag=-FLAG' to pass it.")
build.set_defaults(run_command=run_build)
+# Subcommand for running the test-suite
+runTestSuite = subcommands.add_parser(
+ 'run-test-suite',
+ help="Run the test-suite in the given sandbox.")
+runTestSuite.add_argument(
+ '--sandbox',
+ required=True,
+ help="Path to the sandbox. It must point to a virtualenv with a LNT setup.")
+runTestSuite.add_argument(
+ '--test-suite',
+ dest="testsuite",
+ required=True,
+ help="Path to the test-suite repo.")
+runTestSuite.add_argument(
+ '--use-lit',
+ dest="lit",
+ required=True,
+ help="Path to llvm-lit.")
+runTestSuite.add_argument(
+ '--lnt-flag',
+ dest='flags',
+ metavar='FLAG',
+ default=[],
+ action='append',
+ help="Additional flags to be passed to LNT when running the test-suite."
+ "May be passed several times. If your flag starts with a '-', use "
+ "'--lnt-flag=-FLAG' to pass it.")
+runTestSuite.add_argument(
+ # We can pass --cc through the --lnt-flag interface, but we generally won't
+ # want to test the system compiler, so force the user to be specific.
+ '--cc',
+ required=True,
+ help="The path to the C compiler that we're testing.")
+runTestSuite.add_argument(
+ # For symmetry, we also provide a --cxx argument, but this one isn't
+ # required since LNT tries to guess it based on the value of --cc.
+ '--cxx',
+ required=False,
+ help="The path to the C++ compiler that we're testing.")
+runTestSuite.add_argument(
+ '-n', '--dry-run',
+ dest='dry',
+ action='store_true',
+ default=False,
+ help="Print the commands instead of executing them.")
+runTestSuite.set_defaults(run_command=run_the_test_suite)
+
args = options.parse_args()
if args.subcommand == "projects" and args.add and not args.repos:
projs.error(
diff --git a/tests/cli/testllvmtestsuite.py b/tests/cli/testllvmtestsuite.py
new file mode 100644
index 0000000..be29ae1
--- /dev/null
+++ b/tests/cli/testllvmtestsuite.py
@@ -0,0 +1,129 @@
+"""Command line interface tests for llvm.py run-test-suite.
+
+Note that although this uses the unittest framework, it does *not* contain unit
+tests.
+
+"""
+
+from llvmtestcase import LLVMTestCase, require_command_arg, debug
+
+from re import escape
+
+
+class Testllvmruntestsuite(LLVMTestCase):
+
+ @classmethod
+ def llvm_run_test_suite(cls, *args, **kwargs):
+ return cls.command_with_defaults("run-test-suite", *args, **kwargs)
+
+ @require_command_arg("--sandbox")
+ def test_sandbox_is_compulsory(self):
+ """Test that we get an error if we don't pass the path to the sandbox."""
+ self.run_with_output(self.llvm_run_test_suite("--test-suite=somewhere",
+ "--use-lit=somewhere"
+ "--cc=somewhere"))
+
+ @require_command_arg("--test-suite")
+ def test_test_suite_repo_is_compulsory(self):
+ """
+ Test that we get an error if we don't pass the path to the test-suite
+ repo.
+ """
+ self.run_with_output(self.llvm_run_test_suite("--sandbox=somewhere",
+ "--use-lit=somewhere"
+ "--cc=somewhere"))
+
+ @require_command_arg("--use-lit")
+ def test_llvm_lit_is_compulsory(self):
+ """Test that we get an error if we don't pass the path to llvm-lit."""
+ self.run_with_output(self.llvm_run_test_suite("--sandbox=somewhere",
+ "--test-suite=somewhere"
+ "--cc=somewhere"))
+
+ @require_command_arg("--cc")
+ def test_cc_is_compulsory(self):
+ """
+ Test that we get an error if we don't pass the path to the C compiler.
+ """
+ self.run_with_output(self.llvm_run_test_suite("--sandbox=somewhere",
+ "--test-suite=somewhere",
+ "--use-lit=somewhere"))
+
+ def test_basic_command(self):
+ """
+ Test that we can run a test-suite command with only the required args.
+ """
+ sandbox = "path/to/sandbox"
+ testsuite = "path/to/testuite"
+ lit = "path/to/lit"
+ cc = "path/to/clang"
+
+ output = self.run_with_output(
+ self.llvm_run_test_suite(
+ "--dry-run",
+ "--sandbox", sandbox,
+ "--test-suite", testsuite,
+ "--use-lit", lit,
+ "--cc", cc))
+
+ expectedCommand = escape(
+ "{0}/bin/python {0}/bin/lnt runtest test-suite "
+ "--sandbox={0} --test-suite={1} --use-lit={2} --cc={3}\n".format(
+ sandbox, testsuite, lit, cc))
+ # We don't care about the directory where this is running.
+ self.assertRegex(
+ output, "(.*)\$ {}".format(expectedCommand))
+
+ def test_custom_flags(self):
+ """Test that we can run a test-suite command with custom flags."""
+ sandbox = "path/to/sandbox"
+ testsuite = "path/to/testuite"
+ lit = "path/to/lit"
+ cc = "path/to/clang"
+
+ output = self.run_with_output(
+ self.llvm_run_test_suite(
+ "--dry-run",
+ "--sandbox", sandbox,
+ "--test-suite", testsuite,
+ "--use-lit", lit,
+ "--cc", cc,
+ "--lnt-flag=--cppflags='-mcpu=cortex-a15 -marm'",
+ "--lnt-flag=--threads=4"))
+
+ expectedCommand = escape(
+ "{0}/bin/python {0}/bin/lnt runtest test-suite "
+ "--sandbox={0} --test-suite={1} --use-lit={2} --cc={3} "
+ "--cppflags='-mcpu=cortex-a15 -marm' --threads=4\n".format(
+ sandbox, testsuite, lit, cc))
+ # We don't care about the directory where this is running.
+ self.assertRegex(
+ output, "(.*)\$ {}".format(expectedCommand))
+
+ def test_cxx(self):
+ """Test that we can specify --cxx if we want to."""
+ sandbox = "path/to/sandbox"
+ testsuite = "path/to/testuite"
+ lit = "path/to/lit"
+ cc = "path/to/clang"
+ cxx = "path/to/clang++"
+
+ output = self.run_with_output(
+ self.llvm_run_test_suite(
+ "--dry-run",
+ "--sandbox", sandbox,
+ "--test-suite", testsuite,
+ "--use-lit", lit,
+ "--cc", cc,
+ "--cxx", cxx,
+ "--lnt-flag=--cppflags='-mcpu=cortex-a15 -marm'",
+ "--lnt-flag=--threads=4"))
+
+ expectedCommand = escape(
+ "{0}/bin/python {0}/bin/lnt runtest test-suite "
+ "--sandbox={0} --test-suite={1} --use-lit={2} --cc={3} --cxx={4} "
+ "--cppflags='-mcpu=cortex-a15 -marm' --threads=4\n".format(
+ sandbox, testsuite, lit, cc, cxx))
+ # We don't care about the directory where this is running.
+ self.assertRegex(
+ output, "(.*)\$ {}".format(expectedCommand))
diff --git a/tests/unittests/testruntestsuite.py b/tests/unittests/testruntestsuite.py
new file mode 100644
index 0000000..eb82ca6
--- /dev/null
+++ b/tests/unittests/testruntestsuite.py
@@ -0,0 +1,63 @@
+from modules.llvm import run_test_suite
+
+import os
+
+from unittest import TestCase
+from unittest.mock import MagicMock
+
+
+class TestRunTestSuite(TestCase):
+
+ def test_default_run(self):
+ """Test that we can run the test-suite without any custom flags."""
+
+ sandbox = "path/to/sandbox"
+ testsuite = "path/to/testsuite"
+ lit = "path/to/lit"
+
+ consumer = MagicMock()
+ run_test_suite(consumer, sandbox, testsuite, lit)
+ command, directory = consumer.consume.call_args[0]
+
+ self.assertIsNone(directory)
+ self.assertEqual(
+ command,
+ [
+ os.path.join(sandbox, "bin", "python"),
+ os.path.join(sandbox, "bin", "lnt"),
+ "runtest",
+ "test-suite",
+ "--sandbox={}".format(sandbox),
+ "--test-suite={}".format(testsuite),
+ "--use-lit={}".format(lit),
+ ])
+
+ def test_run_with_flags(self):
+ """Test that we can run the test-suite with custom flags."""
+
+ sandbox = "path/to/sandbox"
+ testsuite = "path/to/testsuite"
+ lit = "path/to/lit"
+
+ consumer = MagicMock()
+ run_test_suite(consumer, sandbox, testsuite, lit,
+ ["--cc", "path/to/clang", "--cxx", "path/to/clang++",
+ "--threads=4", "--build-threads=1"])
+ command, directory = consumer.consume.call_args[0]
+
+ self.assertIsNone(directory)
+ self.assertEqual(
+ command,
+ [
+ os.path.join(sandbox, "bin", "python"),
+ os.path.join(sandbox, "bin", "lnt"),
+ "runtest",
+ "test-suite",
+ "--sandbox={}".format(sandbox),
+ "--test-suite={}".format(testsuite),
+ "--use-lit={}".format(lit),
+ "--cc", "path/to/clang",
+ "--cxx", "path/to/clang++",
+ "--threads=4",
+ "--build-threads=1",
+ ])