tools/test.py

#!/usr/bin/env python2.7

# Copyright 2015, VIXL authors
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions and the following disclaimer in the documentation
#     and/or other materials provided with the distribution.
#   * Neither the name of ARM Limited nor the names of its contributors may be
#     used to endorse or promote products derived from this software without
#     specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import argparse
import fcntl
import git
import itertools
import multiprocessing
import os
from os.path import join
import platform
import re
import subprocess
import sys
import time

import config
import clang_format
import lint
import printer
import test
import threaded_tests
import util


dir_root = config.dir_root

def Optionify(name):
  return '--' + name


# The options that can be tested are abstracted to provide an easy way to add
# new ones.
# Environment options influence the environment. They can be used for example to
# set the compiler used.
# Build options are options passed to scons, with a syntax like `scons opt=val`
# Runtime options are options passed to the test program.
# See the definition of `test_options` below.

# 'all' is a special value for the options. If specified, all other values of
# the option are tested.
class TestOption(object):
  type_environment = 'type_environment'
  type_build = 'type_build'
  type_run = 'type_run'

  def __init__(self, option_type, name, help,
               val_test_choices, val_test_default = None,
               # If unset, the user can pass any value.
               strict_choices = True, test_independently = False):
    self.name = name
    self.option_type = option_type
    self.help = help
    self.val_test_choices = val_test_choices
    self.strict_choices = strict_choices
    self.test_independently = test_independently
    if val_test_default is not None:
      self.val_test_default = val_test_default
    else:
      self.val_test_default = val_test_choices[0]

  def ArgList(self, to_test):
    res = []
    if to_test == 'all':
      for value in self.val_test_choices:
        if value != 'all':
          res.append(self.GetOptionString(value))
    else:
      for value in to_test:
        res.append(self.GetOptionString(value))
    return res

class EnvironmentOption(TestOption):
  option_type = TestOption.type_environment
  def __init__(self, name, environment_variable_name, help,
               val_test_choices, val_test_default = None,
               strict_choices = True):
    super(EnvironmentOption, self).__init__(EnvironmentOption.option_type,
                                      name,
                                      help,
                                      val_test_choices,
                                      val_test_default,
                                      strict_choices = strict_choices)
    self.environment_variable_name = environment_variable_name

  def GetOptionString(self, value):
    return self.environment_variable_name + '=' + value


class BuildOption(TestOption):
  option_type = TestOption.type_build
  def __init__(self, name, help,
               val_test_choices, val_test_default = None,
               strict_choices = True, test_independently = False):
    super(BuildOption, self).__init__(BuildOption.option_type,
                                      name,
                                      help,
                                      val_test_choices,
                                      val_test_default,
                                      strict_choices = strict_choices,
                                      test_independently = test_independently)
  def GetOptionString(self, value):
    return self.name + '=' + value


class RuntimeOption(TestOption):
  option_type = TestOption.type_run
  def __init__(self, name, help,
               val_test_choices, val_test_default = None):
    super(RuntimeOption, self).__init__(RuntimeOption.option_type,
                                        name,
                                        help,
                                        val_test_choices,
                                        val_test_default)
  def GetOptionString(self, value):
    if value == 'on':
      return Optionify(self.name)
    else:
      return None



environment_option_compiler = \
  EnvironmentOption('compiler', 'CXX', 'Test for the specified compilers.',
                    val_test_choices=['all'] + config.tested_compilers,
                    strict_choices = False)
test_environment_options = [
  environment_option_compiler
]

build_option_mode = \
  BuildOption('mode', 'Test with the specified build modes.',
              val_test_choices=['all'] + config.build_options_modes)
build_option_standard = \
  BuildOption('std', 'Test with the specified C++ standard.',
              val_test_choices=['all'] + config.tested_cpp_standards,
              strict_choices = False)
build_option_target = \
  BuildOption('target', 'Test with the specified isa enabled.',
              val_test_choices=['all'] + config.build_options_target,
              strict_choices = False, test_independently = True)
build_option_negative_testing = \
  BuildOption('negative_testing', 'Test with negative testing enabled.',
              val_test_choices=['all'] + config.build_options_negative_testing,
              strict_choices = False, test_independently = True)
test_build_options = [
  build_option_mode,
  build_option_standard,
  build_option_target,
  build_option_negative_testing
]

test_runtime_options = []

test_options = \
    test_environment_options + test_build_options + test_runtime_options


def BuildOptions():
  args = argparse.ArgumentParser(
    description =
    '''This tool runs all tests matching the specified filters for multiple
    environment, build options, and runtime options configurations.''',
    # Print default values.
    formatter_class=argparse.ArgumentDefaultsHelpFormatter)

  args.add_argument('filters', metavar='filter', nargs='*',
                    help='Run tests matching all of the (regexp) filters.')

  # We automatically build the script options from the options to be tested.
  test_arguments = args.add_argument_group(
    'Test options',
    'These options indicate what should be tested')
  for option in test_options:
    choices = option.val_test_choices if option.strict_choices else None
    help = option.help
    if not option.strict_choices:
      help += ' Supported values: {' + ','.join(option.val_test_choices) + '}'
    test_arguments.add_argument(Optionify(option.name),
                                nargs='+',
                                choices=choices,
                                default=option.val_test_default,
                                help=help,
                                action='store')

  general_arguments = args.add_argument_group('General options')
  general_arguments.add_argument('--fast', action='store_true',
                                 help='''Skip the lint and clang-format tests,
                                 and run only with one compiler, in one mode,
                                 with one C++ standard, and with an appropriate
                                 default for runtime options.''')
  general_arguments.add_argument('--dry-run', action='store_true',
                                 help='''Don't actually build or run anything,
                                 but print the configurations that would be
                                 tested.''')
  general_arguments.add_argument(
    '--jobs', '-j', metavar='N', type=int, nargs='?',
    default=multiprocessing.cpu_count(),
    const=multiprocessing.cpu_count(),
    help='''Runs the tests using N jobs. If the option is set but no value is
    provided, the script will use as many jobs as it thinks useful.''')
  general_arguments.add_argument('--clang-format',
                                 default=clang_format.DEFAULT_CLANG_FORMAT,
                                 help='Path to clang-format.')
  general_arguments.add_argument('--nobench', action='store_true',
                                 help='Do not run benchmarks.')
  general_arguments.add_argument('--nolint', action='store_true',
                                 help='Do not run the linter.')
  general_arguments.add_argument('--noclang-format', action='store_true',
                                 help='Do not run clang-format.')
  general_arguments.add_argument('--notest', action='store_true',
                                 help='Do not run tests.')
  general_arguments.add_argument('--fail-early', action='store_true',
                                 help='Exit as soon as a test fails.')
  sim_default = 'none' if platform.machine() == 'aarch64' else 'aarch64'
  general_arguments.add_argument(
    '--simulator', action='store', choices=['aarch64', 'none'],
    default=sim_default,
    help='Explicitly enable or disable the simulator.')
  general_arguments.add_argument(
    '--under_valgrind', action='store_true',
    help='''Run the test-runner commands under Valgrind.
            Note that a few tests are known to fail because of
            issues in Valgrind''')
  return args.parse_args()


def RunCommand(command, environment_options = None):
  # Create a copy of the environment. We do not want to pollute the environment
  # of future commands run.
  environment = os.environ
  # Configure the environment.
  # TODO: We currently pass the options as strings, so we need to parse them. We
  # should instead pass them as a data structure and build the string option
  # later. `environment_options` looks like `['CXX=compiler', 'OPT=val']`.
  if environment_options:
    for option in environment_options:
      opt, val = option.split('=')
      environment[opt] = val

  printable_command = ''
  if environment_options:
    printable_command += ' '.join(environment_options) + ' '
  printable_command += ' '.join(command)

  printable_command_orange = \
    printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR
  printer.PrintOverwritableLine(printable_command_orange)
  sys.stdout.flush()

  # Start a process for the command.
  # Interleave `stderr` and `stdout`.
  p = subprocess.Popen(command,
                       stdout=subprocess.PIPE,
                       stderr=subprocess.STDOUT,
                       env=environment)

  # We want to be able to display a continuously updated 'work indicator' while
  # the process is running. Since the process can hang if the `stdout` pipe is
  # full, we need to pull from it regularly. We cannot do so via the
  # `readline()` function because it is blocking, and would thus cause the
  # indicator to not be updated properly. So use file control mechanisms
  # instead.
  indicator = ' (still working: %d seconds elapsed)'

  # Mark the process output as non-blocking.
  flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL)
  fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)

  t_start = time.time()
  t_last_indication = t_start
  process_output = ''

  # Keep looping as long as the process is running.
  while p.poll() is None:
    # Avoid polling too often.
    time.sleep(0.1)
    # Update the progress indicator.
    t_current = time.time()
    if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1):
      printer.PrintOverwritableLine(
        printable_command_orange + indicator % int(t_current - t_start))
      sys.stdout.flush()
      t_last_indication = t_current
    # Pull from the process output.
    while True:
      try:
        line = os.read(p.stdout.fileno(), 1024)
      except OSError:
        line = ''
        break
      if line == '': break
      process_output += line

  # The process has exited. Don't forget to retrieve the rest of its output.
  out, err = p.communicate()
  rc = p.poll()
  process_output += out

  printable_command += ' (took %d seconds)' % int(t_current - t_start)
  if rc == 0:
    printer.Print(printer.COLOUR_GREEN + printable_command + printer.NO_COLOUR)
  else:
    printer.Print(printer.COLOUR_RED + printable_command + printer.NO_COLOUR)
    printer.Print(process_output)
  return rc


def RunLinter():
  rc, default_tracked_files = lint.GetDefaultFilesToLint()
  if rc:
    return rc
  return lint.RunLinter(map(lambda x: join(dir_root, x), default_tracked_files),
                        jobs = args.jobs, progress_prefix = 'cpp lint: ')


def RunClangFormat():
  return clang_format.ClangFormatFiles(clang_format.GetCppSourceFilesToFormat(),
                                       args.clang_format, jobs = args.jobs,
                                       progress_prefix = 'clang-format: ')



def BuildAll(build_options, jobs):
  scons_command = ["scons", "-C", dir_root, 'all', '-j', str(jobs)]
  scons_command += list(build_options)
  return RunCommand(scons_command, list(environment_options))


# Work out if the given options or args allow to run on the specified arch.
#  * arches is a list of ISA/architecture (a64, aarch32, etc)
#  * options are test.py's command line options if any.
#  * args are the arguments given to the build script.
def CanRunOn(arches, options, args):
  # First we check in the build specific options.
  for option in options:
    if 'target' in option:
      # The option format is 'target=x,y,z'.
      for target in (option.split('='))[1].split(','):
        if target in arches:
          return True

      # There was a target build option but it didn't include the target arch.
      return False

  # No specific build option, check the script arguments.
  # The meaning of 'all' will depend on the platform, e.g. 32-bit compilers
  # cannot handle Aarch64 while 64-bit compiler can handle Aarch32. To avoid
  # any issues no benchmarks are run for target='all'.
  if args.target == 'all': return False

  for target in args.target[0].split(','):
    if target in arches:
      return True

  return False


def CanRunAarch64(options, args):
  return CanRunOn(['aarch64', 'a64'], options, args)


def CanRunAarch32(options, args):
  return CanRunOn(['aarch32', 'a32', 't32'], options, args)


def RunBenchmarks(options, args):
  rc = 0
  if CanRunAarch32(options, args):
    benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch32_benchmarks)
    for bench in benchmark_names:
      rc |= RunCommand(
        [os.path.realpath(
          join(config.dir_build_latest, 'benchmarks/aarch32', bench)), '10'])
  if CanRunAarch64(options, args):
    benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch64_benchmarks)
    for bench in benchmark_names:
      rc |= RunCommand(
        [util.relrealpath(
            join(config.dir_build_latest,
                'benchmarks/aarch64', bench)), '10'])
  return rc


def PrintStatus(success):
  printer.Print('\n$ ' + ' '.join(sys.argv))
  if success:
    printer.Print('SUCCESS')
  else:
    printer.Print('FAILURE')



if __name__ == '__main__':
  util.require_program('scons')
  rc = 0

  args = BuildOptions()

  def MaybeExitEarly(rc):
    if args.fail_early and rc != 0:
      PrintStatus(rc == 0)
      sys.exit(rc)

  if args.under_valgrind:
    util.require_program('valgrind')

  if args.fast:
    def SetFast(option, specified, default):
      option.val_test_choices = \
        [default if specified == 'all' else specified[0]]
    # `g++` is very slow to compile a few aarch32 test files.
    SetFast(environment_option_compiler, args.compiler, 'clang++')
    SetFast(build_option_standard, args.std, 'c++98')
    SetFast(build_option_mode, args.mode, 'debug')

  if not args.nolint and not (args.fast or args.dry_run):
    rc |= RunLinter()
    MaybeExitEarly(rc)

  if not args.noclang_format and not (args.fast or args.dry_run):
    rc |= RunClangFormat()
    MaybeExitEarly(rc)

  # List all combinations of options that will be tested.
  def ListCombinations(args, options):
    opts_list = [
        opt.ArgList(args.__dict__[opt.name])
        for opt in options
        if not opt.test_independently
    ]
    return list(itertools.product(*opts_list))
  # List combinations of options that should only be tested independently.
  def ListIndependentCombinations(args, options, base):
    n = []
    for opt in options:
      if opt.test_independently:
        for o in opt.ArgList(args.__dict__[opt.name]):
          n.append(base + (o,))
    return n
  # TODO: We should refine the configurations we test by default, instead of
  #       always testing all possible combinations.
  test_env_combinations = ListCombinations(args, test_environment_options)
  test_build_combinations = ListCombinations(args, test_build_options)
  if not args.fast:
    test_build_combinations.extend(
        ListIndependentCombinations(args,
                                    test_build_options,
                                    test_build_combinations[0]))
  test_runtime_combinations = ListCombinations(args, test_runtime_options)

  tests = threaded_tests.TestQueue(args.under_valgrind)
  for environment_options in test_env_combinations:
    for build_options in test_build_combinations:
      if (args.dry_run):
        for runtime_options in test_runtime_combinations:
          print(' '.join(filter(None, environment_options)) + ', ' +
                ' '.join(filter(None, build_options)) + ', ' +
                ' '.join(filter(None, runtime_options)))
        continue

      # Avoid going through the build stage if we are not using the build
      # result.
      if not (args.notest and args.nobench):
        build_rc = BuildAll(build_options, args.jobs)
        # Don't run the tests for this configuration if the build failed.
        if build_rc != 0:
          rc |= build_rc
          MaybeExitEarly(rc)
          continue

      # Use the realpath of the test executable so that the commands printed
      # can be copy-pasted and run.
      test_executable = util.relrealpath(
        join(config.dir_build_latest, 'test', 'test-runner'))

      if not args.notest:
        printer.Print(test_executable)

      for runtime_options in test_runtime_combinations:
        if not args.notest:
          runtime_options = [x for x in runtime_options if x is not None]
          prefix = '  ' + ' '.join(runtime_options) + '  '
          tests.Add(test_executable,
                    args.filters,
                    list(runtime_options))
          MaybeExitEarly(rc)

      if not args.nobench:
        rc |= RunBenchmarks(build_options, args)
        MaybeExitEarly(rc)

  rc |= tests.Run(args.jobs)
  if not args.dry_run:
    PrintStatus(rc == 0)

  sys.exit(rc)