blob: c4a43c7df58c67d0d057f75b56dae9b509d5ac4c [file] [log] [blame]
armvixlad96eda2013-06-14 11:42:37 +01001#!/usr/bin/env python2.7
2
Alexandre Ramesb78f1392016-07-01 14:22:22 +01003# Copyright 2015, VIXL authors
armvixlad96eda2013-06-14 11:42:37 +01004# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are met:
8#
9# * Redistributions of source code must retain the above copyright notice,
10# this list of conditions and the following disclaimer.
11# * Redistributions in binary form must reproduce the above copyright notice,
12# this list of conditions and the following disclaimer in the documentation
13# and/or other materials provided with the distribution.
14# * Neither the name of ARM Limited nor the names of its contributors may be
15# used to endorse or promote products derived from this software without
16# specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
19# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
22# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
armvixlad96eda2013-06-14 11:42:37 +010029import argparse
armvixldb644342015-07-21 11:37:10 +010030import fcntl
31import git
32import itertools
armvixl4a102ba2014-07-14 09:02:40 +010033import multiprocessing
armvixldb644342015-07-21 11:37:10 +010034import os
35from os.path import join
36import platform
armvixldb644342015-07-21 11:37:10 +010037import subprocess
38import sys
armvixlad96eda2013-06-14 11:42:37 +010039import time
armvixldb644342015-07-21 11:37:10 +010040
41import config
armvixl0f35e362016-05-10 13:57:58 +010042import clang_format
armvixldb644342015-07-21 11:37:10 +010043import lint
44import printer
45import test
46import threaded_tests
armvixlad96eda2013-06-14 11:42:37 +010047import util
48
49
armvixldb644342015-07-21 11:37:10 +010050dir_root = config.dir_root
51
Anthony Barbierf2986e12019-02-28 16:49:23 +000052
53# Remove duplicates from a list
54def RemoveDuplicates(values):
55 # Convert the list into a set and back to list
56 # as sets guarantee items are unique.
57 return list(set(values))
armvixldb644342015-07-21 11:37:10 +010058
59
Anthony Barbierf2986e12019-02-28 16:49:23 +000060# Custom argparse.Action to automatically add and handle an 'all' option.
61# If no 'default' value is set, it will default to 'all.
62# If accepted options are set using 'choices' then only these values will be
63# allowed.
64# If they're set using 'soft_choices' then 'all' will default to these values,
65# but other values will also be accepted.
66class AllChoiceAction(argparse.Action):
armvixldb644342015-07-21 11:37:10 +010067
Anthony Barbierf2986e12019-02-28 16:49:23 +000068 # At least one option was set by the user.
69 WasSetByUser = False
armvixldb644342015-07-21 11:37:10 +010070
Anthony Barbierf2986e12019-02-28 16:49:23 +000071 def __init__(self, **kwargs):
72 if 'choices' in kwargs:
73 assert 'soft_choices' not in kwargs,\
74 "Can't have both 'choices' and 'soft_choices' options"
75 self.all_choices = list(kwargs['choices'])
76 kwargs['choices'].append('all')
armvixldb644342015-07-21 11:37:10 +010077 else:
Anthony Barbierf2986e12019-02-28 16:49:23 +000078 self.all_choices = kwargs['soft_choices']
79 kwargs['help'] += ' Supported values: {' + ','.join(
80 ['all'] + self.all_choices) + '}'
81 del kwargs['soft_choices']
82 if 'default' not in kwargs:
83 kwargs['default'] = self.all_choices
84 super(AllChoiceAction, self).__init__(**kwargs)
armvixldb644342015-07-21 11:37:10 +010085
Anthony Barbierf2986e12019-02-28 16:49:23 +000086 def __call__(self, parser, namespace, values, option_string=None):
87 AllChoiceAction.WasSetByUser = True
88 if 'all' in values:
89 # Substitute 'all' by the actual values.
90 values = self.all_choices + [value for value in values if value != 'all']
armvixldb644342015-07-21 11:37:10 +010091
Anthony Barbierf2986e12019-02-28 16:49:23 +000092 setattr(namespace, self.dest, RemoveDuplicates(values))
armvixl5799d6c2014-05-01 11:05:00 +010093
94
armvixlad96eda2013-06-14 11:42:37 +010095def BuildOptions():
armvixldb644342015-07-21 11:37:10 +010096 args = argparse.ArgumentParser(
97 description =
armvixl0f35e362016-05-10 13:57:58 +010098 '''This tool runs all tests matching the specified filters for multiple
armvixldb644342015-07-21 11:37:10 +010099 environment, build options, and runtime options configurations.''',
100 # Print default values.
101 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
102
103 args.add_argument('filters', metavar='filter', nargs='*',
104 help='Run tests matching all of the (regexp) filters.')
105
106 # We automatically build the script options from the options to be tested.
107 test_arguments = args.add_argument_group(
108 'Test options',
109 'These options indicate what should be tested')
Anthony Barbierf2986e12019-02-28 16:49:23 +0000110 test_arguments.add_argument(
111 '--negative_testing',
112 help='Tests with negative testing enabled.',
113 action='store_const',
114 const='on',
115 default='off')
116 test_arguments.add_argument(
117 '--compiler',
118 help='Test for the specified compilers.',
119 soft_choices=config.tested_compilers,
120 action=AllChoiceAction,
121 nargs="+")
122 test_arguments.add_argument(
123 '--mode',
124 help='Test with the specified build modes.',
125 choices=config.build_options_modes,
126 action=AllChoiceAction,
127 nargs="+")
128 test_arguments.add_argument(
129 '--std',
130 help='Test with the specified C++ standard.',
131 soft_choices=config.tested_cpp_standards,
132 action=AllChoiceAction,
133 nargs="+")
134 test_arguments.add_argument(
135 '--target',
136 help='Test with the specified isa enabled.',
137 soft_choices=config.build_options_target,
138 action=AllChoiceAction,
139 nargs="+")
armvixldb644342015-07-21 11:37:10 +0100140
141 general_arguments = args.add_argument_group('General options')
Jacob Bramley59d74ae2017-01-18 15:27:45 +0000142 general_arguments.add_argument('--dry-run', action='store_true',
143 help='''Don't actually build or run anything,
144 but print the configurations that would be
145 tested.''')
Anthony Barbier88e1d032019-06-13 15:20:20 +0100146 general_arguments.add_argument('--verbose', action='store_true',
147 help='''Print extra information.''')
armvixldb644342015-07-21 11:37:10 +0100148 general_arguments.add_argument(
149 '--jobs', '-j', metavar='N', type=int, nargs='?',
Anthony Barbier9c4ba7a2019-02-15 15:20:25 +0000150 default=multiprocessing.cpu_count(),
151 const=multiprocessing.cpu_count(),
armvixldb644342015-07-21 11:37:10 +0100152 help='''Runs the tests using N jobs. If the option is set but no value is
153 provided, the script will use as many jobs as it thinks useful.''')
Pierre Langlois44096c42018-05-23 23:15:25 +0100154 general_arguments.add_argument('--clang-format',
155 default=clang_format.DEFAULT_CLANG_FORMAT,
156 help='Path to clang-format.')
armvixldb644342015-07-21 11:37:10 +0100157 general_arguments.add_argument('--nobench', action='store_true',
158 help='Do not run benchmarks.')
159 general_arguments.add_argument('--nolint', action='store_true',
160 help='Do not run the linter.')
armvixl0f35e362016-05-10 13:57:58 +0100161 general_arguments.add_argument('--noclang-format', action='store_true',
162 help='Do not run clang-format.')
armvixldb644342015-07-21 11:37:10 +0100163 general_arguments.add_argument('--notest', action='store_true',
164 help='Do not run tests.')
Alexandre Rames73064a22016-07-08 09:17:03 +0100165 general_arguments.add_argument('--fail-early', action='store_true',
166 help='Exit as soon as a test fails.')
armvixl684cd2a2015-10-23 13:38:33 +0100167 general_arguments.add_argument(
168 '--under_valgrind', action='store_true',
169 help='''Run the test-runner commands under Valgrind.
170 Note that a few tests are known to fail because of
171 issues in Valgrind''')
armvixldb644342015-07-21 11:37:10 +0100172 return args.parse_args()
armvixlad96eda2013-06-14 11:42:37 +0100173
174
armvixldb644342015-07-21 11:37:10 +0100175def RunCommand(command, environment_options = None):
176 # Create a copy of the environment. We do not want to pollute the environment
177 # of future commands run.
Anthony Barbierf2986e12019-02-28 16:49:23 +0000178 environment = os.environ.copy()
armvixlad96eda2013-06-14 11:42:37 +0100179
armvixldb644342015-07-21 11:37:10 +0100180 printable_command = ''
181 if environment_options:
Anthony Barbierf2986e12019-02-28 16:49:23 +0000182 # Add the environment options to the environment:
183 environment.update(environment_options)
184 printable_command += ' ' + DictToString(environment_options) + ' '
armvixldb644342015-07-21 11:37:10 +0100185 printable_command += ' '.join(command)
armvixlad96eda2013-06-14 11:42:37 +0100186
armvixldb644342015-07-21 11:37:10 +0100187 printable_command_orange = \
188 printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR
189 printer.PrintOverwritableLine(printable_command_orange)
190 sys.stdout.flush()
armvixlad96eda2013-06-14 11:42:37 +0100191
armvixldb644342015-07-21 11:37:10 +0100192 # Start a process for the command.
193 # Interleave `stderr` and `stdout`.
194 p = subprocess.Popen(command,
195 stdout=subprocess.PIPE,
196 stderr=subprocess.STDOUT,
197 env=environment)
armvixlad96eda2013-06-14 11:42:37 +0100198
armvixldb644342015-07-21 11:37:10 +0100199 # We want to be able to display a continuously updated 'work indicator' while
200 # the process is running. Since the process can hang if the `stdout` pipe is
201 # full, we need to pull from it regularly. We cannot do so via the
202 # `readline()` function because it is blocking, and would thus cause the
203 # indicator to not be updated properly. So use file control mechanisms
204 # instead.
205 indicator = ' (still working: %d seconds elapsed)'
armvixl5799d6c2014-05-01 11:05:00 +0100206
armvixldb644342015-07-21 11:37:10 +0100207 # Mark the process output as non-blocking.
208 flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL)
209 fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)
armvixl5799d6c2014-05-01 11:05:00 +0100210
armvixldb644342015-07-21 11:37:10 +0100211 t_start = time.time()
Anthony Barbierf2986e12019-02-28 16:49:23 +0000212 t_current = t_start
armvixldb644342015-07-21 11:37:10 +0100213 t_last_indication = t_start
Anthony Barbier7b4df2b2019-03-12 17:36:15 +0000214 t_current = t_start
armvixldb644342015-07-21 11:37:10 +0100215 process_output = ''
armvixl5799d6c2014-05-01 11:05:00 +0100216
armvixldb644342015-07-21 11:37:10 +0100217 # Keep looping as long as the process is running.
218 while p.poll() is None:
219 # Avoid polling too often.
220 time.sleep(0.1)
221 # Update the progress indicator.
222 t_current = time.time()
223 if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1):
224 printer.PrintOverwritableLine(
225 printable_command_orange + indicator % int(t_current - t_start))
226 sys.stdout.flush()
227 t_last_indication = t_current
228 # Pull from the process output.
229 while True:
230 try:
231 line = os.read(p.stdout.fileno(), 1024)
232 except OSError:
233 line = ''
234 break
235 if line == '': break
236 process_output += line
armvixlad96eda2013-06-14 11:42:37 +0100237
armvixldb644342015-07-21 11:37:10 +0100238 # The process has exited. Don't forget to retrieve the rest of its output.
239 out, err = p.communicate()
240 rc = p.poll()
241 process_output += out
armvixlad96eda2013-06-14 11:42:37 +0100242
Anthony Barbierb5f72392019-02-15 15:33:48 +0000243 printable_command += ' (took %d seconds)' % int(t_current - t_start)
armvixldb644342015-07-21 11:37:10 +0100244 if rc == 0:
245 printer.Print(printer.COLOUR_GREEN + printable_command + printer.NO_COLOUR)
armvixl4a102ba2014-07-14 09:02:40 +0100246 else:
armvixldb644342015-07-21 11:37:10 +0100247 printer.Print(printer.COLOUR_RED + printable_command + printer.NO_COLOUR)
248 printer.Print(process_output)
249 return rc
armvixl5799d6c2014-05-01 11:05:00 +0100250
251
Anthony Barbierf2986e12019-02-28 16:49:23 +0000252def RunLinter(jobs):
Jacob Bramleyf89cb482019-06-28 15:53:18 +0100253 return lint.RunLinter(map(lambda x: join(dir_root, x),
254 util.get_source_files()),
armvixldb644342015-07-21 11:37:10 +0100255 jobs = args.jobs, progress_prefix = 'cpp lint: ')
armvixlad96eda2013-06-14 11:42:37 +0100256
armvixlad96eda2013-06-14 11:42:37 +0100257
Anthony Barbierf2986e12019-02-28 16:49:23 +0000258def RunClangFormat(clang_path, jobs):
Jacob Bramleyf89cb482019-06-28 15:53:18 +0100259 return clang_format.ClangFormatFiles(util.get_source_files(),
260 clang_path,
261 jobs = jobs,
armvixl0f35e362016-05-10 13:57:58 +0100262 progress_prefix = 'clang-format: ')
263
264
Anthony Barbierf2986e12019-02-28 16:49:23 +0000265def BuildAll(build_options, jobs, environment_options):
266 scons_command = ['scons', '-C', dir_root, 'all', '-j', str(jobs)]
Anthony Barbier9c4ba7a2019-02-15 15:20:25 +0000267 if util.IsCommandAvailable('ccache'):
268 scons_command += ['compiler_wrapper=ccache']
269 # Fixes warnings for ccache 3.3.1 and lower:
270 environment_options = environment_options.copy()
271 environment_options["CCACHE_CPP2"] = 'yes'
Anthony Barbierf2986e12019-02-28 16:49:23 +0000272 scons_command += DictToString(build_options).split()
273 return RunCommand(scons_command, environment_options)
armvixlad96eda2013-06-14 11:42:37 +0100274
armvixl4a102ba2014-07-14 09:02:40 +0100275
Anthony Barbierf2986e12019-02-28 16:49:23 +0000276def CanRunAarch64(options, args):
277 for target in options['target']:
278 if target in ['aarch64', 'a64']:
Rodolph Perfetta9a9331f2016-12-09 22:05:48 +0000279 return True
280
281 return False
282
283
Rodolph Perfetta9a9331f2016-12-09 22:05:48 +0000284def CanRunAarch32(options, args):
Anthony Barbierf2986e12019-02-28 16:49:23 +0000285 for target in options['target']:
286 if target in ['aarch32', 'a32', 't32']:
287 return True
288 return False
Rodolph Perfetta9a9331f2016-12-09 22:05:48 +0000289
290
291def RunBenchmarks(options, args):
armvixldb644342015-07-21 11:37:10 +0100292 rc = 0
Rodolph Perfetta9a9331f2016-12-09 22:05:48 +0000293 if CanRunAarch32(options, args):
Pierre Langlois1c1488c2016-12-14 18:16:44 +0000294 benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch32_benchmarks)
295 for bench in benchmark_names:
296 rc |= RunCommand(
297 [os.path.realpath(
Martyn Capewell9cd420f2017-05-12 20:30:23 +0100298 join(config.dir_build_latest, 'benchmarks/aarch32', bench)), '10'])
Rodolph Perfetta9a9331f2016-12-09 22:05:48 +0000299 if CanRunAarch64(options, args):
Pierre Langlois1c1488c2016-12-14 18:16:44 +0000300 benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch64_benchmarks)
301 for bench in benchmark_names:
302 rc |= RunCommand(
303 [util.relrealpath(
Martyn Capewell9cd420f2017-05-12 20:30:23 +0100304 join(config.dir_build_latest,
305 'benchmarks/aarch64', bench)), '10'])
armvixldb644342015-07-21 11:37:10 +0100306 return rc
armvixl4a102ba2014-07-14 09:02:40 +0100307
armvixl4a102ba2014-07-14 09:02:40 +0100308
armvixl4a102ba2014-07-14 09:02:40 +0100309
Anthony Barbierf2986e12019-02-28 16:49:23 +0000310# It is a precommit run if the user did not specify any of the
311# options that would affect the automatically generated combinations.
312def IsPrecommitRun(args):
313 return args.negative_testing == "off" and not AllChoiceAction.WasSetByUser
314
315# Generate a list of all the possible combinations of the passed list:
316# ListCombinations( a = [a0, a1], b = [b0, b1] ) will return
317# [ {a : a0, b : b0}, {a : a0, b : b1}, {a: a1, b : b0}, {a : a1, b : b1}]
318def ListCombinations(**kwargs):
319 # End of recursion: no options passed
320 if not kwargs:
321 return [{}]
322 option, values = kwargs.popitem()
323 configs = ListCombinations(**kwargs)
324 retval = []
325 if not isinstance(values, list):
326 values = [values]
327 for value in values:
328 for config in configs:
329 new_config = config.copy()
330 new_config[option] = value
331 retval.append(new_config)
332 return retval
333
334# Convert a dictionary into a space separated string
335# {a : a0, b : b0} --> "a=a0 b=b0"
336def DictToString(options):
337 return " ".join(
338 ["{}={}".format(option, value) for option, value in options.items()])
armvixlad96eda2013-06-14 11:42:37 +0100339
340
341if __name__ == '__main__':
armvixldb644342015-07-21 11:37:10 +0100342 util.require_program('scons')
armvixlad96eda2013-06-14 11:42:37 +0100343
armvixlad96eda2013-06-14 11:42:37 +0100344 args = BuildOptions()
armvixlad96eda2013-06-14 11:42:37 +0100345
Anthony Barbierf2986e12019-02-28 16:49:23 +0000346 rc = util.ReturnCode(args.fail_early, printer.Print)
Alexandre Rames73064a22016-07-08 09:17:03 +0100347
armvixl684cd2a2015-10-23 13:38:33 +0100348 if args.under_valgrind:
349 util.require_program('valgrind')
350
Anthony Barbierb5f72392019-02-15 15:33:48 +0000351 tests = threaded_tests.TestQueue(args.under_valgrind)
Anthony Barbierf2986e12019-02-28 16:49:23 +0000352 if not args.nolint and not args.dry_run:
353 rc.Combine(RunLinter(args.jobs))
Jacob Bramley59d74ae2017-01-18 15:27:45 +0000354
Anthony Barbierf2986e12019-02-28 16:49:23 +0000355 if not args.noclang_format and not args.dry_run:
356 rc.Combine(RunClangFormat(args.clang_format, args.jobs))
357
358 list_options = []
359 if IsPrecommitRun(args):
360 # Maximize the coverage for precommit testing.
361
Pierre Langloisa5b3cef2019-01-28 11:30:38 +0000362 # Debug builds with negative testing and all targets enabled.
Anthony Barbierf2986e12019-02-28 16:49:23 +0000363 list_options += ListCombinations(
364 compiler = args.compiler,
365 negative_testing = 'on',
Anthony Barbierf2986e12019-02-28 16:49:23 +0000366 mode = 'debug',
367 target = 'a64,a32,t32')
368
369 # Release builds with all targets enabled.
370 list_options += ListCombinations(
371 compiler = args.compiler,
372 negative_testing = 'off',
373 std = args.std,
374 mode = 'release',
375 target = 'a64,a32,t32')
376
377 # c++98 builds for Thumb32 target only.
378 list_options += ListCombinations(
379 compiler = args.compiler,
380 negative_testing = 'off',
381 std = 'c++98',
382 mode = args.mode,
383 target = 't32')
384
385 # c++11 builds for Aarch64 target only.
386 list_options += ListCombinations(
387 compiler = args.compiler,
388 negative_testing = 'off',
389 std = 'c++11',
390 mode = args.mode,
391 target = 'a64')
392 else:
393 list_options = ListCombinations(
394 compiler = args.compiler,
395 negative_testing = args.negative_testing,
396 std = args.std,
397 mode = args.mode,
398 target = args.target)
399
400 for options in list_options:
401 if (args.dry_run):
402 print(DictToString(options))
403 continue
404 # Convert 'compiler' into an environment variable:
405 environment_options = {'CXX': options['compiler']}
406 del options['compiler']
407
408 # Avoid going through the build stage if we are not using the build
409 # result.
410 if not (args.notest and args.nobench):
411 build_rc = BuildAll(options, args.jobs, environment_options)
412 # Don't run the tests for this configuration if the build failed.
413 if build_rc != 0:
414 rc.Combine(build_rc)
415 continue
armvixlad96eda2013-06-14 11:42:37 +0100416
armvixldb644342015-07-21 11:37:10 +0100417 # Use the realpath of the test executable so that the commands printed
418 # can be copy-pasted and run.
Alexandre Rames81c76e62016-07-19 09:53:09 +0100419 test_executable = util.relrealpath(
armvixldb644342015-07-21 11:37:10 +0100420 join(config.dir_build_latest, 'test', 'test-runner'))
421
422 if not args.notest:
423 printer.Print(test_executable)
Anthony Barbierf2986e12019-02-28 16:49:23 +0000424 tests.Add(
425 test_executable,
426 args.filters,
427 list())
armvixldb644342015-07-21 11:37:10 +0100428
429 if not args.nobench:
Anthony Barbierf2986e12019-02-28 16:49:23 +0000430 rc.Combine(RunBenchmarks(options, args))
armvixldb644342015-07-21 11:37:10 +0100431
Anthony Barbier88e1d032019-06-13 15:20:20 +0100432 rc.Combine(tests.Run(args.jobs, args.verbose))
Jacob Bramley59d74ae2017-01-18 15:27:45 +0000433 if not args.dry_run:
Anthony Barbierf2986e12019-02-28 16:49:23 +0000434 rc.PrintStatus()
Alexandre Rames7c0ea8b2016-05-18 13:47:42 +0100435
Anthony Barbierf2986e12019-02-28 16:49:23 +0000436 sys.exit(rc.Value)