| #! /usr/bin/env python3 |
| |
| import os |
| import subprocess |
| import sys |
| import platform |
| import argparse |
| import re |
| from glob import glob |
| |
| # Tests require at least CPython 3.3. If your default python3 executable |
| # is of lower version, you can point MICROPY_CPYTHON3 environment var |
| # to the correct executable. |
| if os.name == 'nt': |
| CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe') |
| MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/windows/micropython.exe') |
| else: |
| CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3') |
| MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/unix/micropython') |
| |
| # mpy-cross is only needed if --via-mpy command-line arg is passed |
| MPYCROSS = os.getenv('MICROPY_MPYCROSS', '../mpy-cross/mpy-cross') |
| |
| # Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale |
| os.environ['PYTHONIOENCODING'] = 'utf-8' |
| |
| def rm_f(fname): |
| if os.path.exists(fname): |
| os.remove(fname) |
| |
| |
| # unescape wanted regex chars and escape unwanted ones |
| def convert_regex_escapes(line): |
| cs = [] |
| escape = False |
| for c in str(line, 'utf8'): |
| if escape: |
| escape = False |
| cs.append(c) |
| elif c == '\\': |
| escape = True |
| elif c in ('(', ')', '[', ']', '{', '}', '.', '*', '+', '^', '$'): |
| cs.append('\\' + c) |
| else: |
| cs.append(c) |
| # accept carriage-return(s) before final newline |
| if cs[-1] == '\n': |
| cs[-1] = '\r*\n' |
| return bytes(''.join(cs), 'utf8') |
| |
| |
| def run_micropython(pyb, args, test_file, is_special=False): |
| special_tests = ('micropython/meminfo.py', 'basics/bytes_compare3.py', 'basics/builtin_help.py', 'thread/thread_exc2.py') |
| if pyb is None: |
| # run on PC |
| if test_file.startswith(('cmdline/', 'feature_check/')) or test_file in special_tests: |
| # special handling for tests of the unix cmdline program |
| is_special = True |
| |
| if is_special: |
| # check for any cmdline options needed for this test |
| args = [MICROPYTHON] |
| with open(test_file, 'rb') as f: |
| line = f.readline() |
| if line.startswith(b'# cmdline:'): |
| # subprocess.check_output on Windows only accepts strings, not bytes |
| args += [str(c, 'utf-8') for c in line[10:].strip().split()] |
| |
| # run the test, possibly with redirected input |
| try: |
| if 'repl_' in test_file: |
| # Need to use a PTY to test command line editing |
| try: |
| import pty |
| except ImportError: |
| # in case pty module is not available, like on Windows |
| return b'SKIP\n' |
| import select |
| |
| def get(required=False): |
| rv = b'' |
| while True: |
| ready = select.select([master], [], [], 0.02) |
| if ready[0] == [master]: |
| rv += os.read(master, 1024) |
| else: |
| if not required or rv: |
| return rv |
| |
| def send_get(what): |
| os.write(master, what) |
| return get() |
| |
| with open(test_file, 'rb') as f: |
| # instead of: output_mupy = subprocess.check_output(args, stdin=f) |
| master, slave = pty.openpty() |
| p = subprocess.Popen(args, stdin=slave, stdout=slave, |
| stderr=subprocess.STDOUT, bufsize=0) |
| banner = get(True) |
| output_mupy = banner + b''.join(send_get(line) for line in f) |
| send_get(b'\x04') # exit the REPL, so coverage info is saved |
| p.kill() |
| os.close(master) |
| os.close(slave) |
| else: |
| output_mupy = subprocess.check_output(args + [test_file], stderr=subprocess.STDOUT) |
| except subprocess.CalledProcessError: |
| return b'CRASH' |
| |
| else: |
| # a standard test run on PC |
| |
| # create system command |
| cmdlist = [MICROPYTHON, '-X', 'emit=' + args.emit] |
| if args.heapsize is not None: |
| cmdlist.extend(['-X', 'heapsize=' + args.heapsize]) |
| |
| # if running via .mpy, first compile the .py file |
| if args.via_mpy: |
| subprocess.check_output([MPYCROSS, '-mcache-lookup-bc', '-o', 'mpytest.mpy', test_file]) |
| cmdlist.extend(['-m', 'mpytest']) |
| else: |
| cmdlist.append(test_file) |
| |
| # run the actual test |
| try: |
| output_mupy = subprocess.check_output(cmdlist, stderr=subprocess.STDOUT) |
| except subprocess.CalledProcessError: |
| output_mupy = b'CRASH' |
| |
| # clean up if we had an intermediate .mpy file |
| if args.via_mpy: |
| rm_f('mpytest.mpy') |
| |
| else: |
| # run on pyboard |
| import pyboard |
| pyb.enter_raw_repl() |
| try: |
| output_mupy = pyb.execfile(test_file) |
| except pyboard.PyboardError: |
| output_mupy = b'CRASH' |
| |
| # canonical form for all ports/platforms is to use \n for end-of-line |
| output_mupy = output_mupy.replace(b'\r\n', b'\n') |
| |
| # don't try to convert the output if we should skip this test |
| if output_mupy in (b'SKIP\n', b'CRASH'): |
| return output_mupy |
| |
| if is_special or test_file in special_tests: |
| # convert parts of the output that are not stable across runs |
| with open(test_file + '.exp', 'rb') as f: |
| lines_exp = [] |
| for line in f.readlines(): |
| if line == b'########\n': |
| line = (line,) |
| else: |
| line = (line, re.compile(convert_regex_escapes(line))) |
| lines_exp.append(line) |
| lines_mupy = [line + b'\n' for line in output_mupy.split(b'\n')] |
| if output_mupy.endswith(b'\n'): |
| lines_mupy = lines_mupy[:-1] # remove erroneous last empty line |
| i_mupy = 0 |
| for i in range(len(lines_exp)): |
| if lines_exp[i][0] == b'########\n': |
| # 8x #'s means match 0 or more whole lines |
| line_exp = lines_exp[i + 1] |
| skip = 0 |
| while i_mupy + skip < len(lines_mupy) and not line_exp[1].match(lines_mupy[i_mupy + skip]): |
| skip += 1 |
| if i_mupy + skip >= len(lines_mupy): |
| lines_mupy[i_mupy] = b'######## FAIL\n' |
| break |
| del lines_mupy[i_mupy:i_mupy + skip] |
| lines_mupy.insert(i_mupy, b'########\n') |
| i_mupy += 1 |
| else: |
| # a regex |
| if lines_exp[i][1].match(lines_mupy[i_mupy]): |
| lines_mupy[i_mupy] = lines_exp[i][0] |
| else: |
| #print("don't match: %r %s" % (lines_exp[i][1], lines_mupy[i_mupy])) # DEBUG |
| pass |
| i_mupy += 1 |
| if i_mupy >= len(lines_mupy): |
| break |
| output_mupy = b''.join(lines_mupy) |
| |
| return output_mupy |
| |
| |
| def run_feature_check(pyb, args, base_path, test_file): |
| return run_micropython(pyb, args, base_path + "/feature_check/" + test_file, is_special=True) |
| |
| |
| def run_tests(pyb, tests, args, base_path="."): |
| test_count = 0 |
| testcase_count = 0 |
| passed_count = 0 |
| failed_tests = [] |
| skipped_tests = [] |
| |
| skip_tests = set() |
| skip_native = False |
| skip_int_big = False |
| skip_set_type = False |
| skip_async = False |
| skip_const = False |
| skip_revops = False |
| |
| # Check if micropython.native is supported, and skip such tests if it's not |
| output = run_feature_check(pyb, args, base_path, 'native_check.py') |
| if output == b'CRASH': |
| skip_native = True |
| |
| # Check if arbitrary-precision integers are supported, and skip such tests if it's not |
| output = run_feature_check(pyb, args, base_path, 'int_big.py') |
| if output != b'1000000000000000000000000000000000000000000000\n': |
| skip_int_big = True |
| |
| # Check if set type (and set literals) is supported, and skip such tests if it's not |
| output = run_feature_check(pyb, args, base_path, 'set_check.py') |
| if output == b'CRASH': |
| skip_set_type = True |
| |
| # Check if async/await keywords are supported, and skip such tests if it's not |
| output = run_feature_check(pyb, args, base_path, 'async_check.py') |
| if output == b'CRASH': |
| skip_async = True |
| |
| # Check if const keyword (MicroPython extension) is supported, and skip such tests if it's not |
| output = run_feature_check(pyb, args, base_path, 'const.py') |
| if output == b'CRASH': |
| skip_const = True |
| |
| # Check if __rOP__ special methods are supported, and skip such tests if it's not |
| output = run_feature_check(pyb, args, base_path, 'reverse_ops.py') |
| if output == b'TypeError\n': |
| skip_revops = True |
| |
| # Check if emacs repl is supported, and skip such tests if it's not |
| t = run_feature_check(pyb, args, base_path, 'repl_emacs_check.py') |
| if not 'True' in str(t, 'ascii'): |
| skip_tests.add('cmdline/repl_emacs_keys.py') |
| |
| upy_byteorder = run_feature_check(pyb, args, base_path, 'byteorder.py') |
| upy_float_precision = int(run_feature_check(pyb, args, base_path, 'float.py')) |
| has_complex = run_feature_check(pyb, args, base_path, 'complex.py') == b'complex\n' |
| has_coverage = run_feature_check(pyb, args, base_path, 'coverage.py') == b'coverage\n' |
| cpy_byteorder = subprocess.check_output([CPYTHON3, base_path + '/feature_check/byteorder.py']) |
| skip_endian = (upy_byteorder != cpy_byteorder) |
| |
| # Some tests shouldn't be run under Travis CI |
| if os.getenv('TRAVIS') == 'true': |
| skip_tests.add('basics/memoryerror.py') |
| skip_tests.add('thread/thread_gc1.py') # has reliability issues |
| skip_tests.add('thread/thread_lock4.py') # has reliability issues |
| skip_tests.add('thread/stress_heap.py') # has reliability issues |
| skip_tests.add('thread/stress_recurse.py') # has reliability issues |
| |
| if upy_float_precision == 0: |
| skip_tests.add('extmod/ujson_dumps_float.py') |
| skip_tests.add('extmod/ujson_loads_float.py') |
| skip_tests.add('misc/rge_sm.py') |
| if upy_float_precision < 32: |
| skip_tests.add('float/float2int_intbig.py') # requires fp32, there's float2int_fp30_intbig.py instead |
| skip_tests.add('float/string_format.py') # requires fp32, there's string_format_fp30.py instead |
| skip_tests.add('float/bytes_construct.py') # requires fp32 |
| skip_tests.add('float/bytearray_construct.py') # requires fp32 |
| if upy_float_precision < 64: |
| skip_tests.add('float/float_divmod.py') # tested by float/float_divmod_relaxed.py instead |
| skip_tests.add('float/float2int_doubleprec_intbig.py') |
| skip_tests.add('float/float_parse_doubleprec.py') |
| |
| if not has_complex: |
| skip_tests.add('float/complex1.py') |
| skip_tests.add('float/complex1_intbig.py') |
| skip_tests.add('float/int_big_float.py') |
| skip_tests.add('float/true_value.py') |
| skip_tests.add('float/types.py') |
| |
| if not has_coverage: |
| skip_tests.add('cmdline/cmd_parsetree.py') |
| |
| # Some tests shouldn't be run on a PC |
| if pyb is None: |
| # unix build does not have the GIL so can't run thread mutation tests |
| for t in tests: |
| if t.startswith('thread/mutate_'): |
| skip_tests.add(t) |
| |
| # Some tests shouldn't be run on pyboard |
| if pyb is not None: |
| skip_tests.add('basics/exception_chain.py') # warning is not printed |
| skip_tests.add('micropython/meminfo.py') # output is very different to PC output |
| skip_tests.add('extmod/machine_mem.py') # raw memory access not supported |
| |
| if args.target == 'wipy': |
| skip_tests.add('misc/print_exception.py') # requires error reporting full |
| skip_tests.add('misc/recursion.py') # requires stack checking enabled |
| skip_tests.add('misc/recursive_data.py') # requires stack checking enabled |
| skip_tests.add('misc/recursive_iternext.py') # requires stack checking enabled |
| skip_tests.update({'extmod/uctypes_%s.py' % t for t in 'bytearray le native_le ptr_le ptr_native_le sizeof sizeof_native array_assign_le array_assign_native_le'.split()}) # requires uctypes |
| skip_tests.add('extmod/zlibd_decompress.py') # requires zlib |
| skip_tests.add('extmod/uheapq1.py') # uheapq not supported by WiPy |
| skip_tests.add('extmod/urandom_basic.py') # requires urandom |
| skip_tests.add('extmod/urandom_extra.py') # requires urandom |
| elif args.target == 'esp8266': |
| skip_tests.add('misc/rge_sm.py') # too large |
| elif args.target == 'minimal': |
| skip_tests.add('basics/class_inplace_op.py') # all special methods not supported |
| skip_tests.add('misc/rge_sm.py') # too large |
| skip_tests.add('micropython/opt_level.py') # don't assume line numbers are stored |
| |
| # Some tests are known to fail on 64-bit machines |
| if pyb is None and platform.architecture()[0] == '64bit': |
| pass |
| |
| # Some tests use unsupported features on Windows |
| if os.name == 'nt': |
| skip_tests.add('import/import_file.py') # works but CPython prints forward slashes |
| |
| # Some tests are known to fail with native emitter |
| # Remove them from the below when they work |
| if args.emit == 'native': |
| skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from gen_yield_from_close gen_yield_from_ducktype gen_yield_from_exc gen_yield_from_iter gen_yield_from_send gen_yield_from_stopped gen_yield_from_throw gen_yield_from_throw2 gen_yield_from_throw3 generator1 generator2 generator_args generator_close generator_closure generator_exc generator_return generator_send'.split()}) # require yield |
| skip_tests.update({'basics/%s.py' % t for t in 'bytes_gen class_store_class globals_del string_join'.split()}) # require yield |
| skip_tests.update({'basics/async_%s.py' % t for t in 'def await await2 for for2 with with2'.split()}) # require yield |
| skip_tests.update({'basics/%s.py' % t for t in 'try_reraise try_reraise2'.split()}) # require raise_varargs |
| skip_tests.update({'basics/%s.py' % t for t in 'with_break with_continue with_return'.split()}) # require complete with support |
| skip_tests.add('basics/array_construct2.py') # requires generators |
| skip_tests.add('basics/bool1.py') # seems to randomly fail |
| skip_tests.add('basics/builtin_hash_gen.py') # requires yield |
| skip_tests.add('basics/class_bind_self.py') # requires yield |
| skip_tests.add('basics/del_deref.py') # requires checking for unbound local |
| skip_tests.add('basics/del_local.py') # requires checking for unbound local |
| skip_tests.add('basics/exception_chain.py') # raise from is not supported |
| skip_tests.add('basics/for_range.py') # requires yield_value |
| skip_tests.add('basics/try_finally_loops.py') # requires proper try finally code |
| skip_tests.add('basics/try_finally_return.py') # requires proper try finally code |
| skip_tests.add('basics/try_finally_return2.py') # requires proper try finally code |
| skip_tests.add('basics/unboundlocal.py') # requires checking for unbound local |
| skip_tests.add('import/gen_context.py') # requires yield_value |
| skip_tests.add('misc/features.py') # requires raise_varargs |
| skip_tests.add('misc/rge_sm.py') # requires yield |
| skip_tests.add('misc/print_exception.py') # because native doesn't have proper traceback info |
| skip_tests.add('misc/sys_exc_info.py') # sys.exc_info() is not supported for native |
| skip_tests.add('micropython/emg_exc.py') # because native doesn't have proper traceback info |
| skip_tests.add('micropython/heapalloc_traceback.py') # because native doesn't have proper traceback info |
| skip_tests.add('micropython/heapalloc_iter.py') # requires generators |
| skip_tests.add('micropython/schedule.py') # native code doesn't check pending events |
| |
| for test_file in tests: |
| test_file = test_file.replace('\\', '/') |
| test_basename = os.path.basename(test_file) |
| test_name = os.path.splitext(test_basename)[0] |
| is_native = test_name.startswith("native_") or test_name.startswith("viper_") |
| is_endian = test_name.endswith("_endian") |
| is_int_big = test_name.startswith("int_big") or test_name.endswith("_intbig") |
| is_set_type = test_name.startswith("set_") or test_name.startswith("frozenset") |
| is_async = test_name.startswith("async_") |
| is_const = test_name.startswith("const") |
| |
| skip_it = test_file in skip_tests |
| skip_it |= skip_native and is_native |
| skip_it |= skip_endian and is_endian |
| skip_it |= skip_int_big and is_int_big |
| skip_it |= skip_set_type and is_set_type |
| skip_it |= skip_async and is_async |
| skip_it |= skip_const and is_const |
| skip_it |= skip_revops and test_name.startswith("class_reverse_op") |
| |
| if skip_it: |
| print("skip ", test_file) |
| skipped_tests.append(test_name) |
| continue |
| |
| # get expected output |
| test_file_expected = test_file + '.exp' |
| if os.path.isfile(test_file_expected): |
| # expected output given by a file, so read that in |
| with open(test_file_expected, 'rb') as f: |
| output_expected = f.read() |
| else: |
| # run CPython to work out expected output |
| try: |
| output_expected = subprocess.check_output([CPYTHON3, '-B', test_file]) |
| if args.write_exp: |
| with open(test_file_expected, 'wb') as f: |
| f.write(output_expected) |
| except subprocess.CalledProcessError: |
| output_expected = b'CPYTHON3 CRASH' |
| |
| # canonical form for all host platforms is to use \n for end-of-line |
| output_expected = output_expected.replace(b'\r\n', b'\n') |
| |
| if args.write_exp: |
| continue |
| |
| # run MicroPython |
| output_mupy = run_micropython(pyb, args, test_file) |
| |
| if output_mupy == b'SKIP\n': |
| print("skip ", test_file) |
| skipped_tests.append(test_name) |
| continue |
| |
| testcase_count += len(output_expected.splitlines()) |
| |
| filename_expected = test_basename + ".exp" |
| filename_mupy = test_basename + ".out" |
| |
| if output_expected == output_mupy: |
| print("pass ", test_file) |
| passed_count += 1 |
| rm_f(filename_expected) |
| rm_f(filename_mupy) |
| else: |
| with open(filename_expected, "wb") as f: |
| f.write(output_expected) |
| with open(filename_mupy, "wb") as f: |
| f.write(output_mupy) |
| print("FAIL ", test_file) |
| failed_tests.append(test_name) |
| |
| test_count += 1 |
| |
| print("{} tests performed ({} individual testcases)".format(test_count, testcase_count)) |
| print("{} tests passed".format(passed_count)) |
| |
| if len(skipped_tests) > 0: |
| print("{} tests skipped: {}".format(len(skipped_tests), ' '.join(skipped_tests))) |
| if len(failed_tests) > 0: |
| print("{} tests failed: {}".format(len(failed_tests), ' '.join(failed_tests))) |
| return False |
| |
| # all tests succeeded |
| return True |
| |
| def main(): |
| cmd_parser = argparse.ArgumentParser(description='Run tests for MicroPython.') |
| cmd_parser.add_argument('--target', default='unix', help='the target platform') |
| cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device or the IP address of the pyboard') |
| cmd_parser.add_argument('-b', '--baudrate', default=115200, help='the baud rate of the serial device') |
| cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username') |
| cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password') |
| cmd_parser.add_argument('-d', '--test-dirs', nargs='*', help='input test directories (if no files given)') |
| cmd_parser.add_argument('--write-exp', action='store_true', help='save .exp files to run tests w/o CPython') |
| cmd_parser.add_argument('--emit', default='bytecode', help='MicroPython emitter to use (bytecode or native)') |
| cmd_parser.add_argument('--heapsize', help='heapsize to use (use default if not specified)') |
| cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first') |
| cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests') |
| cmd_parser.add_argument('files', nargs='*', help='input test files') |
| args = cmd_parser.parse_args() |
| |
| EXTERNAL_TARGETS = ('pyboard', 'wipy', 'esp8266', 'minimal') |
| if args.target in EXTERNAL_TARGETS: |
| import pyboard |
| pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password) |
| pyb.enter_raw_repl() |
| elif args.target == 'unix': |
| pyb = None |
| else: |
| raise ValueError('target must be either %s or unix' % ", ".join(EXTERNAL_TARGETS)) |
| |
| if len(args.files) == 0: |
| if args.test_dirs is None: |
| if args.target == 'pyboard': |
| # run pyboard tests |
| test_dirs = ('basics', 'micropython', 'float', 'misc', 'stress', 'extmod', 'pyb', 'pybnative', 'inlineasm') |
| elif args.target in ('esp8266', 'minimal'): |
| test_dirs = ('basics', 'micropython', 'float', 'misc', 'extmod') |
| elif args.target == 'wipy': |
| # run WiPy tests |
| test_dirs = ('basics', 'micropython', 'misc', 'extmod', 'wipy') |
| else: |
| # run PC tests |
| test_dirs = ('basics', 'micropython', 'float', 'import', 'io', 'misc', 'stress', 'unicode', 'extmod', 'unix', 'cmdline') |
| else: |
| # run tests from these directories |
| test_dirs = args.test_dirs |
| tests = sorted(test_file for test_files in (glob('{}/*.py'.format(dir)) for dir in test_dirs) for test_file in test_files) |
| else: |
| # tests explicitly given |
| tests = args.files |
| |
| if not args.keep_path: |
| # clear search path to make sure tests use only builtin modules |
| os.environ['MICROPYPATH'] = '' |
| |
| # Even if we run completely different tests in a different directory, |
| # we need to access feature_check's from the same directory as the |
| # run-tests script itself. |
| base_path = os.path.dirname(sys.argv[0]) or "." |
| try: |
| res = run_tests(pyb, tests, args, base_path) |
| finally: |
| if pyb: |
| pyb.close() |
| |
| if not res: |
| sys.exit(1) |
| |
| if __name__ == "__main__": |
| main() |