|
| 1 | +#!/usr/bin/env python |
| 2 | + |
| 3 | +# Copyright JS Foundation and other contributors, http://js.foundation |
| 4 | +# |
| 5 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | +# you may not use this file except in compliance with the License. |
| 7 | +# You may obtain a copy of the License at |
| 8 | +# |
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +# |
| 11 | +# Unless required by applicable law or agreed to in writing, software |
| 12 | +# distributed under the License is distributed on an "AS IS" BASIS |
| 13 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | +# See the License for the specific language governing permissions and |
| 15 | +# limitations under the License. |
| 16 | + |
| 17 | +from __future__ import print_function |
| 18 | +import argparse |
| 19 | +import glob |
| 20 | +import os |
| 21 | +import subprocess |
| 22 | +import sys |
| 23 | + |
| 24 | +TERM_NORMAL = '\033[0m' |
| 25 | +TERM_RED = '\033[1;31m' |
| 26 | +TERM_GREEN = '\033[1;32m' |
| 27 | + |
| 28 | +def get_arguments(): |
| 29 | + parser = argparse.ArgumentParser() |
| 30 | + parser.add_argument('-q', '--quiet', action='store_true', |
| 31 | + help='Only print out failing tests') |
| 32 | + parser.add_argument('--engine', metavar='FILE', |
| 33 | + help='JerryScript binary to run tests with') |
| 34 | + parser.add_argument('--test-list', metavar='FILE', |
| 35 | + help='File contains test paths to run') |
| 36 | + parser.add_argument('--skip-list', metavar='LIST', |
| 37 | + help='Add a comma separated list of patterns of the excluded JS-tests') |
| 38 | + parser.add_argument('--test-dir', metavar='DIR', |
| 39 | + help='Directory contains tests to run') |
| 40 | + parser.add_argument('--snapshot', action='store_true', |
| 41 | + help='Snapshot test') |
| 42 | + |
| 43 | + script_args = parser.parse_args() |
| 44 | + if script_args.skip_list: |
| 45 | + script_args.skip_list = script_args.skip_list.split(',') |
| 46 | + else: |
| 47 | + script_args.skip_list = [] |
| 48 | + |
| 49 | + return script_args |
| 50 | + |
| 51 | + |
| 52 | +def get_tests(test_dir, test_list, skip_list): |
| 53 | + tests = [] |
| 54 | + if test_dir: |
| 55 | + tests = [] |
| 56 | + for root, dirs, files in os.walk(test_dir): |
| 57 | + tests.extend([os.path.join(root, file) for file in files if file.endswith('.js')]) |
| 58 | + |
| 59 | + if test_list: |
| 60 | + with open(test_list, "r") as fd: |
| 61 | + lines = fd.read().splitlines() |
| 62 | + dirname = os.path.dirname(test_list) |
| 63 | + for test in lines: |
| 64 | + tests.append(os.path.normpath(os.path.join(dirname, test))) |
| 65 | + |
| 66 | + tests.sort() |
| 67 | + |
| 68 | + def filter_tests(test): |
| 69 | + for skipped in skip_list: |
| 70 | + if skipped in test: |
| 71 | + return False |
| 72 | + return True |
| 73 | + |
| 74 | + return filter(filter_tests, tests) |
| 75 | + |
| 76 | + |
| 77 | +def get_platform_cmd_prefix(): |
| 78 | + if sys.platform == 'win32': |
| 79 | + return ['cmd', '/S', '/C'] |
| 80 | + return [] |
| 81 | + |
| 82 | + |
| 83 | +def main(args): |
| 84 | + tests = get_tests(args.test_dir, args.test_list, args.skip_list) |
| 85 | + total = len(tests) |
| 86 | + if total == 0: |
| 87 | + print("No test to execute.") |
| 88 | + return 1 |
| 89 | + |
| 90 | + tested = 0 |
| 91 | + passed = 0 |
| 92 | + failed = 0 |
| 93 | + |
| 94 | + runtime = os.environ.get('RUNTIME') |
| 95 | + |
| 96 | + test_cmd = get_platform_cmd_prefix() |
| 97 | + if runtime: |
| 98 | + test_cmd.append(runtime) |
| 99 | + test_cmd.append(args.engine) |
| 100 | + |
| 101 | + generate_snapshot_cmd = get_platform_cmd_prefix() |
| 102 | + if runtime: |
| 103 | + generate_snapshot_cmd.append(runtime) |
| 104 | + |
| 105 | + if sys.platform == 'win32': |
| 106 | + generate_snapshot_cmd.append(args.engine[:-4] + '-snapshot.exe') |
| 107 | + else: |
| 108 | + generate_snapshot_cmd.append(args.engine + '-snapshot') |
| 109 | + |
| 110 | + generate_snapshot_cmd.append('generate') |
| 111 | + |
| 112 | + for test in tests: |
| 113 | + tested += 1 |
| 114 | + testpath = os.path.relpath(test) |
| 115 | + is_expected_to_fail = '\\fail\\' in test |
| 116 | + |
| 117 | + if args.snapshot: |
| 118 | + process = subprocess.Popen(generate_snapshot_cmd + [test], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) |
| 119 | + stdout = process.communicate()[0] |
| 120 | + |
| 121 | + if is_expected_to_fail or not process.returncode: |
| 122 | + if not args.quiet: |
| 123 | + print("[%4d/%4d] %sPASS%s: %s (generate snapshot)%s" % (tested, total, TERM_GREEN,' (XFAIL)' if process.returncode else '', testpath, TERM_NORMAL)) |
| 124 | + else: |
| 125 | + print("[%4d/%4d] %sFAIL (%d): %s (generate snapshot)%s" % (tested, total, TERM_RED, process.returncode, testpath, TERM_NORMAL)) |
| 126 | + print("================================================") |
| 127 | + print(stdout) |
| 128 | + print("================================================") |
| 129 | + |
| 130 | + if process.returncode: |
| 131 | + if is_expected_to_fail: |
| 132 | + passed += 1 |
| 133 | + else: |
| 134 | + failed += 1 |
| 135 | + continue |
| 136 | + |
| 137 | + if args.snapshot: |
| 138 | + process = subprocess.Popen(test_cmd + ['--exec-snapshot', 'js.snapshot'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) |
| 139 | + stdout = process.communicate()[0] |
| 140 | + os.remove('js.snapshot') |
| 141 | + else: |
| 142 | + process = subprocess.Popen(test_cmd + [test], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) |
| 143 | + stdout = process.communicate()[0] |
| 144 | + |
| 145 | + |
| 146 | + if bool(process.returncode) == is_expected_to_fail: |
| 147 | + passed += 1 |
| 148 | + if not args.quiet: |
| 149 | + print("[%4d/%4d] %sPASS%s: %s%s%s" % (tested, total, TERM_GREEN,' (XFAIL)' if is_expected_to_fail else '', testpath, ' (execute snapshot)' if args.snapshot else '',TERM_NORMAL)) |
| 150 | + else: |
| 151 | + failed += 1 |
| 152 | + print("[%4d/%4d] %sFAIL%s (%d): %s%s%s" % (tested, total, TERM_RED, ' (XPASS)' if is_expected_to_fail else '', process.returncode, testpath, ' (execute snapshot)' if args.snapshot else '',TERM_NORMAL)) |
| 153 | + print("================================================") |
| 154 | + print(stdout) |
| 155 | + print("================================================") |
| 156 | + |
| 157 | + print ("\n[summary] ... \n") # TODO: create similar summary string to sh runner |
| 158 | + print("TOTAL: %d" % total) |
| 159 | + print("%sPASS: %d%s" % (TERM_GREEN, passed, TERM_NORMAL)) |
| 160 | + print("%sFAIL: %d%s\n" % (TERM_RED, failed, TERM_NORMAL)) |
| 161 | + |
| 162 | + success_color = TERM_GREEN if passed == total else TERM_RED |
| 163 | + print("%sSuccess: %d%%%s" % (success_color, passed*100/total, TERM_NORMAL)) |
| 164 | + |
| 165 | + if failed > 0: |
| 166 | + return 1 |
| 167 | + |
| 168 | + return 0 |
| 169 | + |
| 170 | + |
| 171 | +if __name__ == "__main__": |
| 172 | + sys.exit(main(get_arguments())) |
0 commit comments