Changeset View
Changeset View
Standalone View
Standalone View
test/functional/test_runner.py
| Show First 20 Lines • Show All 52 Lines • ▼ Show 20 Lines | if os.name == 'posix': | ||||
| BLUE = ('\033[0m', '\033[0;34m') | BLUE = ('\033[0m', '\033[0;34m') | ||||
| RED = ('\033[0m', '\033[0;31m') | RED = ('\033[0m', '\033[0;31m') | ||||
| GREY = ('\033[0m', '\033[1;30m') | GREY = ('\033[0m', '\033[1;30m') | ||||
| TEST_EXIT_PASSED = 0 | TEST_EXIT_PASSED = 0 | ||||
| TEST_EXIT_SKIPPED = 77 | TEST_EXIT_SKIPPED = 77 | ||||
| NON_SCRIPTS = [ | NON_SCRIPTS = [ | ||||
| # These are python files that live in the functional tests directory, but are not test scripts. | # These are python files that live in the functional tests directory, but | ||||
| # are not test scripts. | |||||
| "combine_logs.py", | "combine_logs.py", | ||||
| "create_cache.py", | "create_cache.py", | ||||
| "test_runner.py", | "test_runner.py", | ||||
| ] | ] | ||||
| TEST_PARAMS = { | TEST_PARAMS = { | ||||
| # Some test can be run with additional parameters. | # Some test can be run with additional parameters. | ||||
| # When a test is listed here, the it will be run without parameters | # When a test is listed here, the it will be run without parameters | ||||
| # as well as with additional parameters listed here. | # as well as with additional parameters listed here. | ||||
| # This: | # This: | ||||
| # example "testName" : [["--param1", "--param2"] , ["--param3"]] | # example "testName" : [["--param1", "--param2"] , ["--param3"]] | ||||
| # will run the test 3 times: | # will run the test 3 times: | ||||
| # testName | # testName | ||||
| # testName --param1 --param2 | # testName --param1 --param2 | ||||
| # testname --param3 | # testname --param3 | ||||
| "wallet_txn_doublespend.py": [["--mineblock"]], | "wallet_txn_doublespend.py": [["--mineblock"]], | ||||
| "wallet_txn_clone.py": [["--mineblock"]], | "wallet_txn_clone.py": [["--mineblock"]], | ||||
| "wallet_createwallet.py": [["--usecli"]], | "wallet_createwallet.py": [["--usecli"]], | ||||
| "wallet_multiwallet.py": [["--usecli"]], | "wallet_multiwallet.py": [["--usecli"]], | ||||
| } | } | ||||
| # Used to limit the number of tests, when list of tests is not provided on command line | # Used to limit the number of tests, when list of tests is not provided on command line | ||||
| # When --extended is specified, we run all tests, otherwise | # When --extended is specified, we run all tests, otherwise | ||||
| # we only run a test if its execution time in seconds does not exceed EXTENDED_CUTOFF | # we only run a test if its execution time in seconds does not exceed | ||||
| # EXTENDED_CUTOFF | |||||
| DEFAULT_EXTENDED_CUTOFF = 40 | DEFAULT_EXTENDED_CUTOFF = 40 | ||||
| DEFAULT_JOBS = (multiprocessing.cpu_count() // 3) + 1 | DEFAULT_JOBS = (multiprocessing.cpu_count() // 3) + 1 | ||||
| class TestCase(): | class TestCase(): | ||||
| """ | """ | ||||
| Data structure to hold and run information necessary to launch a test case. | Data structure to hold and run information necessary to launch a test case. | ||||
| """ | """ | ||||
| Show All 29 Lines | def run(self, portseed_offset): | ||||
| log_stdout.close(), log_stderr.close() | log_stdout.close(), log_stderr.close() | ||||
| if process.returncode == TEST_EXIT_PASSED and stderr == "": | if process.returncode == TEST_EXIT_PASSED and stderr == "": | ||||
| status = "Passed" | status = "Passed" | ||||
| elif process.returncode == TEST_EXIT_SKIPPED: | elif process.returncode == TEST_EXIT_SKIPPED: | ||||
| status = "Skipped" | status = "Skipped" | ||||
| else: | else: | ||||
| status = "Failed" | status = "Failed" | ||||
| return TestResult(self.test_num, name, testdir, status, int(time.time() - time0), stdout, stderr) | return TestResult(self.test_num, name, testdir, status, | ||||
| int(time.time() - time0), stdout, stderr) | |||||
| def on_ci(): | def on_ci(): | ||||
| return os.getenv('TRAVIS') == 'true' or os.getenv('TEAMCITY_VERSION') is not None | return os.getenv('TRAVIS') == 'true' or os.getenv( | ||||
| 'TEAMCITY_VERSION') is not None | |||||
| def main(): | def main(): | ||||
| # Read config generated by configure. | # Read config generated by configure. | ||||
| config = configparser.ConfigParser() | config = configparser.ConfigParser() | ||||
| configfile = os.path.join(os.path.abspath( | configfile = os.path.join(os.path.abspath( | ||||
| os.path.dirname(__file__)), "..", "config.ini") | os.path.dirname(__file__)), "..", "config.ini") | ||||
| config.read_file(open(configfile, encoding="utf8")) | config.read_file(open(configfile, encoding="utf8")) | ||||
| ▲ Show 20 Lines • Show All 155 Lines • ▼ Show 20 Lines | def main(): | ||||
| if not args.keepcache: | if not args.keepcache: | ||||
| shutil.rmtree(os.path.join(build_dir, "test", | shutil.rmtree(os.path.join(build_dir, "test", | ||||
| "cache"), ignore_errors=True) | "cache"), ignore_errors=True) | ||||
| run_tests(test_list, build_dir, tests_dir, args.junitoutput, | run_tests(test_list, build_dir, tests_dir, args.junitoutput, | ||||
| tmpdir, args.jobs, args.testsuitename, args.coverage, passon_args, args.combinedlogslen, build_timings) | tmpdir, args.jobs, args.testsuitename, args.coverage, passon_args, args.combinedlogslen, build_timings) | ||||
| def run_tests(test_list, build_dir, tests_dir, junitoutput, tmpdir, num_jobs, test_suite_name, enable_coverage=False, args=[], combined_logs_len=0, build_timings=None): | def run_tests(test_list, build_dir, tests_dir, junitoutput, tmpdir, num_jobs, test_suite_name, | ||||
| enable_coverage=False, args=[], combined_logs_len=0, build_timings=None): | |||||
| # Warn if bitcoind is already running (unix only) | # Warn if bitcoind is already running (unix only) | ||||
| try: | try: | ||||
| pidofOutput = subprocess.check_output(["pidof", "bitcoind"]) | pidofOutput = subprocess.check_output(["pidof", "bitcoind"]) | ||||
| if pidofOutput is not None and pidofOutput != b'': | if pidofOutput is not None and pidofOutput != b'': | ||||
| print("{}WARNING!{} There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!".format( | print("{}WARNING!{} There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!".format( | ||||
| BOLD[1], BOLD[0])) | BOLD[1], BOLD[0])) | ||||
| except (OSError, subprocess.SubprocessError): | except (OSError, subprocess.SubprocessError): | ||||
| pass | pass | ||||
| ▲ Show 20 Lines • Show All 174 Lines • ▼ Show 20 Lines | def execute_test_processes(num_jobs, test_list, tests_dir, tmpdir, flags): | ||||
| # Flush our queues so the threads exit | # Flush our queues so the threads exit | ||||
| update_queue.put(None) | update_queue.put(None) | ||||
| for j in range(num_jobs): | for j in range(num_jobs): | ||||
| job_queue.put(None) | job_queue.put(None) | ||||
| return test_results | return test_results | ||||
| def print_results(test_results, tests_dir, max_len_name, runtime, combined_logs_len): | def print_results(test_results, tests_dir, max_len_name, | ||||
| runtime, combined_logs_len): | |||||
| results = "\n" + BOLD[1] + "{} | {} | {}\n\n".format( | results = "\n" + BOLD[1] + "{} | {} | {}\n\n".format( | ||||
| "TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0] | "TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0] | ||||
| test_results.sort(key=TestResult.sort_key) | test_results.sort(key=TestResult.sort_key) | ||||
| all_passed = True | all_passed = True | ||||
| time_sum = 0 | time_sum = 0 | ||||
| for test_result in test_results: | for test_result in test_results: | ||||
| all_passed = all_passed and test_result.was_successful | all_passed = all_passed and test_result.was_successful | ||||
| time_sum += test_result.time | time_sum += test_result.time | ||||
| test_result.padding = max_len_name | test_result.padding = max_len_name | ||||
| results += str(test_result) | results += str(test_result) | ||||
| testdir = test_result.testdir | testdir = test_result.testdir | ||||
| if combined_logs_len and os.path.isdir(testdir): | if combined_logs_len and os.path.isdir(testdir): | ||||
| # Print the final `combinedlogslen` lines of the combined logs | # Print the final `combinedlogslen` lines of the combined logs | ||||
| print('{}Combine the logs and print the last {} lines ...{}'.format( | print('{}Combine the logs and print the last {} lines ...{}'.format( | ||||
| BOLD[1], combined_logs_len, BOLD[0])) | BOLD[1], combined_logs_len, BOLD[0])) | ||||
| print('\n============') | print('\n============') | ||||
| print('{}Combined log for {}:{}'.format(BOLD[1], testdir, BOLD[0])) | print('{}Combined log for {}:{}'.format(BOLD[1], testdir, BOLD[0])) | ||||
| print('============\n') | print('============\n') | ||||
| combined_logs, _ = subprocess.Popen([sys.executable, os.path.join( | combined_logs, _ = subprocess.Popen([sys.executable, os.path.join( | ||||
| tests_dir, 'combine_logs.py'), '-c', testdir], universal_newlines=True, stdout=subprocess.PIPE).communicate() | tests_dir, 'combine_logs.py'), '-c', testdir], universal_newlines=True, stdout=subprocess.PIPE).communicate() | ||||
| print("\n".join(deque(combined_logs.splitlines(), combined_logs_len))) | print( | ||||
| "\n".join( | |||||
| deque( | |||||
| combined_logs.splitlines(), | |||||
| combined_logs_len))) | |||||
| status = TICK + "Passed" if all_passed else CROSS + "Failed" | status = TICK + "Passed" if all_passed else CROSS + "Failed" | ||||
| if not all_passed: | if not all_passed: | ||||
| results += RED[1] | results += RED[1] | ||||
| results += BOLD[1] + "\n{} | {} | {} s (accumulated) \n".format( | results += BOLD[1] + "\n{} | {} | {} s (accumulated) \n".format( | ||||
| "ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0] | "ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0] | ||||
| if not all_passed: | if not all_passed: | ||||
| results += RED[0] | results += RED[0] | ||||
| ▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines | def get_tests_to_run(test_list, test_params, cutoff, src_timings): | ||||
| Timings from build directory override those from src directory | Timings from build directory override those from src directory | ||||
| """ | """ | ||||
| def get_test_time(test): | def get_test_time(test): | ||||
| # Return 0 if test is unknown to always run it | # Return 0 if test is unknown to always run it | ||||
| return next( | return next( | ||||
| (x['time'] for x in src_timings.existing_timings if x['name'] == test), 0) | (x['time'] for x in src_timings.existing_timings if x['name'] == test), 0) | ||||
| # Some tests must also be run with additional parameters. Add them to the list. | # Some tests must also be run with additional parameters. Add them to the | ||||
| # list. | |||||
| tests_with_params = [] | tests_with_params = [] | ||||
| for test_name in test_list: | for test_name in test_list: | ||||
| # always execute a test without parameters | # always execute a test without parameters | ||||
| tests_with_params.append(test_name) | tests_with_params.append(test_name) | ||||
| params = test_params.get(test_name) | params = test_params.get(test_name) | ||||
| if params is not None: | if params is not None: | ||||
| tests_with_params.extend( | tests_with_params.extend( | ||||
| [test_name + " " + " ".join(p) for p in params]) | [test_name + " " + " ".join(p) for p in params]) | ||||
| ▲ Show 20 Lines • Show All 160 Lines • Show Last 20 Lines | |||||