Changeset View
Changeset View
Standalone View
Standalone View
test/functional/test_runner.py
| Show First 20 Lines • Show All 171 Lines • ▼ Show 20 Lines | def main(): | ||||
| parser.add_argument('--keepcache', '-k', action='store_true', | parser.add_argument('--keepcache', '-k', action='store_true', | ||||
| help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') | help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') | ||||
| parser.add_argument('--quiet', '-q', action='store_true', | parser.add_argument('--quiet', '-q', action='store_true', | ||||
| help='only print results summary and failure logs') | help='only print results summary and failure logs') | ||||
| parser.add_argument('--tmpdirprefix', '-t', | parser.add_argument('--tmpdirprefix', '-t', | ||||
| default=os.path.join(build_dir, 'test', 'tmp'), help="Root directory for datadirs") | default=os.path.join(build_dir, 'test', 'tmp'), help="Root directory for datadirs") | ||||
| parser.add_argument('--junitoutput', '-J', default='junit_results.xml', | parser.add_argument('--junitoutput', '-J', default='junit_results.xml', | ||||
| help="File that will store JUnit formatted test results. If no absolute path is given it is treated as relative to the temporary directory.") | help="File that will store JUnit formatted test results. If no absolute path is given it is treated as relative to the temporary directory.") | ||||
| parser.add_argument('--testsuitename', '-n', default='Bitcoin ABC functional tests', | |||||
| help="Name of the test suite, as it will appear in the logs and in the JUnit report.") | |||||
| args, unknown_args = parser.parse_known_args() | args, unknown_args = parser.parse_known_args() | ||||
| # args to be passed on always start with two dashes; tests are the | # args to be passed on always start with two dashes; tests are the | ||||
| # remaining unknown args | # remaining unknown args | ||||
| tests = [arg for arg in unknown_args if arg[:2] != "--"] | tests = [arg for arg in unknown_args if arg[:2] != "--"] | ||||
| passon_args = [arg for arg in unknown_args if arg[:2] == "--"] | passon_args = [arg for arg in unknown_args if arg[:2] == "--"] | ||||
| passon_args.append("--configfile={}".format(configfile)) | passon_args.append("--configfile={}".format(configfile)) | ||||
| # Set up logging | # Set up logging | ||||
| logging_level = logging.INFO if args.quiet else logging.DEBUG | logging_level = logging.INFO if args.quiet else logging.DEBUG | ||||
| logging.basicConfig(format='%(message)s', level=logging_level) | logging.basicConfig(format='%(message)s', level=logging_level) | ||||
| logging.info("Starting {}".format(args.testsuitename)) | |||||
| # Create base test directory | # Create base test directory | ||||
| tmpdir = os.path.join("{}", "bitcoin_test_runner_{:%Y%m%d_%H%M%S}").format( | tmpdir = os.path.join("{}", "bitcoin_test_runner_{:%Y%m%d_%H%M%S}").format( | ||||
| args.tmpdirprefix, datetime.datetime.now()) | args.tmpdirprefix, datetime.datetime.now()) | ||||
| os.makedirs(tmpdir) | os.makedirs(tmpdir) | ||||
| logging.debug("Temporary test directory at {}".format(tmpdir)) | logging.debug("Temporary test directory at {}".format(tmpdir)) | ||||
| ▲ Show 20 Lines • Show All 97 Lines • ▼ Show 20 Lines | if args.help: | ||||
| [sys.executable, os.path.join(tests_dir, test_list[0]), '-h']) | [sys.executable, os.path.join(tests_dir, test_list[0]), '-h']) | ||||
| sys.exit(0) | sys.exit(0) | ||||
| if not args.keepcache: | if not args.keepcache: | ||||
| shutil.rmtree(os.path.join(build_dir, "test", | shutil.rmtree(os.path.join(build_dir, "test", | ||||
| "cache"), ignore_errors=True) | "cache"), ignore_errors=True) | ||||
| run_tests(test_list, build_dir, tests_dir, args.junitoutput, | run_tests(test_list, build_dir, tests_dir, args.junitoutput, | ||||
| tmpdir, args.jobs, args.coverage, passon_args, args.combinedlogslen, build_timings) | tmpdir, args.jobs, args.testsuitename, args.coverage, passon_args, args.combinedlogslen, build_timings) | ||||
| def run_tests(test_list, build_dir, tests_dir, junitoutput, tmpdir, num_jobs, enable_coverage=False, args=[], combined_logs_len=0, build_timings=None): | def run_tests(test_list, build_dir, tests_dir, junitoutput, tmpdir, num_jobs, test_suite_name, enable_coverage=False, args=[], combined_logs_len=0, build_timings=None): | ||||
| # Warn if bitcoind is already running (unix only) | # Warn if bitcoind is already running (unix only) | ||||
| try: | try: | ||||
| pidofOutput = subprocess.check_output(["pidof", "bitcoind"]) | pidofOutput = subprocess.check_output(["pidof", "bitcoind"]) | ||||
| if pidofOutput is not None and pidofOutput != b'': | if pidofOutput is not None and pidofOutput != b'': | ||||
| print("{}WARNING!{} There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!".format( | print("{}WARNING!{} There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!".format( | ||||
| BOLD[1], BOLD[0])) | BOLD[1], BOLD[0])) | ||||
| except (OSError, subprocess.SubprocessError): | except (OSError, subprocess.SubprocessError): | ||||
| pass | pass | ||||
| Show All 27 Lines | def run_tests(test_list, build_dir, tests_dir, junitoutput, tmpdir, num_jobs, test_suite_name, enable_coverage=False, args=[], combined_logs_len=0, build_timings=None): | ||||
| time0 = time.time() | time0 = time.time() | ||||
| test_results = execute_test_processes( | test_results = execute_test_processes( | ||||
| num_jobs, test_list, tests_dir, tmpdir, flags) | num_jobs, test_list, tests_dir, tmpdir, flags) | ||||
| runtime = int(time.time() - time0) | runtime = int(time.time() - time0) | ||||
| max_len_name = len(max(test_list, key=len)) | max_len_name = len(max(test_list, key=len)) | ||||
| print_results(test_results, tests_dir, max_len_name, | print_results(test_results, tests_dir, max_len_name, | ||||
| runtime, combined_logs_len) | runtime, combined_logs_len) | ||||
| save_results_as_junit(test_results, junitoutput, runtime) | save_results_as_junit(test_results, junitoutput, runtime, test_suite_name) | ||||
| if (build_timings is not None): | if (build_timings is not None): | ||||
| build_timings.save_timings(test_results) | build_timings.save_timings(test_results) | ||||
| if coverage: | if coverage: | ||||
| coverage.report_rpc_coverage() | coverage.report_rpc_coverage() | ||||
| logging.debug("Cleaning up coverage data") | logging.debug("Cleaning up coverage data") | ||||
| ▲ Show 20 Lines • Show All 312 Lines • ▼ Show 20 Lines | def _get_uncovered_rpc_commands(self): | ||||
| for filename in coverage_filenames: | for filename in coverage_filenames: | ||||
| with open(filename, 'r', encoding="utf8") as f: | with open(filename, 'r', encoding="utf8") as f: | ||||
| covered_cmds.update([i.strip() for i in f.readlines()]) | covered_cmds.update([i.strip() for i in f.readlines()]) | ||||
| return all_cmds - covered_cmds | return all_cmds - covered_cmds | ||||
| def save_results_as_junit(test_results, file_name, time): | def save_results_as_junit(test_results, file_name, time, test_suite_name): | ||||
| """ | """ | ||||
| Save tests results to file in JUnit format | Save tests results to file in JUnit format | ||||
| See http://llg.cubic.org/docs/junit/ for specification of format | See http://llg.cubic.org/docs/junit/ for specification of format | ||||
| """ | """ | ||||
| e_test_suite = ET.Element("testsuite", | e_test_suite = ET.Element("testsuite", | ||||
| {"name": "bitcoin_abc_tests", | {"name": "{}".format(test_suite_name), | ||||
| "tests": str(len(test_results)), | "tests": str(len(test_results)), | ||||
| # "errors": | # "errors": | ||||
| "failures": str(len([t for t in test_results if t.status == "Failed"])), | "failures": str(len([t for t in test_results if t.status == "Failed"])), | ||||
| "id": "0", | "id": "0", | ||||
| "skipped": str(len([t for t in test_results if t.status == "Skipped"])), | "skipped": str(len([t for t in test_results if t.status == "Skipped"])), | ||||
| "time": str(time), | "time": str(time), | ||||
| "timestamp": datetime.datetime.now().isoformat('T') | "timestamp": datetime.datetime.now().isoformat('T') | ||||
| }) | }) | ||||
| ▲ Show 20 Lines • Show All 70 Lines • Show Last 20 Lines | |||||