Changeset View
Changeset View
Standalone View
Standalone View
test/functional/test_runner.py
| Show First 20 Lines • Show All 98 Lines • ▼ Show 20 Lines | class TestCase(): | ||||
| def run(self, portseed_offset): | def run(self, portseed_offset): | ||||
| t = self.test_case | t = self.test_case | ||||
| portseed = self.test_num + portseed_offset | portseed = self.test_num + portseed_offset | ||||
| portseed_arg = ["--portseed={}".format(portseed)] | portseed_arg = ["--portseed={}".format(portseed)] | ||||
| log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16) | log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16) | ||||
| log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) | log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) | ||||
| test_argv = t.split() | test_argv = t.split() | ||||
| tmpdir = [os.path.join("--tmpdir=%s", "%s_%s") % | tmpdir = [os.path.join("--tmpdir={}", "{}_{}").format( | ||||
| (self.tmpdir, re.sub(".py$", "", t), portseed)] | self.tmpdir, re.sub(".py$", "", t), portseed)] | ||||
| name = t | name = t | ||||
| time0 = time.time() | time0 = time.time() | ||||
| process = subprocess.Popen([os.path.join(self.tests_dir, test_argv[0])] + test_argv[1:] + self.flags + portseed_arg + tmpdir, | process = subprocess.Popen([os.path.join(self.tests_dir, test_argv[0])] + test_argv[1:] + self.flags + portseed_arg + tmpdir, | ||||
| universal_newlines=True, | universal_newlines=True, | ||||
| stdout=log_stdout, | stdout=log_stdout, | ||||
| stderr=log_stderr) | stderr=log_stderr) | ||||
| process.wait() | process.wait() | ||||
| ▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines | def main(): | ||||
| parser.add_argument('--junitouput', '-ju', | parser.add_argument('--junitouput', '-ju', | ||||
| default=os.path.join(build_dir, 'junit_results.xml'), help="file that will store JUnit formated test results.") | default=os.path.join(build_dir, 'junit_results.xml'), help="file that will store JUnit formated test results.") | ||||
| args, unknown_args = parser.parse_known_args() | args, unknown_args = parser.parse_known_args() | ||||
| # Create a set to store arguments and create the passon string | # Create a set to store arguments and create the passon string | ||||
| tests = set(arg for arg in unknown_args if arg[:2] != "--") | tests = set(arg for arg in unknown_args if arg[:2] != "--") | ||||
| passon_args = [arg for arg in unknown_args if arg[:2] == "--"] | passon_args = [arg for arg in unknown_args if arg[:2] == "--"] | ||||
| passon_args.append("--configfile=%s" % configfile) | passon_args.append("--configfile={}".format(configfile)) | ||||
| # Set up logging | # Set up logging | ||||
| logging_level = logging.INFO if args.quiet else logging.DEBUG | logging_level = logging.INFO if args.quiet else logging.DEBUG | ||||
| logging.basicConfig(format='%(message)s', level=logging_level) | logging.basicConfig(format='%(message)s', level=logging_level) | ||||
| # Create base test directory | # Create base test directory | ||||
| tmpdir = os.path.join("%s", "bitcoin_test_runner_%s") % ( | tmpdir = os.path.join("{}", "bitcoin_test_runner_{:%Y%m%d_%H%M%S}").format( | ||||
| args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) | args.tmpdirprefix, datetime.datetime.now()) | ||||
| os.makedirs(tmpdir) | os.makedirs(tmpdir) | ||||
| logging.debug("Temporary test directory at %s" % tmpdir) | logging.debug("Temporary test directory at {}".format(tmpdir)) | ||||
| enable_wallet = config["components"].getboolean("ENABLE_WALLET") | enable_wallet = config["components"].getboolean("ENABLE_WALLET") | ||||
| enable_utils = config["components"].getboolean("ENABLE_UTILS") | enable_utils = config["components"].getboolean("ENABLE_UTILS") | ||||
| enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") | enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") | ||||
| if config["environment"]["EXEEXT"] == ".exe" and not args.force: | if config["environment"]["EXEEXT"] == ".exe" and not args.force: | ||||
| # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 | # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 | ||||
| # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 | # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 | ||||
| ▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines | run_tests(test_list, build_dir, tests_dir, args.junitouput, | ||||
| config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args, build_timings) | config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args, build_timings) | ||||
| def run_tests(test_list, build_dir, tests_dir, junitouput, exeext, tmpdir, num_jobs, enable_coverage=False, args=[], build_timings=None): | def run_tests(test_list, build_dir, tests_dir, junitouput, exeext, tmpdir, num_jobs, enable_coverage=False, args=[], build_timings=None): | ||||
| # Warn if bitcoind is already running (unix only) | # Warn if bitcoind is already running (unix only) | ||||
| try: | try: | ||||
| pidofOutput = subprocess.check_output(["pidof", "bitcoind"]) | pidofOutput = subprocess.check_output(["pidof", "bitcoind"]) | ||||
| if pidofOutput is not None and pidofOutput != b'': | if pidofOutput is not None and pidofOutput != b'': | ||||
| print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % ( | print("{}WARNING!{} There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!".format( | ||||
| BOLD[1], BOLD[0])) | BOLD[1], BOLD[0])) | ||||
| except (OSError, subprocess.SubprocessError): | except (OSError, subprocess.SubprocessError): | ||||
| pass | pass | ||||
| # Warn if there is a cache directory | # Warn if there is a cache directory | ||||
| cache_dir = os.path.join(build_dir, "test", "cache") | cache_dir = os.path.join(build_dir, "test", "cache") | ||||
| if os.path.isdir(cache_dir): | if os.path.isdir(cache_dir): | ||||
| print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % ( | print("{}WARNING!{} There is a cache directory here: {}. If tests fail unexpectedly, try deleting the cache directory.".format( | ||||
| BOLD[1], BOLD[0], cache_dir)) | BOLD[1], BOLD[0], cache_dir)) | ||||
| # Set env vars | # Set env vars | ||||
| if "BITCOIND" not in os.environ: | if "BITCOIND" not in os.environ: | ||||
| os.environ["BITCOIND"] = os.path.join( | os.environ["BITCOIND"] = os.path.join( | ||||
| build_dir, 'src', 'bitcoind' + exeext) | build_dir, 'src', 'bitcoind' + exeext) | ||||
| os.environ["BITCOINCLI"] = os.path.join( | os.environ["BITCOINCLI"] = os.path.join( | ||||
| build_dir, 'src', 'bitcoin-cli' + exeext) | build_dir, 'src', 'bitcoin-cli' + exeext) | ||||
| flags = [os.path.join("--srcdir={}".format(build_dir), "src")] + args | flags = [os.path.join("--srcdir={}".format(build_dir), "src")] + args | ||||
| flags.append("--cachedir=%s" % cache_dir) | flags.append("--cachedir={}".format(cache_dir)) | ||||
| if enable_coverage: | if enable_coverage: | ||||
| coverage = RPCCoverage() | coverage = RPCCoverage() | ||||
| flags.append(coverage.flag) | flags.append(coverage.flag) | ||||
| logging.debug("Initializing coverage directory at %s" % coverage.dir) | logging.debug( | ||||
| "Initializing coverage directory at {}".format(coverage.dir)) | |||||
| else: | else: | ||||
| coverage = None | coverage = None | ||||
| if len(test_list) > 1 and num_jobs > 1: | if len(test_list) > 1 and num_jobs > 1: | ||||
| # Populate cache | # Populate cache | ||||
| try: | try: | ||||
| subprocess.check_output( | subprocess.check_output( | ||||
| [os.path.join(tests_dir, 'create_cache.py')] + flags + [os.path.join("--tmpdir=%s", "cache") % tmpdir]) | [os.path.join(tests_dir, 'create_cache.py')] + flags + [os.path.join("--tmpdir={}", "cache") .format(tmpdir)]) | ||||
| except Exception as e: | except Exception as e: | ||||
| print(e.output) | print(e.output) | ||||
| raise e | raise e | ||||
| # Run Tests | # Run Tests | ||||
| time0 = time.time() | time0 = time.time() | ||||
| test_results = execute_test_processes( | test_results = execute_test_processes( | ||||
| num_jobs, test_list, tests_dir, tmpdir, flags) | num_jobs, test_list, tests_dir, tmpdir, flags) | ||||
| ▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | def handle_message(message, running_jobs): | ||||
| if isinstance(message, TestResult): | if isinstance(message, TestResult): | ||||
| test_result = message | test_result = message | ||||
| running_jobs.remove(test_result.name) | running_jobs.remove(test_result.name) | ||||
| test_results.append(test_result) | test_results.append(test_result) | ||||
| if test_result.status == "Passed": | if test_result.status == "Passed": | ||||
| print("%s%s%s passed, Duration: %s s" % ( | print("{}{}{} passed, Duration: {} s".format( | ||||
| BOLD[1], test_result.name, BOLD[0], test_result.time)) | BOLD[1], test_result.name, BOLD[0], test_result.time)) | ||||
| elif test_result.status == "Skipped": | elif test_result.status == "Skipped": | ||||
| print("%s%s%s skipped" % | print("{}{}{} skipped".format( | ||||
| (BOLD[1], test_result.name, BOLD[0])) | BOLD[1], test_result.name, BOLD[0])) | ||||
| else: | else: | ||||
| print("%s%s%s failed, Duration: %s s\n" % | print("{}{}{} failed, Duration: {} s\n".format( | ||||
| (BOLD[1], test_result.name, BOLD[0], test_result.time)) | BOLD[1], test_result.name, BOLD[0], test_result.time)) | ||||
| print(BOLD[1] + 'stdout:' + BOLD[0]) | print(BOLD[1] + 'stdout:' + BOLD[0]) | ||||
| print(test_result.stdout) | print(test_result.stdout) | ||||
| print(BOLD[1] + 'stderr:' + BOLD[0]) | print(BOLD[1] + 'stderr:' + BOLD[0]) | ||||
| print(test_result.stderr) | print(test_result.stderr) | ||||
| return | return | ||||
| assert False, "we should not be here" | assert False, "we should not be here" | ||||
| ▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines | def execute_test_processes(num_jobs, test_list, tests_dir, tmpdir, flags): | ||||
| update_queue.put(None) | update_queue.put(None) | ||||
| for j in range(num_jobs): | for j in range(num_jobs): | ||||
| job_queue.put(None) | job_queue.put(None) | ||||
| return test_results | return test_results | ||||
| def print_results(test_results, max_len_name, runtime): | def print_results(test_results, max_len_name, runtime): | ||||
| results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ( | results = "\n" + BOLD[1] + "{} | {} | {}\n\n".format( | ||||
| "TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0] | "TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0] | ||||
| test_results.sort(key=lambda result: result.name.lower()) | test_results.sort(key=lambda result: result.name.lower()) | ||||
| all_passed = True | all_passed = True | ||||
| time_sum = 0 | time_sum = 0 | ||||
| for test_result in test_results: | for test_result in test_results: | ||||
| all_passed = all_passed and test_result.was_successful | all_passed = all_passed and test_result.was_successful | ||||
| time_sum += test_result.time | time_sum += test_result.time | ||||
| test_result.padding = max_len_name | test_result.padding = max_len_name | ||||
| results += str(test_result) | results += str(test_result) | ||||
| status = TICK + "Passed" if all_passed else CROSS + "Failed" | status = TICK + "Passed" if all_passed else CROSS + "Failed" | ||||
| results += BOLD[1] + "\n%s | %s | %s s (accumulated) \n" % ( | results += BOLD[1] + "\n{} | {} | {} s (accumulated) \n".format( | ||||
| "ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0] | "ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0] | ||||
| results += "Runtime: %s s\n" % (runtime) | results += "Runtime: {} s\n".format(runtime) | ||||
| print(results) | print(results) | ||||
| class TestResult(): | class TestResult(): | ||||
| """ | """ | ||||
| Simple data structure to store test result values and print them properly | Simple data structure to store test result values and print them properly | ||||
| """ | """ | ||||
| Show All 11 Lines | def __repr__(self): | ||||
| glyph = TICK | glyph = TICK | ||||
| elif self.status == "Failed": | elif self.status == "Failed": | ||||
| color = RED | color = RED | ||||
| glyph = CROSS | glyph = CROSS | ||||
| elif self.status == "Skipped": | elif self.status == "Skipped": | ||||
| color = GREY | color = GREY | ||||
| glyph = CIRCLE | glyph = CIRCLE | ||||
| return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0] | return color[1] + "{} | {}{} | {} s\n".format( | ||||
| self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0] | |||||
| @property | @property | ||||
| def was_successful(self): | def was_successful(self): | ||||
| return self.status != "Failed" | return self.status != "Failed" | ||||
| def get_all_scripts_from_disk(test_dir, non_scripts): | def get_all_scripts_from_disk(test_dir, non_scripts): | ||||
| """ | """ | ||||
| ▲ Show 20 Lines • Show All 193 Lines • Show Last 20 Lines | |||||