diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -9,6 +9,8 @@ This module calls down into individual test cases via subprocess. It will forward all unrecognized arguments onto the individual test scripts. +RPC tests are disabled on Windows by default. Use --force to run them anyway. + For a description of arguments recognized by test scripts, see `qa/pull-tester/test_framework/test_framework.py:BitcoinTestFramework.main`. @@ -24,31 +26,6 @@ import tempfile import re -# Parse arguments and pass through unrecognised args -parser = argparse.ArgumentParser(add_help=False, - usage='%(prog)s [rpc-test.py options] [script options] [scripts]', - description=__doc__, - epilog=''' -Help text and arguments for individual test script:''', - formatter_class=argparse.RawTextHelpFormatter) -parser.add_argument('--coverage', action='store_true', - help='generate a basic coverage report for the RPC interface') -parser.add_argument('--extended', action='store_true', - help='run the extended test suite in addition to the basic tests') -parser.add_argument('--help', '-h', '-?', action='store_true', - help='print help text and exit') -parser.add_argument('--jobs', '-j', type=int, default=4, - help='how many test scripts to run in parallel. Default=4.') -parser.add_argument('--nozmq', action='store_true', - help='do not run the zmq tests') -parser.add_argument('--win', action='store_true', - help='signal that this is running in a Windows environment and that we should run the tests') -(args, unknown_args) = parser.parse_known_args() - -# Create a set to store arguments and create the passon string -tests = set(arg for arg in unknown_args if arg[:2] != "--") -passon_args = [arg for arg in unknown_args if arg[:2] == "--"] - BOLD = ("", "") RED = ("", "") GREEN = ("", "") @@ -59,48 +36,8 @@ RED = ("\033[0m", "\033[31m") GREEN = ("\033[0m", "\033[32m") -# Read config generated by configure. -config = configparser.ConfigParser() -config.read_file(open(os.path.dirname(__file__) + "/tests_config.ini")) - -ENABLE_WALLET = config["components"]["ENABLE_WALLET"] == "True" -ENABLE_UTILS = config["components"]["ENABLE_UTILS"] == "True" -ENABLE_BITCOIND = config["components"]["ENABLE_BITCOIND"] == "True" -ENABLE_ZMQ = config["components"]["ENABLE_ZMQ"] == "True" and not args.nozmq - -RPC_TESTS_DIR = config["environment"]["SRCDIR"] + '/qa/rpc-tests/' - -print_help = args.help -jobs = args.jobs - -# Set env vars -if "BITCOIND" not in os.environ: - os.environ["BITCOIND"] = config["environment"]["BUILDDIR"] + \ - '/src/bitcoind' + config["environment"]["EXEEXT"] - -if config["environment"]["EXEEXT"] == ".exe" and not args.win: - # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 - # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 - print( - "Win tests currently disabled by default. Use -win option to enable") - sys.exit(0) - -if not (ENABLE_WALLET and ENABLE_UTILS and ENABLE_BITCOIND): - print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled") - sys.exit(0) - -# python3-zmq may not be installed. Handle this gracefully and with some -# helpful info -if ENABLE_ZMQ: - try: - import zmq - except ImportError: - print("ERROR: \"import zmq\" failed. Use -nozmq to run without the ZMQ tests." - "To run zmq tests, see dependency info in /qa/README.md.") - raise - BASE_SCRIPTS = [ - # longest test should go first, to favor running tests in parallel + # Longest test should go first, to favor running tests in parallel 'wallet-hd.py', 'walletbackup.py', # vv Tests less than 5m vv @@ -159,9 +96,14 @@ 'abc-rpc.py', 'mempool-accept-txn.py', ] -ZMQ_SCRIPTS = ["zmq_test.py"] + +ZMQ_SCRIPTS = [ + # ZMQ test can only be run if bitcoin was built with zmq-enabled. + # call rpc_tests.py with -nozmq to explicitly exclude these tests. + "zmq_test.py"] EXTENDED_SCRIPTS = [ + # Longest test should go first, to favor running tests in parallel 'pruning.py', # vv Tests less than 20m vv 'smartfees.py', @@ -193,22 +135,81 @@ ALL_SCRIPTS = BASE_SCRIPTS + ZMQ_SCRIPTS + EXTENDED_SCRIPTS -def runtests(): +def main(): + # Parse arguments and pass through unrecognised args + parser = argparse.ArgumentParser(add_help=False, + usage='%(prog)s [rpc-test.py options] [script options] [scripts]', + description=__doc__, + epilog=''' + Help text and arguments for individual test script:''', + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument('--coverage', action='store_true', + help='generate a basic coverage report for the RPC interface') + parser.add_argument('--extended', action='store_true', + help='run the extended test suite in addition to the basic tests') + parser.add_argument('--force', '-f', action='store_true', + help='run tests even on platforms where they are disabled by default (e.g. windows).') + parser.add_argument('--help', '-h', '-?', + action='store_true', help='print help text and exit') + parser.add_argument('--jobs', '-j', type=int, default=4, + help='how many test scripts to run in parallel. Default=4.') + parser.add_argument('--nozmq', action='store_true', + help='do not run the zmq tests') + args, unknown_args = parser.parse_known_args() + + # Create a set to store arguments and create the passon string + tests = set(arg for arg in unknown_args if arg[:2] != "--") + passon_args = [arg for arg in unknown_args if arg[:2] == "--"] + + # Read config generated by configure. + config = configparser.ConfigParser() + config.read_file(open(os.path.dirname(__file__) + "/tests_config.ini")) + + enable_wallet = config["components"].getboolean("ENABLE_WALLET") + enable_utils = config["components"].getboolean("ENABLE_UTILS") + enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") + enable_zmq = config["components"].getboolean( + "ENABLE_ZMQ") and not args.nozmq + + if config["environment"]["EXEEXT"] == ".exe" and not args.force: + # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 + # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 + print( + "Tests currently disabled on Windows by default. Use --force option to enable") + sys.exit(0) + + if not (enable_wallet and enable_utils and enable_bitcoind): + print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled") + print("Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make") + sys.exit(0) + + # python3-zmq may not be installed. Handle this gracefully and with some helpful info + if enable_zmq: + try: + import zmq + except ImportError: + print("ERROR: \"import zmq\" failed. Use -nozmq to run without the ZMQ tests." + "To run zmq tests, see dependency info in /qa/README.md.") + raise + # Build list of tests - if len(tests) != 0: + if tests: # Individual tests have been specified. Run specified tests that exist # in the ALL_SCRIPTS list. Accept the name with or without .py extension. - test_list = [t for t in ALL_SCRIPTS if - (t in tests or re.sub(".py$", "", t) in tests)] - if len(test_list) == 0: + test_list = [t for t in ALL_SCRIPTS if ( + t in tests or re.sub(".py$", "", t) in tests)] + if not test_list: print("No valid test scripts specified. Check that your test is in one " "of the test lists in rpc-tests.py or run rpc-tests.py with no arguments to run all tests") + print("Scripts not found:") + print(test_list) sys.exit(0) + else: # No individual tests have been specified. Run base tests, and # optionally ZMQ tests and extended tests. test_list = BASE_SCRIPTS - if ENABLE_ZMQ: + if enable_zmq: test_list += ZMQ_SCRIPTS if args.extended: test_list += EXTENDED_SCRIPTS @@ -219,32 +220,46 @@ if args.help: # Print help for rpc-tests.py, then print help of the first script and exit. parser.print_help() - subprocess.check_call((RPC_TESTS_DIR + test_list[0]).split() + ['-h']) + subprocess.check_call( + (config["environment"]["SRCDIR"] + '/qa/rpc-tests/' + test_list[0]).split() + ['-h']) sys.exit(0) - coverage = None + run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], + config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args) + + +def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]): + # Set env vars + if "BITCOIND" not in os.environ: + os.environ["BITCOIND"] = build_dir + '/src/bitcoind' + exeext - if args.coverage: + tests_dir = src_dir + '/qa/rpc-tests/' + + flags = ["--srcdir=" + src_dir] + args + flags.append("--cachedir={}/qa/cache".format(build_dir)) + + if enable_coverage: coverage = RPCCoverage() - print("Initializing coverage directory at %s\n" % coverage.dir) - flags = ["--srcdir=%s/src" % - config["environment"]["BUILDDIR"]] + passon_args - flags.append("--cachedir=%s/qa/cache" % config["environment"]["BUILDDIR"]) - if coverage: flags.append(coverage.flag) + print("Initializing coverage directory at {dir}\n".format( + dir=coverage.dir)) + else: + coverage = None if len(test_list) > 1 and jobs > 1: # Populate cache - subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + flags) + subprocess.check_output([tests_dir + 'create_cache.py'] + flags) # Run Tests - max_len_name = len(max(test_list, key=len)) + all_passed = True time_sum = 0 time0 = time.time() - job_queue = RPCTestHandler(jobs, test_list, flags) + + job_queue = RPCTestHandler(jobs, tests_dir, test_list, flags) + + max_len_name = len(max(test_list, key=len)) results = BOLD[1] + "%s | %s | %s\n\n" % ( "TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0] - all_passed = True for _ in range(len(test_list)): (name, stdout, stderr, passed, duration) = job_queue.get_next() all_passed = all_passed and passed @@ -253,18 +268,19 @@ print('\n' + BOLD[1] + name + BOLD[0] + ":") print('' if passed else stdout + '\n', end='') print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='') - result = "%s | %s | %s s\n" % ( - name.ljust(max_len_name), str(passed).ljust(6), duration) + print("Pass: {bold}{result}{unbold}, Duration: {duration}s\n".format( + bold=BOLD[1], result=passed, unbold=BOLD[0], duration=duration)) + result = "{name} | {passed} | {duration}s\n".format(name=name.ljust( + max_len_name), passed=str(passed).ljust(6), duration=duration) if passed: results += GREEN[1] + result + GREEN[0] else: results += RED[1] + result + RED[0] - print("Pass: %s%s%s, Duration: %s s\n" % - (BOLD[1], passed, BOLD[0], duration)) - results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ( - "ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0] + + results += BOLD[1] + "\n{name} | {passed} | {duration}s (accumulated)".format( + name="ALL".ljust(max_len_name), passed=str(all_passed).ljust(6), duration=time_sum) + BOLD[0] print(results) - print("\nRuntime: %s s" % (int(time.time() - time0))) + print("\nRuntime: {} s".format(int(time.time() - time0))) if coverage: coverage.report_rpc_coverage() @@ -281,9 +297,10 @@ Trigger the testscrips passed in via the list. """ - def __init__(self, num_tests_parallel, test_list=None, flags=None): + def __init__(self, num_tests_parallel, tests_dir, test_list=None, flags=None): assert(num_tests_parallel >= 1) self.num_jobs = num_tests_parallel + self.tests_dir = tests_dir self.test_list = test_list self.flags = flags self.num_running = 0 @@ -304,12 +321,10 @@ log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) self.jobs.append((t, time.time(), - subprocess.Popen( - (RPC_TESTS_DIR + t).split() + - self.flags + port_seed, - universal_newlines=True, - stdout=log_stdout, - stderr=log_stderr), + subprocess.Popen((self.tests_dir + t).split() + self.flags + port_seed, + universal_newlines=True, + stdout=log_stdout, + stderr=log_stderr), log_stdout, log_stderr)) if not self.jobs: @@ -351,7 +366,7 @@ def __init__(self): self.dir = tempfile.mkdtemp(prefix="coverage") - self.flag = '--coveragedir=%s' % self.dir + self.flag = '--coveragedir={}'.format(self.dir) def report_rpc_coverage(self): """ @@ -362,7 +377,7 @@ if uncovered: print("Uncovered RPC commands:") - print("".join((" - %s\n" % i) for i in sorted(uncovered))) + print("".join((" - {}\n".format(i)) for i in sorted(uncovered))) else: print("All RPC commands covered.") @@ -375,10 +390,10 @@ """ # This is shared from `qa/rpc-tests/test-framework/coverage.py` - REFERENCE_FILENAME = 'rpc_interface.txt' - COVERAGE_FILE_PREFIX = 'coverage.' + reference_filename = 'rpc_interface.txt' + coverage_file_prefix = 'coverage.' - coverage_ref_filename = os.path.join(self.dir, REFERENCE_FILENAME) + coverage_ref_filename = os.path.join(self.dir, reference_filename) coverage_filenames = set() all_cmds = set() covered_cmds = set() @@ -391,7 +406,7 @@ for root, dirs, files in os.walk(self.dir): for filename in files: - if filename.startswith(COVERAGE_FILE_PREFIX): + if filename.startswith(coverage_file_prefix): coverage_filenames.add(os.path.join(root, filename)) for filename in coverage_filenames: @@ -402,4 +417,4 @@ if __name__ == '__main__': - runtests() + main() diff --git a/qa/pull-tester/tests_config.ini.in b/qa/pull-tester/tests_config.ini.in --- a/qa/pull-tester/tests_config.ini.in +++ b/qa/pull-tester/tests_config.ini.in @@ -5,13 +5,6 @@ # These environment variables are set by the build process and read by # rpc-tests.py -[DEFAULT] -# Provides default values for whether different components are enabled -ENABLE_WALLET=False -ENABLE_UTILS=False -ENABLE_BITCOIND=False -ENABLE_ZMQ=False - [environment] SRCDIR=@abs_top_srcdir@ BUILDDIR=@abs_top_builddir@ @@ -19,7 +12,7 @@ [components] # Which components are enabled. These are commented out by `configure` if they were disabled when running config. -@ENABLE_WALLET_TRUE@ENABLE_WALLET=True -@BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=True -@BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=True -@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=True +@ENABLE_WALLET_TRUE@ENABLE_WALLET=true +@BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=true +@BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true +@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true