diff --git a/test/functional/README.md b/test/functional/README.md --- a/test/functional/README.md +++ b/test/functional/README.md @@ -43,6 +43,8 @@ - When calling RPCs with lots of arguments, consider using named keyword arguments instead of positional arguments to make the intent of the call clear to readers. +- before commiting a new test run `test_runner.py` with the `--updatetiming` + to record the test timing to `timing.json` #### RPC and P2P definitions diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -28,6 +28,7 @@ import re import logging import xml.etree.ElementTree as ET +import json # Formatting. Default colors to empty strings. BOLD, BLUE, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "") @@ -53,110 +54,6 @@ TEST_EXIT_PASSED = 0 TEST_EXIT_SKIPPED = 77 -BASE_SCRIPTS = [ - # Longest test should go first, to favor running tests in parallel - 'wallet-hd.py', - 'walletbackup.py', - # vv Tests less than 5m vv - 'p2p-fullblocktest.py', - 'fundrawtransaction.py', - 'p2p-compactblocks.py', - # vv Tests less than 2m vv - 'wallet.py', - 'wallet-accounts.py', - 'wallet-dump.py', - 'listtransactions.py', - # vv Tests less than 60s vv - 'sendheaders.py', - 'zapwallettxes.py', - 'importmulti.py', - 'mempool_limit.py', - 'merkle_blocks.py', - 'receivedby.py', - 'abandonconflict.py', - 'bip68-112-113-p2p.py', - 'rawtransactions.py', - 'reindex.py', - # vv Tests less than 30s vv - 'keypool-topup.py', - 'zmq_test.py', - 'bitcoin_cli.py', - 'mempool_resurrect_test.py', - 'txn_doublespend.py --mineblock', - 'txn_clone.py', - 'getchaintips.py', - 'rest.py', - 'mempool_spendcoinbase.py', - 'mempool_reorg.py', - 'mempool_persist.py', - 'multiwallet.py', - 'httpbasics.py', - 'multi_rpc.py', - 'proxy_test.py', - 'signrawtransactions.py', - 'disconnect_ban.py', - 'decodescript.py', - 'blockchain.py', - 'disablewallet.py', - 'net.py', - 'keypool.py', - 'p2p-mempool.py', - 'prioritise_transaction.py', - 'high_priority_transaction.py', - 'invalidblockrequest.py', - 'invalidtxrequest.py', - 'p2p-versionbits-warning.py', - 'preciousblock.py', - 'importprunedfunds.py', - 'signmessages.py', - 'nulldummy.py', - 'import-rescan.py', - 'mining.py', - 'rpcnamedargs.py', - 'listsinceblock.py', - 'p2p-leaktests.py', - 'abc-cmdline.py', - 'abc-p2p-fullblocktest.py', - 'abc-rpc.py', - 'mempool-accept-txn.py', - 'wallet-encryption.py', - 'bipdersig-p2p.py', - 'bip65-cltv-p2p.py', - 'uptime.py', - 'resendwallettransactions.py', -] - -EXTENDED_SCRIPTS = [ - # Longest test should go first, to favor running tests in parallel - 'pruning.py', - # vv Tests less than 20m vv - 'smartfees.py', - # vv Tests less than 5m vv - 'maxuploadtarget.py', - 'mempool_packages.py', - 'dbcrash.py', - # vv Tests less than 2m vv - 'bip68-sequence.py', - 'getblocktemplate_longpoll.py', - 'p2p-timeouts.py', - # vv Tests less than 60s vv - 'bip9-softforks.py', - 'p2p-feefilter.py', - 'rpcbind_test.py', - # vv Tests less than 30s vv - 'assumevalid.py', - 'example_test.py', - 'txn_doublespend.py', - 'txn_clone.py --mineblock', - 'forknotify.py', - 'invalidateblock.py', - 'p2p-acceptblock.py', -] - -# Place EXTENDED_SCRIPTS first since it has the 3 longest running tests -ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS - - NON_SCRIPTS = [ # These are python files that live in the functional tests directory, but are not test scripts. "combine_logs.py", @@ -164,6 +61,18 @@ "test_runner.py", ] +TEST_PARAMS = [ + # Some test requires parameters. When a test is listed here, those parameters will + # automatically be added when executing the test + "txn_doublespend.py --mineblock", + "txn_clone.py --mineblock" +] + +# Used to limit the number of tests, when list of tests is not provided on command line +# When --extended is specified, we run all tests, otherwise +# we only run a test if its time does not exceed EXTENDED_CUTOFF +EXTENDED_CUTOFF = 40 + def on_ci(): return os.getenv('TRAVIS') == 'true' or os.getenv('TEAMCITY_VERSION') != None @@ -172,11 +81,12 @@ def main(): # Read config generated by configure. config = configparser.ConfigParser() - configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" + configfile = os.path.join(os.path.abspath( + os.path.dirname(__file__)), "..", "config.ini") config.read_file(open(configfile)) src_dir = config["environment"]["SRCDIR"] - tests_dir = src_dir + '/test/functional/' + tests_dir = os.path.join(src_dir, 'test', 'functional') # Parse arguments and pass through unrecognised args parser = argparse.ArgumentParser(add_help=False, @@ -202,9 +112,12 @@ parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs') parser.add_argument('--tmpdirprefix', '-t', - default=tempfile.gettempdir(), help="Root directory for datadirs") + default=tempfile.gettempdir(), help="root directory for datadirs") parser.add_argument('--junitouput', '-ju', - default=tests_dir + 'junit_results.xml', help="file that will store JUnit formated test results ") + default=os.path.join(tests_dir, 'junit_results.xml'), help="file that will store JUnit formated test results.") + parser.add_argument('--updatetiming', '-ut', action='store_true', + help='Write timings of passed tests to timing.json. Use this flag when you create new or change existing tests.') + args, unknown_args = parser.parse_known_args() # Create a set to store arguments and create the passon string @@ -217,7 +130,7 @@ logging.basicConfig(format='%(message)s', level=logging_level) # Create base test directory - tmpdir = "%s/bitcoin_test_runner_%s" % ( + tmpdir = os.path.join("%s", "bitcoin_test_runner_%s") % ( args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) os.makedirs(tmpdir) @@ -241,22 +154,24 @@ "Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make") sys.exit(0) + all_scripts = get_all_scripts(tests_dir) + # Build list of tests if tests: # Individual tests have been specified. Run specified tests that exist - # in the ALL_SCRIPTS list. Accept the name with or without .py + # in the all_scripts list. Accept the name with or without .py # extension. - test_list = [t for t in ALL_SCRIPTS if + test_list = [t for t in all_scripts if (t in tests or re.sub(".py$", "", t) in tests)] + cutoff = sys.maxsize # do not cut off explicitly specified tests else: # No individual tests have been specified. - # Run all base tests, and optionally run extended tests. - test_list = BASE_SCRIPTS + # Run all tests that do not exceed + # EXTENDED_CUTOFF, unless --extended was specified + test_list = all_scripts + cutoff = EXTENDED_CUTOFF if args.extended: - test_list += EXTENDED_SCRIPTS - # TODO: BASE_SCRIPTS and EXTENDED_SCRIPTS are sorted by runtime - # (for parallel running efficiency). This combined list will is no - # longer sorted. + cutoff = sys.maxsize # Remove the test cases that the user has explicitly asked to exclude. if args.exclude: @@ -264,9 +179,16 @@ if exclude_test + ".py" in test_list: test_list.remove(exclude_test + ".py") + timings = Timings(tests_dir, all_scripts) + test_list = timings.get_tests_to_run(test_list, cutoff) + + if not tests and not timings.existing_timings: + print("%sWARNING!%s Timings file not found - will run all the tests. Run with --updatetiming flag to create the timing file." % ( + BOLD[1], BOLD[0])) + if not test_list: - print("No valid test scripts specified. Check that your test is in one " - "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests") + print("No valid test scripts specified. Check that your test " + "actually exists, or run test_runner.py with --extended to run all tests") sys.exit(0) if args.help: @@ -274,20 +196,18 @@ # and exit. parser.print_help() subprocess.check_call( - (tests_dir + test_list[0]).split() + ['-h']) + [os.path.join(tests_dir, test_list[0]), '-h']) sys.exit(0) - check_script_list(src_dir) - if not args.keepcache: shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) run_tests(test_list, src_dir, config["environment"]["BUILDDIR"], tests_dir, args.junitouput, - config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args) + config["environment"]["EXEEXT"], tmpdir, timings, args.jobs, args.coverage, args.updatetiming, passon_args) -def run_tests(test_list, src_dir, build_dir, tests_dir, junitouput, exeext, tmpdir, jobs=1, enable_coverage=False, args=[]): +def run_tests(test_list, src_dir, build_dir, tests_dir, junitouput, exeext, tmpdir, timings, jobs=1, enable_coverage=False, update_timing=False, args=[]): # Warn if bitcoind is already running (unix only) try: pidofOutput = subprocess.check_output(["pidof", "bitcoind"]) @@ -298,17 +218,19 @@ pass # Warn if there is a cache directory - cache_dir = "%s/test/cache" % build_dir + cache_dir = os.path.join(build_dir, "test", "cache") if os.path.isdir(cache_dir): print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % ( BOLD[1], BOLD[0], cache_dir)) # Set env vars if "BITCOIND" not in os.environ: - os.environ["BITCOIND"] = build_dir + '/src/bitcoind' + exeext - os.environ["BITCOINCLI"] = build_dir + '/src/bitcoin-cli' + exeext + os.environ["BITCOIND"] = os.path.join( + build_dir, 'src', 'bitcoind' + exeext) + os.environ["BITCOINCLI"] = os.path.join( + build_dir, 'src', 'bitcoin-cli' + exeext) - flags = ["--srcdir={}/src".format(build_dir)] + args + flags = [os.path.join("--srcdir={}".format(build_dir), "src")] + args flags.append("--cachedir=%s" % cache_dir) if enable_coverage: @@ -321,7 +243,7 @@ if len(test_list) > 1 and jobs > 1: # Populate cache subprocess.check_output( - [tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) + [os.path.join(tests_dir, 'create_cache.py')] + flags + [os.path.join("--tmpdir=%s", "cache") % tmpdir]) # Run Tests job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags) @@ -349,6 +271,8 @@ runtime = int(time.time() - time0) print_results(test_results, max_len_name, runtime) save_results_as_junit(test_results, junitouput, runtime) + if update_timing: + timings.save_timings(test_results) if coverage: coverage.report_rpc_coverage() @@ -416,15 +340,16 @@ portseed_arg = ["--portseed={}".format(portseed)] log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16) log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) - test_argv = t.split() - tmpdir = ["--tmpdir=%s/%s_%s" % - (self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)] + test_argv = get_test_params(t) + tmpdir = [os.path.join("--tmpdir=%s", "%s_%s") % + (self.tmpdir, re.sub(".py$", "", t), portseed)] self.jobs.append((t, time.time(), - subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir, + subprocess.Popen([os.path.join(self.tests_dir, t)] + test_argv + self.flags + portseed_arg + tmpdir, universal_newlines=True, stdout=log_stdout, - stderr=log_stderr), + stderr=log_stderr + ), log_stdout, log_stderr)) if not self.jobs: @@ -479,21 +404,19 @@ return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0] -def check_script_list(src_dir): - """Check scripts directory. +def get_test_params(test_name): + """ + Return list of parameters that should be used when running given test + """ + return next((t.split()[1:] for t in TEST_PARAMS if len(t.split()) > 1 and t.split()[0] == test_name), []) + - Check that there are no scripts in the functional tests directory which are - not being run by pull-tester.py.""" - script_dir = src_dir + '/test/functional/' - python_files = set([t for t in os.listdir(script_dir) if t[-3:] == ".py"]) - missed_tests = list( - python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS))) - if len(missed_tests) != 0: - print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % ( - BOLD[1], BOLD[0], str(missed_tests))) - if on_ci(): - # On CI this warning is an error to prevent merging incomplete commits into master - sys.exit(1) +def get_all_scripts(test_dir): + """ + Return all available test script from script directory (excluding NON_SCRIPTS) + """ + python_files = set([t for t in os.listdir(test_dir) if t[-3:] == ".py"]) + return list(python_files - set(NON_SCRIPTS)) class RPCCoverage(): @@ -601,5 +524,72 @@ file_name, "UTF-8", xml_declaration=True) +class Timings(): + """ + Takes care of loading, merging and saving tests execution times. + """ + + def __init__(self, tests_dir, all_scripts): + self.all_scripts = all_scripts + self.timing_file = os.path.join(tests_dir, 'timing.json') + self.existing_timings = self.load_timings() + + def load_timings(self): + if os.path.isfile(self.timing_file): + with open(self.timing_file) as f: + return json.load(f) + else: + return [] + + def get_tests_to_run(self, test_list, cutoff): + """ + Returns only test that will not run longer that cutoff. + Long running tests are returned first to favor running tests in parallel + """ + + def get_test_time(test_name): + return next(( + x['time'] for x in self.existing_timings if x['name'] == test_name), + 0 # 0 -> always run test if timing is unknown + ) + + result = [t for t in test_list if get_test_time(t) <= cutoff] + result.sort(key=lambda x: (-get_test_time(x), x)) + return result + + def get_merged_timings(self, new_timings): + """ + Update existing test list with new timings + """ + + key = 'name' + merged = {} + for item in self.existing_timings + new_timings: + if item[key] in merged: + merged[item[key]].update(item) + else: + merged[item[key]] = item + + # Remove tests that do not exists + merged = [v for (k, v) in merged.items() if k in self.all_scripts] + + # Sort the result to preserve test ordering in file + merged.sort(key=lambda t, key=key: t[key]) + return merged + + def save_timings(self, test_results): + + passed_results = [t for t in test_results if t.status == 'Passed'] + new_timings = list(map(lambda t: {'name': t.name, 'time': t.time}, + passed_results)) # only update passed tests + + merged_timings = self.get_merged_timings(new_timings) + + with open(self.timing_file, 'w') as f: + json.dump(merged_timings, f, indent=True) + + self.existing_timings = merged_timings + + if __name__ == '__main__': main() diff --git a/test/functional/timing.json b/test/functional/timing.json new file mode 100644 --- /dev/null +++ b/test/functional/timing.json @@ -0,0 +1,322 @@ +[ + { + "name": "abandonconflict.py", + "time": 11 + }, + { + "name": "abc-cmdline.py", + "time": 8 + }, + { + "name": "abc-p2p-fullblocktest.py", + "time": 32 + }, + { + "name": "abc-rpc.py", + "time": 3 + }, + { + "name": "assumevalid.py", + "time": 13 + }, + { + "name": "bip65-cltv-p2p.py", + "time": 6 + }, + { + "name": "bip68-112-113-p2p.py", + "time": 21 + }, + { + "name": "bip68-sequence.py", + "time": 21 + }, + { + "name": "bip9-softforks.py", + "time": 46 + }, + { + "name": "bipdersig-p2p.py", + "time": 5 + }, + { + "name": "bitcoin_cli.py", + "time": 3 + }, + { + "name": "blockchain.py", + "time": 8 + }, + { + "name": "dbcrash.py", + "time": 1110 + }, + { + "name": "decodescript.py", + "time": 3 + }, + { + "name": "disablewallet.py", + "time": 3 + }, + { + "name": "disconnect_ban.py", + "time": 7 + }, + { + "name": "example_test.py", + "time": 3 + }, + { + "name": "forknotify.py", + "time": 4 + }, + { + "name": "fundrawtransaction.py", + "time": 29 + }, + { + "name": "getblocktemplate_longpoll.py", + "time": 68 + }, + { + "name": "getchaintips.py", + "time": 4 + }, + { + "name": "high_priority_transaction.py", + "time": 8 + }, + { + "name": "httpbasics.py", + "time": 3 + }, + { + "name": "import-rescan.py", + "time": 6 + }, + { + "name": "importmulti.py", + "time": 6 + }, + { + "name": "importprunedfunds.py", + "time": 3 + }, + { + "name": "invalidateblock.py", + "time": 8 + }, + { + "name": "invalidblockrequest.py", + "time": 4 + }, + { + "name": "invalidtxrequest.py", + "time": 4 + }, + { + "name": "keypool-topup.py", + "time": 10 + }, + { + "name": "keypool.py", + "time": 7 + }, + { + "name": "listsinceblock.py", + "time": 4 + }, + { + "name": "listtransactions.py", + "time": 7 + }, + { + "name": "maxuploadtarget.py", + "time": 53 + }, + { + "name": "mempool-accept-txn.py", + "time": 4 + }, + { + "name": "mempool_limit.py", + "time": 6 + }, + { + "name": "mempool_packages.py", + "time": 9 + }, + { + "name": "mempool_persist.py", + "time": 18 + }, + { + "name": "mempool_reorg.py", + "time": 4 + }, + { + "name": "mempool_resurrect_test.py", + "time": 3 + }, + { + "name": "mempool_spendcoinbase.py", + "time": 3 + }, + { + "name": "merkle_blocks.py", + "time": 3 + }, + { + "name": "mining.py", + "time": 3 + }, + { + "name": "multi_rpc.py", + "time": 4 + }, + { + "name": "multiwallet.py", + "time": 6 + }, + { + "name": "net.py", + "time": 3 + }, + { + "name": "nulldummy.py", + "time": 3 + }, + { + "name": "p2p-acceptblock.py", + "time": 7 + }, + { + "name": "p2p-compactblocks.py", + "time": 19 + }, + { + "name": "p2p-feefilter.py", + "time": 16 + }, + { + "name": "p2p-fullblocktest.py", + "time": 204 + }, + { + "name": "p2p-leaktests.py", + "time": 8 + }, + { + "name": "p2p-mempool.py", + "time": 3 + }, + { + "name": "p2p-timeouts.py", + "time": 65 + }, + { + "name": "p2p-versionbits-warning.py", + "time": 9 + }, + { + "name": "preciousblock.py", + "time": 3 + }, + { + "name": "prioritise_transaction.py", + "time": 6 + }, + { + "name": "proxy_test.py", + "time": 4 + }, + { + "name": "pruning.py", + "time": 1500 + }, + { + "name": "rawtransactions.py", + "time": 14 + }, + { + "name": "receivedby.py", + "time": 14 + }, + { + "name": "reindex.py", + "time": 14 + }, + { + "name": "resendwallettransactions.py", + "time": 6 + }, + { + "name": "rest.py", + "time": 21 + }, + { + "name": "rpcbind_test.py", + "time": 28 + }, + { + "name": "rpcnamedargs.py", + "time": 5 + }, + { + "name": "sendheaders.py", + "time": 30 + }, + { + "name": "signmessages.py", + "time": 3 + }, + { + "name": "signrawtransactions.py", + "time": 3 + }, + { + "name": "smartfees.py", + "time": 305 + }, + { + "name": "txn_clone.py", + "time": 4 + }, + { + "name": "txn_doublespend.py", + "time": 5 + }, + { + "name": "uptime.py", + "time": 3 + }, + { + "name": "wallet-accounts.py", + "time": 4 + }, + { + "name": "wallet-dump.py", + "time": 7 + }, + { + "name": "wallet-encryption.py", + "time": 8 + }, + { + "name": "wallet-hd.py", + "time": 29 + }, + { + "name": "wallet.py", + "time": 40 + }, + { + "name": "walletbackup.py", + "time": 125 + }, + { + "name": "zapwallettxes.py", + "time": 17 + } +] \ No newline at end of file