diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -274,19 +274,8 @@ if exclude_test + ".py" in test_list: test_list.remove(exclude_test + ".py") - # Use and update timings from build_dir only if separate - # build directory is used. We do not want to pollute source directory. - build_timings = None - if (src_dir != build_dir): - build_timings = Timings(os.path.join(build_dir, 'timing.json')) - - # Always use timings from scr_dir if present - src_timings = Timings(os.path.join( - src_dir, "test", "functional", 'timing.json')) - # Add test parameters and remove long running tests if needed - test_list = get_tests_to_run( - test_list, TEST_PARAMS, cutoff, src_timings, build_timings) + test_list = get_tests_to_run(test_list, TEST_PARAMS) if not test_list: print("No valid test scripts specified. Check that your test is in one " @@ -306,10 +295,10 @@ "cache"), ignore_errors=True) run_tests(test_list, build_dir, tests_dir, args.junitouput, - config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args, build_timings, args.extended) + config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args, args.extended) -def run_tests(test_list, build_dir, tests_dir, junitouput, exeext, tmpdir, num_jobs, enable_coverage=False, args=[], build_timings=None, run_extended=None): +def run_tests(test_list, build_dir, tests_dir, junitouput, exeext, tmpdir, num_jobs, enable_coverage=False, args=[], run_extended=False): # Warn if bitcoind is already running (unix only) try: pidofOutput = subprocess.check_output(["pidof", "bitcoind"]) @@ -357,9 +346,6 @@ print_results(test_results, max_len_name, runtime) save_results_as_junit(test_results, junitouput, runtime) - if (build_timings is not None): - build_timings.save_timings(test_results) - if coverage: coverage.report_rpc_coverage() @@ -563,24 +549,11 @@ return list(python_files - set(non_scripts)) -def get_tests_to_run(test_list, test_params, cutoff, src_timings, build_timings=None): +def get_tests_to_run(test_list, test_params): """ - Returns only test that will not run longer that cutoff. - Long running tests are returned first to favor running tests in parallel - Timings from build directory override those from src directory + Returns all combinations of tests with testing flags """ - def get_test_time(test): - if build_timings is not None: - timing = next( - (x['time'] for x in build_timings.existing_timings if x['name'] == test), None) - if timing is not None: - return timing - - # try source directory. Return 0 if test is unknown to always run it - return next( - (x['time'] for x in src_timings.existing_timings if x['name'] == test), 0) - # Some tests must also be run with additional parameters. Add them to the list. tests_with_params = [] for test_name in test_list: @@ -591,9 +564,7 @@ tests_with_params.extend( [test_name + " " + " ".join(p) for p in params]) - result = [t for t in tests_with_params if get_test_time(t) <= cutoff] - result.sort(key=lambda x: (-get_test_time(x), x)) - return result + return tests_with_params class RPCCoverage(): @@ -701,52 +672,5 @@ file_name, "UTF-8", xml_declaration=True) -class Timings(): - """ - Takes care of loading, merging and saving tests execution times. - """ - - def __init__(self, timing_file): - self.timing_file = timing_file - self.existing_timings = self.load_timings() - - def load_timings(self): - if os.path.isfile(self.timing_file): - with open(self.timing_file) as f: - return json.load(f) - else: - return [] - - def get_merged_timings(self, new_timings): - """ - Return new list containing existing timings updated with new timings - Tests that do not exists are not removed - """ - - key = 'name' - merged = {} - for item in self.existing_timings + new_timings: - if item[key] in merged: - merged[item[key]].update(item) - else: - merged[item[key]] = item - - # Sort the result to preserve test ordering in file - merged = list(merged.values()) - merged.sort(key=lambda t, key=key: t[key]) - return merged - - def save_timings(self, test_results): - # we only save test that have passed - timings for failed test might be - # wrong (timeouts or early fails) - passed_results = [t for t in test_results if t.status == 'Passed'] - new_timings = list(map(lambda t: {'name': t.name, 'time': t.time}, - passed_results)) - merged_timings = self.get_merged_timings(new_timings) - - with open(self.timing_file, 'w') as f: - json.dump(merged_timings, f, indent=True) - - if __name__ == '__main__': main() diff --git a/test/functional/timing.json b/test/functional/timing.json deleted file mode 100644 --- a/test/functional/timing.json +++ /dev/null @@ -1,363 +0,0 @@ -[ - { - "name": "abandonconflict.py", - "time": 17 - }, - { - "name": "abc-checkdatasig-activation.py", - "time": 3 - }, - { - "name": "abc-cmdline.py", - "time": 8 - }, - { - "name": "abc-finalize-block.py", - "time": 4 - }, - { - "name": "abc-high_priority_transaction.py", - "time": 6 - }, - { - "name": "abc-magnetic-anomaly-activation.py", - "time": 6 - }, - { - "name": "abc-mempool-accept-txn.py", - "time": 3 - }, - { - "name": "abc-p2p-compactblocks.py", - "time": 273 - }, - { - "name": "abc-p2p-fullblocktest.py", - "time": 62 - }, - { - "name": "abc-parkedchain.py", - "time": 17 - }, - { - "name": "abc-replay-protection.py", - "time": 4 - }, - { - "name": "abc-rpc.py", - "time": 2 - }, - { - "name": "abc-transaction-ordering.py", - "time": 6 - }, - { - "name": "assumevalid.py", - "time": 17 - }, - { - "name": "bip65-cltv-p2p.py", - "time": 6 - }, - { - "name": "bip68-112-113-p2p.py", - "time": 15 - }, - { - "name": "bip68-sequence.py", - "time": 32 - }, - { - "name": "bipdersig-p2p.py", - "time": 6 - }, - { - "name": "bitcoin_cli.py", - "time": 4 - }, - { - "name": "blockchain.py", - "time": 19 - }, - { - "name": "dbcrash.py", - "time": 1292 - }, - { - "name": "decodescript.py", - "time": 2 - }, - { - "name": "deprecated_rpc.py", - "time": 2 - }, - { - "name": "disablewallet.py", - "time": 2 - }, - { - "name": "disconnect_ban.py", - "time": 6 - }, - { - "name": "example_test.py", - "time": 16 - }, - { - "name": "fundrawtransaction.py", - "time": 40 - }, - { - "name": "getblocktemplate_longpoll.py", - "time": 69 - }, - { - "name": "getchaintips.py", - "time": 4 - }, - { - "name": "httpbasics.py", - "time": 17 - }, - { - "name": "import-rescan.py", - "time": 18 - }, - { - "name": "importmulti.py", - "time": 11 - }, - { - "name": "importprunedfunds.py", - "time": 5 - }, - { - "name": "invalidateblock.py", - "time": 8 - }, - { - "name": "invalidblockrequest.py", - "time": 5 - }, - { - "name": "invalidtxrequest.py", - "time": 15 - }, - { - "name": "keypool-topup.py", - "time": 34 - }, - { - "name": "keypool.py", - "time": 7 - }, - { - "name": "listsinceblock.py", - "time": 24 - }, - { - "name": "listtransactions.py", - "time": 23 - }, - { - "name": "maxuploadtarget.py", - "time": 28 - }, - { - "name": "mempool_limit.py", - "time": 8 - }, - { - "name": "mempool_packages.py", - "time": 55 - }, - { - "name": "mempool_persist.py", - "time": 19 - }, - { - "name": "mempool_reorg.py", - "time": 6 - }, - { - "name": "mempool_resurrect_test.py", - "time": 2 - }, - { - "name": "mempool_spendcoinbase.py", - "time": 5 - }, - { - "name": "merkle_blocks.py", - "time": 32 - }, - { - "name": "minchainwork.py", - "time": 6 - }, - { - "name": "mining.py", - "time": 5 - }, - { - "name": "multi_rpc.py", - "time": 18 - }, - { - "name": "multiwallet.py", - "time": 20 - }, - { - "name": "net.py", - "time": 13 - }, - { - "name": "notifications.py", - "time": 6 - }, - { - "name": "nulldummy.py", - "time": 5 - }, - { - "name": "p2p-acceptblock.py", - "time": 14 - }, - { - "name": "p2p-compactblocks.py", - "time": 22 - }, - { - "name": "p2p-feefilter.py", - "time": 51 - }, - { - "name": "p2p-fullblocktest.py", - "time": 148 - }, - { - "name": "p2p-leaktests.py", - "time": 8 - }, - { - "name": "p2p-mempool.py", - "time": 3 - - }, - { - "name": "p2p-timeouts.py", - "time": 64 - }, - { - "name": "preciousblock.py", - "time": 3 - }, - { - "name": "prioritise_transaction.py", - "time": 34 - }, - { - "name": "proxy_test.py", - "time": 3 - }, - { - "name": "pruning.py", - "time": 1511 - }, - { - "name": "rawtransactions.py", - "time": 16 - }, - { - "name": "receivedby.py", - "time": 37 - }, - { - "name": "reindex.py", - "time": 18 - }, - { - "name": "resendwallettransactions.py", - "time": 18 - }, - { - "name": "rest.py", - "time": 20 - }, - { - "name": "rpcbind_test.py", - "time": 28 - }, - { - "name": "rpcnamedargs.py", - "time": 2 - }, - { - "name": "sendheaders.py", - "time": 16 - }, - { - "name": "signmessages.py", - "time": 3 - }, - { - "name": "signrawtransactions.py", - "time": 2 - }, - { - "name": "txn_clone.py", - "time": 8 - }, - { - "name": "txn_clone.py --mineblock", - "time": 6 - }, - { - "name": "txn_doublespend.py", - "time": 7 - }, - { - "name": "txn_doublespend.py --mineblock", - "time": 7 - }, - { - "name": "uptime.py", - "time": 2 - }, - { - "name": "wallet.py", - "time": 34 - }, - { - "name": "wallet_accounts.py", - "time": 5 - }, - { - "name": "wallet_dump.py", - "time": 6 - }, - { - "name": "wallet_encryption.py", - "time": 8 - }, - { - "name": "wallet_hd.py", - "time": 54 - }, - { - "name": "wallet_receivedby.py", - "time": 9 - }, - { - "name": "walletbackup.py", - "time": 116 - }, - { - "name": "zapwallettxes.py", - "time": 10 - }, - { - "name": "zmq_test.py", - "time": 9 - } -] \ No newline at end of file