Changeset View
Changeset View
Standalone View
Standalone View
test/functional/test_runner.py
Show First 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | if os.name == 'posix': | ||||
BOLD = ('\033[0m', '\033[1m') | BOLD = ('\033[0m', '\033[1m') | ||||
BLUE = ('\033[0m', '\033[0;34m') | BLUE = ('\033[0m', '\033[0;34m') | ||||
RED = ('\033[0m', '\033[0;31m') | RED = ('\033[0m', '\033[0;31m') | ||||
GREY = ('\033[0m', '\033[1;30m') | GREY = ('\033[0m', '\033[1;30m') | ||||
TEST_EXIT_PASSED = 0 | TEST_EXIT_PASSED = 0 | ||||
TEST_EXIT_SKIPPED = 77 | TEST_EXIT_SKIPPED = 77 | ||||
BASE_SCRIPTS = [ | |||||
# Longest test should go first, to favor running tests in parallel | |||||
'wallet-hd.py', | |||||
'walletbackup.py', | |||||
# vv Tests less than 5m vv | |||||
'p2p-fullblocktest.py', | |||||
'fundrawtransaction.py', | |||||
'p2p-compactblocks.py', | |||||
# vv Tests less than 2m vv | |||||
'wallet.py', | |||||
'wallet-accounts.py', | |||||
'wallet-dump.py', | |||||
'listtransactions.py', | |||||
# vv Tests less than 60s vv | |||||
'sendheaders.py', | |||||
'zapwallettxes.py', | |||||
'importmulti.py', | |||||
'mempool_limit.py', | |||||
'merkle_blocks.py', | |||||
'receivedby.py', | |||||
'abandonconflict.py', | |||||
'bip68-112-113-p2p.py', | |||||
'rawtransactions.py', | |||||
'reindex.py', | |||||
# vv Tests less than 30s vv | |||||
'keypool-topup.py', | |||||
'zmq_test.py', | |||||
'bitcoin_cli.py', | |||||
'mempool_resurrect_test.py', | |||||
'txn_doublespend.py --mineblock', | |||||
'txn_clone.py', | |||||
'getchaintips.py', | |||||
'rest.py', | |||||
'mempool_spendcoinbase.py', | |||||
'mempool_reorg.py', | |||||
'mempool_persist.py', | |||||
'multiwallet.py', | |||||
'httpbasics.py', | |||||
'multi_rpc.py', | |||||
'proxy_test.py', | |||||
'signrawtransactions.py', | |||||
'disconnect_ban.py', | |||||
'decodescript.py', | |||||
'blockchain.py', | |||||
'disablewallet.py', | |||||
'net.py', | |||||
'keypool.py', | |||||
'p2p-mempool.py', | |||||
'prioritise_transaction.py', | |||||
'invalidblockrequest.py', | |||||
'invalidtxrequest.py', | |||||
'p2p-versionbits-warning.py', | |||||
'preciousblock.py', | |||||
'importprunedfunds.py', | |||||
'signmessages.py', | |||||
'nulldummy.py', | |||||
'import-rescan.py', | |||||
'mining.py', | |||||
'rpcnamedargs.py', | |||||
'listsinceblock.py', | |||||
'p2p-leaktests.py', | |||||
'abc-cmdline.py', | |||||
'abc-high_priority_transaction.py', | |||||
'abc-mempool-accept-txn.py', | |||||
'abc-p2p-compactblocks.py', | |||||
'abc-p2p-fullblocktest.py', | |||||
'abc-rpc.py', | |||||
'wallet-encryption.py', | |||||
'bipdersig-p2p.py', | |||||
'bip65-cltv-p2p.py', | |||||
'uptime.py', | |||||
'resendwallettransactions.py', | |||||
'minchainwork.py', | |||||
] | |||||
EXTENDED_SCRIPTS = [ | |||||
# Longest test should go first, to favor running tests in parallel | |||||
'pruning.py', | |||||
# vv Tests less than 20m vv | |||||
'smartfees.py', | |||||
# vv Tests less than 5m vv | |||||
'maxuploadtarget.py', | |||||
'mempool_packages.py', | |||||
'dbcrash.py', | |||||
# vv Tests less than 2m vv | |||||
'bip68-sequence.py', | |||||
'getblocktemplate_longpoll.py', | |||||
'p2p-timeouts.py', | |||||
# vv Tests less than 60s vv | |||||
'bip9-softforks.py', | |||||
'p2p-feefilter.py', | |||||
'rpcbind_test.py', | |||||
# vv Tests less than 30s vv | |||||
'assumevalid.py', | |||||
'example_test.py', | |||||
'txn_doublespend.py', | |||||
'txn_clone.py --mineblock', | |||||
'forknotify.py', | |||||
'invalidateblock.py', | |||||
'p2p-acceptblock.py', | |||||
] | |||||
# Place EXTENDED_SCRIPTS first since it has the 3 longest running tests | |||||
ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS | |||||
NON_SCRIPTS = [ | NON_SCRIPTS = [ | ||||
# These are python files that live in the functional tests directory, but are not test scripts. | # These are python files that live in the functional tests directory, but are not test scripts. | ||||
"combine_logs.py", | "combine_logs.py", | ||||
"create_cache.py", | "create_cache.py", | ||||
"test_runner.py", | "test_runner.py", | ||||
] | ] | ||||
TEST_PARAMS = { | |||||
# Some test can be run with additional parameters. | |||||
# When a test is listed here, the it will be run without parameters | |||||
# as well as with additional parameters listed here. | |||||
# This: | |||||
# example "testName" : [["--param1", "--param2"] , ["--param3"]] | |||||
# will run the test 3 times: | |||||
# testName | |||||
# testName --param1 --param2 | |||||
# testname --param3 | |||||
"txn_doublespend.py": [["--mineblock"]], | |||||
"txn_clone.py": [["--mineblock"]] | |||||
} | |||||
# Used to limit the number of tests, when list of tests is not provided on command line | |||||
# When --extended is specified, we run all tests, otherwise | |||||
# we only run a test if its execution time in seconds does not exceed EXTENDED_CUTOFF | |||||
EXTENDED_CUTOFF = 40 | |||||
def on_ci(): | def on_ci(): | ||||
return os.getenv('TRAVIS') == 'true' or os.getenv('TEAMCITY_VERSION') != None | return os.getenv('TRAVIS') == 'true' or os.getenv('TEAMCITY_VERSION') != None | ||||
def main(): | def main(): | ||||
# Read config generated by configure. | # Read config generated by configure. | ||||
config = configparser.ConfigParser() | config = configparser.ConfigParser() | ||||
▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines | def main(): | ||||
if not (enable_wallet and enable_utils and enable_bitcoind): | if not (enable_wallet and enable_utils and enable_bitcoind): | ||||
print( | print( | ||||
"No functional tests to run. Wallet, utils, and bitcoind must all be enabled") | "No functional tests to run. Wallet, utils, and bitcoind must all be enabled") | ||||
print( | print( | ||||
"Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make") | "Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make") | ||||
sys.exit(0) | sys.exit(0) | ||||
# Build list of tests | # Build list of tests | ||||
all_scripts = get_all_scripts_from_disk(tests_dir, NON_SCRIPTS) | |||||
if tests: | if tests: | ||||
# Individual tests have been specified. Run specified tests that exist | # Individual tests have been specified. Run specified tests that exist | ||||
# in the ALL_SCRIPTS list. Accept the name with or without .py | # in the all_scripts list. Accept the name with or without .py | ||||
# extension. | # extension. | ||||
test_list = [t for t in ALL_SCRIPTS if | test_list = [t for t in all_scripts if | ||||
(t in tests or re.sub(".py$", "", t) in tests)] | (t in tests or re.sub(".py$", "", t) in tests)] | ||||
cutoff = sys.maxsize # do not cut off explicitly specified tests | |||||
else: | else: | ||||
# No individual tests have been specified. | # No individual tests have been specified. | ||||
# Run all base tests, and optionally run extended tests. | # Run all tests that do not exceed | ||||
test_list = BASE_SCRIPTS | # EXTENDED_CUTOFF, unless --extended was specified | ||||
test_list = all_scripts | |||||
cutoff = EXTENDED_CUTOFF | |||||
if args.extended: | if args.extended: | ||||
test_list += EXTENDED_SCRIPTS | cutoff = sys.maxsize | ||||
# TODO: BASE_SCRIPTS and EXTENDED_SCRIPTS are sorted by runtime | |||||
# (for parallel running efficiency). This combined list will is no | |||||
# longer sorted. | |||||
# Remove the test cases that the user has explicitly asked to exclude. | # Remove the test cases that the user has explicitly asked to exclude. | ||||
if args.exclude: | if args.exclude: | ||||
for exclude_test in args.exclude.split(','): | for exclude_test in args.exclude.split(','): | ||||
if exclude_test + ".py" in test_list: | if exclude_test + ".py" in test_list: | ||||
test_list.remove(exclude_test + ".py") | test_list.remove(exclude_test + ".py") | ||||
# Use and update timings from build_dir only if separate | # Use and update timings from build_dir only if separate | ||||
# build directory is used. We do not want to pollute source directory. | # build directory is used. We do not want to pollute source directory. | ||||
build_timings = None | build_timings = None | ||||
if (src_dir != build_dir): | if (src_dir != build_dir): | ||||
build_timings = Timings(build_dir) | build_timings = Timings(os.path.join(build_dir, 'timing.json')) | ||||
# Always use timings from scr_dir if present | |||||
src_timings = Timings(os.path.join( | |||||
src_dir, "test", "functional", 'timing.json')) | |||||
# Add test parameters and remove long running tests if needed | |||||
test_list = get_tests_to_run( | |||||
test_list, TEST_PARAMS, cutoff, src_timings, build_timings) | |||||
if not test_list: | if not test_list: | ||||
print("No valid test scripts specified. Check that your test is in one " | print("No valid test scripts specified. Check that your test is in one " | ||||
"of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests") | "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests") | ||||
sys.exit(0) | sys.exit(0) | ||||
if args.help: | if args.help: | ||||
# Print help for test_runner.py, then print help of the first script | # Print help for test_runner.py, then print help of the first script | ||||
# and exit. | # and exit. | ||||
parser.print_help() | parser.print_help() | ||||
subprocess.check_call( | subprocess.check_call( | ||||
[os.path.join(tests_dir, test_list[0]), '-h']) | [os.path.join(tests_dir, test_list[0]), '-h']) | ||||
sys.exit(0) | sys.exit(0) | ||||
check_script_list(src_dir) | |||||
if not args.keepcache: | if not args.keepcache: | ||||
shutil.rmtree(os.path.join(build_dir, "test", | shutil.rmtree(os.path.join(build_dir, "test", | ||||
"cache"), ignore_errors=True) | "cache"), ignore_errors=True) | ||||
run_tests(test_list, build_dir, tests_dir, args.junitouput, | run_tests(test_list, build_dir, tests_dir, args.junitouput, | ||||
config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args, build_timings) | config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args, build_timings) | ||||
▲ Show 20 Lines • Show All 189 Lines • ▼ Show 20 Lines | def __repr__(self): | ||||
glyph = CROSS | glyph = CROSS | ||||
elif self.status == "Skipped": | elif self.status == "Skipped": | ||||
color = GREY | color = GREY | ||||
glyph = CIRCLE | glyph = CIRCLE | ||||
return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0] | return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0] | ||||
def check_script_list(src_dir): | def get_all_scripts_from_disk(test_dir, non_scripts): | ||||
"""Check scripts directory. | """ | ||||
Return all available test script from script directory (excluding NON_SCRIPTS) | |||||
""" | |||||
python_files = set([t for t in os.listdir(test_dir) if t[-3:] == ".py"]) | |||||
return list(python_files - set(non_scripts)) | |||||
def get_tests_to_run(test_list, test_params, cutoff, src_timings, build_timings=None): | |||||
""" | |||||
Returns only test that will not run longer that cutoff. | |||||
Long running tests are returned first to favor running tests in parallel | |||||
Timings from build directory override those from src directory | |||||
""" | |||||
def get_test_time(test): | |||||
if build_timings is not None: | |||||
timing = next( | |||||
(x['time'] for x in build_timings.existing_timings if x['name'] == test), None) | |||||
if timing is not None: | |||||
return timing | |||||
# try source directory. Return 0 if test is unknown to always run it | |||||
return next( | |||||
(x['time'] for x in src_timings.existing_timings if x['name'] == test), 0) | |||||
# Some tests must also be run with additional parameters. Add them to the list. | |||||
tests_with_params = [] | |||||
for test_name in test_list: | |||||
# always execute a test without parameters | |||||
tests_with_params.append(test_name) | |||||
params = test_params.get(test_name) | |||||
if params is not None: | |||||
tests_with_params.extend( | |||||
[test_name + " " + " ".join(p) for p in params]) | |||||
Check that there are no scripts in the functional tests directory which are | result = [t for t in tests_with_params if get_test_time(t) <= cutoff] | ||||
not being run by pull-tester.py.""" | result.sort(key=lambda x: (-get_test_time(x), x)) | ||||
script_dir = os.path.join(src_dir, 'test', 'functional') | return result | ||||
python_files = set([t for t in os.listdir(script_dir) if t[-3:] == ".py"]) | |||||
missed_tests = list( | |||||
python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS))) | |||||
if len(missed_tests) != 0: | |||||
print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % ( | |||||
BOLD[1], BOLD[0], str(missed_tests))) | |||||
if on_ci(): | |||||
# On CI this warning is an error to prevent merging incomplete commits into master | |||||
sys.exit(1) | |||||
class RPCCoverage(): | class RPCCoverage(): | ||||
""" | """ | ||||
Coverage reporting utilities for test_runner. | Coverage reporting utilities for test_runner. | ||||
Coverage calculation works by having each test script subprocess write | Coverage calculation works by having each test script subprocess write | ||||
coverage files into a particular directory. These files contain the RPC | coverage files into a particular directory. These files contain the RPC | ||||
▲ Show 20 Lines • Show All 92 Lines • ▼ Show 20 Lines | for test_result in test_results: | ||||
ET.SubElement(e_test_case, "system-out").text = test_result.stdout | ET.SubElement(e_test_case, "system-out").text = test_result.stdout | ||||
ET.SubElement(e_test_case, "system-err").text = test_result.stderr | ET.SubElement(e_test_case, "system-err").text = test_result.stderr | ||||
ET.ElementTree(e_test_suite).write( | ET.ElementTree(e_test_suite).write( | ||||
file_name, "UTF-8", xml_declaration=True) | file_name, "UTF-8", xml_declaration=True) | ||||
class Timings(): | class Timings(): | ||||
""" | """ | ||||
Takes care of loading, merging and saving tests execution times. | Takes care of loading, merging and saving tests execution times. | ||||
""" | """ | ||||
def __init__(self, dir): | def __init__(self, timing_file): | ||||
self.dir = dir | self.timing_file = timing_file | ||||
self.timing_file = os.path.join(dir, 'timing.json') | |||||
self.existing_timings = self.load_timings() | self.existing_timings = self.load_timings() | ||||
def load_timings(self): | def load_timings(self): | ||||
if os.path.isfile(self.timing_file): | if os.path.isfile(self.timing_file): | ||||
with open(self.timing_file) as f: | with open(self.timing_file) as f: | ||||
return json.load(f) | return json.load(f) | ||||
else: | else: | ||||
return [] | return [] | ||||
Show All 34 Lines |