diff --git a/doc/functional-tests.md b/doc/functional-tests.md --- a/doc/functional-tests.md +++ b/doc/functional-tests.md @@ -224,6 +224,20 @@ `set_test_params()`, `add_options()` and `setup_xxxx()` methods at the top of the subclass, then locally-defined helper methods, then the `run_test()` method. +#### Naming guidelines + +- Name the test `_test.py`, where area can be one of the following: + - `feature` for tests for full features that aren't wallet/mining/mempool, eg `feature_rbf.py` + - `interface` for tests for other interfaces (REST, ZMQ, etc), eg `interface_rest.py` + - `mempool` for tests for mempool behaviour, eg `mempool_reorg.py` + - `mining` for tests for mining features, eg `mining_prioritisetransaction.py` + - `p2p` for tests that explicitly test the p2p interface, eg `p2p_disconnect_ban.py` + - `rpc` for tests for individual RPC methods or features, eg `rpc_listtransactions.py` + - `wallet` for tests for wallet features, eg `wallet_keypool.py` +- use an underscore to separate words + - exception: for tests for specific RPCs or command line options which don't include underscores, name the test after the exact RPC or argument name, eg `rpc_decodescript.py`, not `rpc_decode_script.py` +- Don't use the redundant word `test` in the name, eg `interface_zmq.py`, not `interface_zmq_test.py` + #### General test-writing advice - Set `self.num_nodes` to the minimum number of nodes necessary for the test. @@ -271,7 +285,7 @@ P2PInterface object and override the callback methods. - Can be used to write tests where specific P2P protocol behavior is tested. -Examples tests are `p2p_unrequested_blocks.py`, `p2p_compactblocks.py`. +Examples tests are `p2p-acceptblock.py`, `p2p-compactblocks.py`. ### test-framework modules diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -300,6 +300,8 @@ [sys.executable, os.path.join(tests_dir, test_list[0]), '-h']) sys.exit(0) + check_script_prefixes(all_scripts) + if not args.keepcache: shutil.rmtree(os.path.join(build_dir, "test", "cache"), ignore_errors=True) @@ -594,6 +596,37 @@ return list(python_files - set(non_scripts)) +def check_script_prefixes(all_scripts): + """Check that no more than `EXPECTED_VIOLATION_COUNT` of the + test scripts don't start with one of the allowed name prefixes.""" + EXPECTED_VIOLATION_COUNT = 29 + + # LEEWAY is provided as a transition measure, so that pull-requests + # that introduce new tests that don't conform with the naming + # convention don't immediately cause the tests to fail. + LEEWAY = 10 + + good_prefixes_re = re.compile( + "(example|feature|interface|mempool|mining|p2p|rpc|wallet)_") + bad_script_names = [ + script for script in all_scripts if good_prefixes_re.match(script) is None] + + if len(bad_script_names) < EXPECTED_VIOLATION_COUNT: + print( + "{}HURRAY!{} Number of functional tests violating naming convention reduced!".format( + BOLD[1], + BOLD[0])) + print("Consider reducing EXPECTED_VIOLATION_COUNT from {} to {}".format( + EXPECTED_VIOLATION_COUNT, len(bad_script_names))) + elif len(bad_script_names) > EXPECTED_VIOLATION_COUNT: + print( + "INFO: {} tests not meeting naming conventions (expected {}):".format(len(bad_script_names), EXPECTED_VIOLATION_COUNT)) + print(" {}".format("\n ".join(sorted(bad_script_names)))) + assert len(bad_script_names) <= EXPECTED_VIOLATION_COUNT + \ + LEEWAY, "Too many tests not following naming convention! ({} found, expected: <= {})".format( + len(bad_script_names), EXPECTED_VIOLATION_COUNT) + + def get_tests_to_run(test_list, test_params, cutoff, src_timings): """ Returns only test that will not run longer that cutoff.