diff --git a/test/functional/abc-p2p-compactblocks.py b/test/functional/abc-p2p-compactblocks.py --- a/test/functional/abc-p2p-compactblocks.py +++ b/test/functional/abc-p2p-compactblocks.py @@ -61,7 +61,7 @@ self.last_cmpctblock = None -@extended +@tags('slow') class FullBlockTest(ComparisonTestFramework): # Can either run this test as 1 node with expected answers, or two and compare them. diff --git a/test/functional/abc-p2p-fullblocktest.py b/test/functional/abc-p2p-fullblocktest.py --- a/test/functional/abc-p2p-fullblocktest.py +++ b/test/functional/abc-p2p-fullblocktest.py @@ -33,7 +33,7 @@ self.n = n # the output we're spending -@extended +@tags('slow') class FullBlockTest(ComparisonTestFramework): # Can either run this test as 1 node with expected answers, or two and compare them. diff --git a/test/functional/dbcrash.py b/test/functional/dbcrash.py --- a/test/functional/dbcrash.py +++ b/test/functional/dbcrash.py @@ -45,7 +45,7 @@ pass -@extended +@tags('slow') class ChainstateWriteCrashTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 diff --git a/test/functional/getblocktemplate_longpoll.py b/test/functional/getblocktemplate_longpoll.py --- a/test/functional/getblocktemplate_longpoll.py +++ b/test/functional/getblocktemplate_longpoll.py @@ -26,7 +26,7 @@ self.node.getblocktemplate({'longpollid': self.longpollid}) -@extended +@tags('slow') class GetBlockTemplateLPTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 diff --git a/test/functional/p2p-fullblocktest.py b/test/functional/p2p-fullblocktest.py --- a/test/functional/p2p-fullblocktest.py +++ b/test/functional/p2p-fullblocktest.py @@ -57,7 +57,7 @@ return r -@extended +@tags('slow') class FullBlockTest(ComparisonTestFramework): # Can either run this test as 1 node with expected answers, or two and compare them. # Change the "outcome" variable from each TestInstance object to only do the comparison. diff --git a/test/functional/p2p-timeouts.py b/test/functional/p2p-timeouts.py --- a/test/functional/p2p-timeouts.py +++ b/test/functional/p2p-timeouts.py @@ -35,7 +35,7 @@ pass -@extended +@tags('slow') class TimeoutsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True diff --git a/test/functional/pruning.py b/test/functional/pruning.py --- a/test/functional/pruning.py +++ b/test/functional/pruning.py @@ -31,7 +31,7 @@ return sum(os.path.getsize(blockdir + f) for f in os.listdir(blockdir) if os.path.isfile(blockdir + f)) / (1024. * 1024.) -@extended +@tags('slow') class PruneTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True diff --git a/test/functional/test_framework/decorators.py b/test/functional/test_framework/decorators.py --- a/test/functional/test_framework/decorators.py +++ b/test/functional/test_framework/decorators.py @@ -8,6 +8,8 @@ """ -def extended(cls): - cls.extended = True - return cls +def tags(*tags): + def decorater(cls): + cls.test_tags = tags + return cls + return decorater diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -103,9 +103,9 @@ self.test_num = test_num self.flags = flags - def run(self, portseed_offset, run_extended): + def run(self, portseed_offset, run_tags): t = self.test_case - portseed = self.test_num * 10 + portseed_offset + portseed = self.test_num * 1000 + portseed_offset portseed_arg = ["--portseed={}".format(portseed)] test_argv = t.split() tmpdir = [os.path.join("--tmpdir=%s", "%s_%s") % @@ -139,8 +139,13 @@ obj = getattr(test_module, prop) if inspect.isclass(obj) and issubclass(obj, BitcoinTestFramework) and obj is not BitcoinTestFramework and obj is not ComparisonTestFramework: test_instance = obj() - is_extended = hasattr(test_instance, 'extended') - if (is_extended and run_extended) or not is_extended: + + # Give every test the fast tag by default unless otherwise specified + tags = ["fast"] + if hasattr(test_instance, 'test_tags'): + tags = test_instance.test_tags + + if compare_tags(tags, run_tags): exit_code = obj().main( test_argv[1:] + self.flags + portseed_arg + tmpdir) else: @@ -214,9 +219,14 @@ default=tempfile.gettempdir(), help="Root directory for datadirs") parser.add_argument('--junitouput', '-ju', default=os.path.join(build_dir, 'junit_results.xml'), help="file that will store JUnit formated test results.") + parser.add_argument( + '--tags', nargs='+', default=[".*", "!slow"], help="List of tags to be run. Use '!' to negate") args, unknown_args = parser.parse_known_args() + if args.extended: + args.tags.extend('slow') + # Create a set to store arguments and create the passon string tests = set(arg for arg in unknown_args if arg[:2] != "--") passon_args = [arg for arg in unknown_args if arg[:2] == "--"] @@ -260,14 +270,10 @@ # extension. test_list = [t for t in all_scripts if (t in tests or re.sub(".py$", "", t) in tests)] - cutoff = sys.maxsize # do not cut off explicitly specified tests else: # No individual tests have been specified. # Run all tests that do not exceed test_list = all_scripts - cutoff = args.cutoff - if args.extended: - cutoff = sys.maxsize # Remove the test cases that the user has explicitly asked to exclude. if args.exclude: @@ -296,10 +302,10 @@ "cache"), ignore_errors=True) run_tests(test_list, build_dir, tests_dir, args.junitouput, - config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args, args.extended) + config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args, args.tags) -def run_tests(test_list, build_dir, tests_dir, junitouput, exeext, tmpdir, num_jobs, enable_coverage=False, args=[], run_extended=False): +def run_tests(test_list, build_dir, tests_dir, junitouput, exeext, tmpdir, num_jobs, enable_coverage=False, args=[], tags=[]): # Warn if bitcoind is already running (unix only) try: pidofOutput = subprocess.check_output(["pidof", "bitcoind"]) @@ -340,7 +346,7 @@ # Run Tests time0 = time.time() test_results = execute_test_processes( - num_jobs, test_list, tests_dir, tmpdir, flags, run_extended) + num_jobs, test_list, tests_dir, tmpdir, flags, tags) runtime = int(time.time() - time0) max_len_name = len(max(test_list, key=len)) @@ -426,7 +432,7 @@ printed_status = True -def handle_test_cases(job_queue, update_queue, run_extended): +def handle_test_cases(job_queue, update_queue, tags): """ job_runner represents a single thread that is part of a worker pool. It waits for a test, then executes that test. It also reports start @@ -444,12 +450,31 @@ # Signal that the test is starting to inform the poor waiting # programmer update_queue.put(test) - result = test.run(portseed_offset, run_extended) + result = test.run(portseed_offset, tags) update_queue.put(result) job_queue.task_done() -def execute_test_processes(num_jobs, test_list, tests_dir, tmpdir, flags, run_extended): +def compare_tags(test_tags, run_tags): + """ + Compare two sets of tags. Tags are evaludated in order, so if an + include is specified after an exclusion, then we will still run the test. + """ + run_test = False + for tag in run_tags: + run = True + if tag.startswith('!'): + run = False + tag = tag[1:] + + for test_tag in test_tags: + if re.match(tag, test_tag): + run_test = run + + return run_test + + +def execute_test_processes(num_jobs, test_list, tests_dir, tmpdir, flags, tags): ctx = mp.get_context('spawn') update_queue = ctx.JoinableQueue() job_queue = ctx.JoinableQueue() @@ -466,7 +491,7 @@ # Start some worker threads for j in range(num_jobs): t = ctx.Process(target=handle_test_cases, - args=(job_queue, update_queue, run_extended,)) + args=(job_queue, update_queue, tags,)) t.start() # Push all our test cases into the job queue. diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -14,7 +14,7 @@ import os -@extended +@tags('slow') class WalletHDTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True diff --git a/test/functional/walletbackup.py b/test/functional/walletbackup.py --- a/test/functional/walletbackup.py +++ b/test/functional/walletbackup.py @@ -38,7 +38,7 @@ from test_framework.util import * -@extended +@tags('slow') class WalletBackupTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4