diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -41,11 +41,6 @@ SKIPPED = 3 -TEST_EXIT_PASSED = 0 -TEST_EXIT_FAILED = 1 -TEST_EXIT_SKIPPED = 77 - - class BitcoinTestFramework(): """Base class for a bitcoin test script. @@ -68,7 +63,7 @@ self.nodes = [] self.mocktime = 0 - def main(self): + def main(self, args=None): """Main function. This should not be overridden by the subclass test scripts.""" parser = argparse.ArgumentParser(usage="%(prog)s [options]") @@ -95,7 +90,7 @@ parser.add_argument("--pdbonfailure", dest="pdbonfailure", default=False, action="store_true", help="Attach a python debugger if test fails") self.add_options(parser) - self.options = parser.parse_args() + self.options = parser.parse_args(args) self.set_test_params() assert hasattr( @@ -175,15 +170,20 @@ if success == TestStatus.PASSED: self.log.info("Tests successful") - sys.exit(TEST_EXIT_PASSED) elif success == TestStatus.SKIPPED: self.log.info("Test skipped") - sys.exit(TEST_EXIT_SKIPPED) else: self.log.error( "Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) - logging.shutdown() - sys.exit(TEST_EXIT_FAILED) + + # Manually close our handlers as we may be sharing logging with another test. + handlers = logging._handlers.copy() + for handler in handlers: + log.removeHandler(handler) + handler.flush() + handler.close() + + return success # Methods to override in subclass test scripts. def set_test_params(self): @@ -359,7 +359,7 @@ def _start_logging(self): # Add logger and logging handlers - self.log = logging.getLogger('TestFramework') + self.log = logging.getLogger(self.options.tmpdir) self.log.setLevel(logging.DEBUG) # Create file handler to log all messages fh = logging.FileHandler(self.options.tmpdir + '/test_framework.log') diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -33,10 +33,12 @@ import multiprocessing import importlib import importlib.util +import inspect import multiprocessing as mp from queue import Full, Empty +from io import StringIO -from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_framework import BitcoinTestFramework, ComparisonTestFramework, TestStatus from test_framework.cdefs import get_srcdir # Formatting. Default colors to empty strings. @@ -60,8 +62,6 @@ RED = ('\033[0m', '\033[0;31m') GREY = ('\033[0m', '\033[1;30m') -TEST_EXIT_PASSED = 0 -TEST_EXIT_SKIPPED = 77 NON_SCRIPTS = [ # These are python files that live in the functional tests directory, but are not test scripts. @@ -114,20 +114,43 @@ (self.tmpdir, re.sub(".py$", "", t), portseed)] name = t time0 = time.time() - process = subprocess.Popen([os.path.join(self.tests_dir, test_argv[0])] + test_argv[1:] + self.flags + portseed_arg + tmpdir, - universal_newlines=True, - stdout=log_stdout, - stderr=log_stderr) - - process.wait() - log_stdout.seek(0), log_stderr.seek(0) - [stdout, stderr] = [l.read().decode('utf-8') - for l in (log_stdout, log_stderr)] - log_stdout.close(), log_stderr.close() - - if process.returncode == TEST_EXIT_PASSED and stderr == "": + + # Dynamically import the test so we can introspect it + test_modulepath = os.path.abspath( + os.path.join(self.tests_dir, test_argv[0])) + test_spec = importlib.util.spec_from_file_location( + os.path.splitext(test_argv[0])[0], test_modulepath) + test_module = importlib.util.module_from_spec(test_spec) + test_spec.loader.exec_module(test_module) + + # Setup output capturing + original_stdout = sys.stdout + original_stderr = sys.stderr + test_stdout = StringIO() + test_stderr = StringIO() + sys.stdout = test_stdout + sys.stderr = test_stderr + + exit_code = None + try: + print("Entry") + for prop in dir(test_module): + obj = getattr(test_module, prop) + if inspect.isclass(obj) and issubclass(obj, BitcoinTestFramework) and obj is not BitcoinTestFramework and obj is not ComparisonTestFramework: + exit_code = obj().main( + test_argv[1:] + self.flags + portseed_arg + tmpdir) + except Exception as e: + print(e) + finally: + sys.stdout = original_stdout + sys.stderr = original_stderr + + [stdout, stderr] = [test_stdout.getvalue(), test_stderr.getvalue()] + test_stdout.close(), test_stderr.close() + + if exit_code == TestStatus.PASSED and stderr == "": status = "Passed" - elif process.returncode == TEST_EXIT_SKIPPED: + elif exit_code == TestStatus.SKIPPED: status = "Skipped" else: status = "Failed"