Differential D476 Diff 1194 qa/pull-tester/rpc-tests.py

Changeset View

Standalone View

qa/pull-tester/rpc-tests.py

Show All 27 Lines
import sys		import sys
import subprocess		import subprocess
import tempfile		import tempfile
import re		import re

sys.path.append("qa/pull-tester/")		sys.path.append("qa/pull-tester/")
from tests_config import *		from tests_config import *

BOLD = ("","")		BOLD = ("", "")
if os.name == 'posix':		if os.name == 'posix':
# primitive formatting on supported		# primitive formatting on supported
# terminal via ANSI escape sequences:		# terminal via ANSI escape sequences:
BOLD = ('\033[0m', '\033[1m')		BOLD = ('\033[0m', '\033[1m')

RPC_TESTS_DIR = SRCDIR + '/qa/rpc-tests/'		RPC_TESTS_DIR = SRCDIR + '/qa/rpc-tests/'

#If imported values are not defined then set to zero (or disabled)		# If imported values are not defined then set to zero (or disabled)
if 'ENABLE_WALLET' not in vars():		if 'ENABLE_WALLET' not in vars():
ENABLE_WALLET=0		ENABLE_WALLET = 0
if 'ENABLE_BITCOIND' not in vars():		if 'ENABLE_BITCOIND' not in vars():
ENABLE_BITCOIND=0		ENABLE_BITCOIND = 0
if 'ENABLE_UTILS' not in vars():		if 'ENABLE_UTILS' not in vars():
ENABLE_UTILS=0		ENABLE_UTILS = 0
if 'ENABLE_ZMQ' not in vars():		if 'ENABLE_ZMQ' not in vars():
ENABLE_ZMQ=0		ENABLE_ZMQ = 0

ENABLE_COVERAGE=0		ENABLE_COVERAGE = 0

#Create a set to store arguments and create the passon string		# Create a set to store arguments and create the passon string
opts = set()		opts = set()
passon_args = []		passon_args = []
PASSON_REGEX = re.compile("^--")		PASSON_REGEX = re.compile("^--")
PARALLEL_REGEX = re.compile('^-parallel=')		PARALLEL_REGEX = re.compile('^-parallel=')

print_help = False		print_help = False
run_parallel = 4		run_parallel = 4

for arg in sys.argv[1:]:		for arg in sys.argv[1:]:
if arg == "--help" or arg == "-h" or arg == "-?":		if arg == "--help" or arg == "-h" or arg == "-?":
print_help = True		print_help = True
break		break
if arg == '--coverage':		if arg == '--coverage':
ENABLE_COVERAGE = 1		ENABLE_COVERAGE = 1
elif PASSON_REGEX.match(arg):		elif PASSON_REGEX.match(arg):
passon_args.append(arg)		passon_args.append(arg)
elif PARALLEL_REGEX.match(arg):		elif PARALLEL_REGEX.match(arg):
run_parallel = int(arg.split(sep='=', maxsplit=1)[1])		run_parallel = int(arg.split(sep='=', maxsplit=1)[1])
else:		else:
opts.add(arg)		opts.add(arg)

#Set env vars		# Set env vars
if "BITCOIND" not in os.environ:		if "BITCOIND" not in os.environ:
os.environ["BITCOIND"] = BUILDDIR + '/src/bitcoind' + EXEEXT		os.environ["BITCOIND"] = BUILDDIR + '/src/bitcoind' + EXEEXT

if EXEEXT == ".exe" and "-win" not in opts:		if EXEEXT == ".exe" and "-win" not in opts:
# https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9		# https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
# https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964		# https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
print("Win tests currently disabled by default. Use -win option to enable")		print(
		"Win tests currently disabled by default. Use -win option to enable")
sys.exit(0)		sys.exit(0)

if not (ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1):		if not (ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1):
print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled")		print(
		"No rpc tests to run. Wallet, utils, and bitcoind must all be enabled")
sys.exit(0)		sys.exit(0)

# python3-zmq may not be installed. Handle this gracefully and with some helpful info		# python3-zmq may not be installed. Handle this gracefully and with some
		# helpful info
if ENABLE_ZMQ:		if ENABLE_ZMQ:
try:		try:
import zmq		import zmq
except ImportError:		except ImportError:
print("ERROR: \"import zmq\" failed. Set ENABLE_ZMQ=0 or "		print("ERROR: \"import zmq\" failed. Set ENABLE_ZMQ=0 or "
"to run zmq tests, see dependency info in /qa/README.md.")		"to run zmq tests, see dependency info in /qa/README.md.")
# ENABLE_ZMQ=0		# ENABLE_ZMQ=0
raise		raise
▲ Show 20 Lines • Show All 115 Lines • ▼ Show 20 Lines	def runtests():
flags.append("--cachedir=%s/qa/cache" % BUILDDIR)		flags.append("--cachedir=%s/qa/cache" % BUILDDIR)
if coverage:		if coverage:
flags.append(coverage.flag)		flags.append(coverage.flag)

if len(test_list) > 1 and run_parallel > 1:		if len(test_list) > 1 and run_parallel > 1:
# Populate cache		# Populate cache
subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + flags)		subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + flags)

#Run Tests		# Run Tests
max_len_name = len(max(test_list, key=len))		max_len_name = len(max(test_list, key=len))
time_sum = 0		time_sum = 0
time0 = time.time()		time0 = time.time()
job_queue = RPCTestHandler(run_parallel, test_list, flags)		job_queue = RPCTestHandler(run_parallel, test_list, flags)
results = BOLD[1] + "%s \| %s \| %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0]		results = BOLD[1] + "%s \| %s \| %s\n\n" % (
		"TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0]
all_passed = True		all_passed = True
for _ in range(len(test_list)):		for _ in range(len(test_list)):
(name, stdout, stderr, passed, duration) = job_queue.get_next()		(name, stdout, stderr, passed, duration) = job_queue.get_next()
all_passed = all_passed and passed		all_passed = all_passed and passed
time_sum += duration		time_sum += duration

print('\n' + BOLD[1] + name + BOLD[0] + ":")		print('\n' + BOLD[1] + name + BOLD[0] + ":")
print('' if passed else stdout + '\n', end='')		print('' if passed else stdout + '\n', end='')
print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='')		print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='')
results += "%s \| %s \| %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration)		results += "%s \| %s \| %s s\n" % (
print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration))		name.ljust(max_len_name), str(passed).ljust(6), duration)
results += BOLD[1] + "\n%s \| %s \| %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0]		print("Pass: %s%s%s, Duration: %s s\n" %
		(BOLD[1], passed, BOLD[0], duration))
		results += BOLD[1] + "\n%s \| %s \| %s s (accumulated)" % (
		"ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0]
print(results)		print(results)
print("\nRuntime: %s s" % (int(time.time() - time0)))		print("\nRuntime: %s s" % (int(time.time() - time0)))

if coverage:		if coverage:
coverage.report_rpc_coverage()		coverage.report_rpc_coverage()

print("Cleaning up coverage data")		print("Cleaning up coverage data")
coverage.cleanup()		coverage.cleanup()

sys.exit(not all_passed)		sys.exit(not all_passed)


class RPCTestHandler:		class RPCTestHandler:

"""		"""
Trigger the testscrips passed in via the list.		Trigger the testscrips passed in via the list.
"""		"""

def __init__(self, num_tests_parallel, test_list=None, flags=None):		def __init__(self, num_tests_parallel, test_list=None, flags=None):
assert(num_tests_parallel >= 1)		assert(num_tests_parallel >= 1)
self.num_jobs = num_tests_parallel		self.num_jobs = num_tests_parallel
self.test_list = test_list		self.test_list = test_list
self.flags = flags		self.flags = flags
self.num_running = 0		self.num_running = 0
# In case there is a graveyard of zombie bitcoinds, we can apply a		# In case there is a graveyard of zombie bitcoinds, we can apply a
# pseudorandom offset to hopefully jump over them.		# pseudorandom offset to hopefully jump over them.
# (625 is PORT_RANGE/MAX_NODES)		# (625 is PORT_RANGE/MAX_NODES)
self.portseed_offset = int(time.time() * 1000) % 625		self.portseed_offset = int(time.time() * 1000) % 625
self.jobs = []		self.jobs = []

def get_next(self):		def get_next(self):
while self.num_running < self.num_jobs and self.test_list:		while self.num_running < self.num_jobs and self.test_list:
# Add tests		# Add tests
self.num_running += 1		self.num_running += 1
t = self.test_list.pop(0)		t = self.test_list.pop(0)
port_seed = ["--portseed={}".format(len(self.test_list) + self.portseed_offset)]		port_seed = ["--portseed={}".format(
		len(self.test_list) + self.portseed_offset)]
log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)		log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)		log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
self.jobs.append((t,		self.jobs.append((t,
time.time(),		time.time(),
subprocess.Popen((RPC_TESTS_DIR + t).split() + self.flags + port_seed,		subprocess.Popen(
		(RPC_TESTS_DIR + t).split() +
		self.flags + port_seed,
universal_newlines=True,		universal_newlines=True,
stdout=log_stdout,		stdout=log_stdout,
stderr=log_stderr),		stderr=log_stderr),
log_stdout,		log_stdout,
log_stderr))		log_stderr))
if not self.jobs:		if not self.jobs:
raise IndexError('pop from empty list')		raise IndexError('pop from empty list')
while True:		while True:
# Return first proc that finishes		# Return first proc that finishes
time.sleep(.5)		time.sleep(.5)
for j in self.jobs:		for j in self.jobs:
(name, time0, proc, log_out, log_err) = j		(name, time0, proc, log_out, log_err) = j
if proc.poll() is not None:		if proc.poll() is not None:
log_out.seek(0), log_err.seek(0)		log_out.seek(0), log_err.seek(0)
[stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)]		[stdout, stderr] = [l.read().decode('utf-8')
		for l in (log_out, log_err)]
log_out.close(), log_err.close()		log_out.close(), log_err.close()
passed = stderr == "" and proc.returncode == 0		passed = stderr == "" and proc.returncode == 0
self.num_running -= 1		self.num_running -= 1
self.jobs.remove(j)		self.jobs.remove(j)
return name, stdout, stderr, passed, int(time.time() - time0)		return name, stdout, stderr, passed, int(time.time() - time0)
print('.', end='', flush=True)		print('.', end='', flush=True)


class RPCCoverage(object):		class RPCCoverage(object):

"""		"""
Coverage reporting utilities for pull-tester.		Coverage reporting utilities for pull-tester.

Coverage calculation works by having each test script subprocess write		Coverage calculation works by having each test script subprocess write
coverage files into a particular directory. These files contain the RPC		coverage files into a particular directory. These files contain the RPC
commands invoked during testing, as well as a complete listing of RPC		commands invoked during testing, as well as a complete listing of RPC
commands per `bitcoin-cli help` (`rpc_interface.txt`).		commands per `bitcoin-cli help` (`rpc_interface.txt`).

After all tests complete, the commands run are combined and diff'd against		After all tests complete, the commands run are combined and diff'd against
the complete list to calculate uncovered RPC commands.		the complete list to calculate uncovered RPC commands.

See also: qa/rpc-tests/test_framework/coverage.py		See also: qa/rpc-tests/test_framework/coverage.py

"""		"""

def __init__(self):		def __init__(self):
self.dir = tempfile.mkdtemp(prefix="coverage")		self.dir = tempfile.mkdtemp(prefix="coverage")
self.flag = '--coveragedir=%s' % self.dir		self.flag = '--coveragedir=%s' % self.dir

def report_rpc_coverage(self):		def report_rpc_coverage(self):
"""		"""
Print out RPC commands that were unexercised by tests.		Print out RPC commands that were unexercised by tests.

▲ Show 20 Lines • Show All 46 Lines • Show Last 20 Lines