Changeset View
Changeset View
Standalone View
Standalone View
test/functional/test_framework/util.py
#!/usr/bin/env python3 | #!/usr/bin/env python3 | ||||
# Copyright (c) 2014-2016 The Bitcoin Core developers | # Copyright (c) 2014-2016 The Bitcoin Core developers | ||||
# Distributed under the MIT software license, see the accompanying | # Distributed under the MIT software license, see the accompanying | ||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php. | # file COPYING or http://www.opensource.org/licenses/mit-license.php. | ||||
# | |||||
# Helpful routines for regression testing | |||||
# | |||||
from base64 import b64encode | from base64 import b64encode | ||||
from binascii import hexlify, unhexlify | from binascii import hexlify, unhexlify | ||||
from decimal import Decimal, ROUND_DOWN | from decimal import Decimal, ROUND_DOWN | ||||
import json | import json | ||||
import http.client | import logging | ||||
import os | import os | ||||
import sys | import sys | ||||
import random | import random | ||||
import shutil | |||||
import subprocess | |||||
import tempfile | |||||
import time | |||||
import re | import re | ||||
import errno | import time | ||||
import logging | |||||
from . import coverage | from . import coverage | ||||
from .authproxy import AuthServiceProxy, JSONRPCException | from .authproxy import AuthServiceProxy, JSONRPCException | ||||
DEFAULT_BITCOIND = 'bitcoind' | |||||
COVERAGE_DIR = None | |||||
logger = logging.getLogger("TestFramework.utils") | logger = logging.getLogger("TestFramework.utils") | ||||
# The maximum number of nodes a single test can spawn | # Assert functions | ||||
MAX_NODES = 8 | ################## | ||||
# Don't assign rpc or p2p ports lower than this | |||||
PORT_MIN = 11000 | |||||
# The number of ports to "reserve" for p2p and rpc, each | |||||
PORT_RANGE = 5000 | |||||
BITCOIND_PROC_WAIT_TIMEOUT = 60 | |||||
class PortSeed: | |||||
# Must be initialized with a unique integer for each process | |||||
n = None | |||||
def assert_fee_amount(fee, tx_size, fee_per_kB): | |||||
"""Assert the fee was in range""" | |||||
target_fee = tx_size * fee_per_kB / 1000 | |||||
if fee < target_fee: | |||||
raise AssertionError( | |||||
"Fee of %s BTC too low! (Should be %s BTC)" % (str(fee), str(target_fee))) | |||||
# allow the wallet's estimation to be at most 2 bytes off | |||||
if fee > (tx_size + 2) * fee_per_kB / 1000: | |||||
raise AssertionError( | |||||
"Fee of %s BTC too high! (Should be %s BTC)" % (str(fee), str(target_fee))) | |||||
# Set Mocktime default to OFF. | |||||
# MOCKTIME is only needed for scripts that use the | |||||
# cached version of the blockchain. If the cached | |||||
# version of the blockchain is used without MOCKTIME | |||||
# then the mempools will not sync due to IBD. | |||||
MOCKTIME = 0 | |||||
def assert_equal(thing1, thing2, *args): | |||||
if thing1 != thing2 or any(thing1 != arg for arg in args): | |||||
raise AssertionError("not(%s)" % " == ".join(str(arg) | |||||
for arg in (thing1, thing2) + args)) | |||||
def enable_mocktime(): | |||||
# For backwared compatibility of the python scripts | |||||
# with previous versions of the cache, set MOCKTIME | |||||
# to Jan 1, 2014 + (201 * 10 * 60) | |||||
global MOCKTIME | |||||
MOCKTIME = 1388534400 + (201 * 10 * 60) | |||||
def assert_greater_than(thing1, thing2): | |||||
if thing1 <= thing2: | |||||
raise AssertionError("%s <= %s" % (str(thing1), str(thing2))) | |||||
def disable_mocktime(): | |||||
global MOCKTIME | |||||
MOCKTIME = 0 | |||||
def assert_greater_than_or_equal(thing1, thing2): | |||||
if thing1 < thing2: | |||||
raise AssertionError("%s < %s" % (str(thing1), str(thing2))) | |||||
def get_mocktime(): | |||||
return MOCKTIME | |||||
def assert_raises(exc, fun, *args, **kwds): | |||||
assert_raises_message(exc, None, fun, *args, **kwds) | |||||
def enable_coverage(dirname): | |||||
"""Maintain a log of which RPC calls are made during testing.""" | |||||
global COVERAGE_DIR | |||||
COVERAGE_DIR = dirname | |||||
def assert_raises_message(exc, message, fun, *args, **kwds): | |||||
try: | |||||
fun(*args, **kwds) | |||||
except exc as e: | |||||
if message is not None and message not in e.error['message']: | |||||
raise AssertionError( | |||||
"Expected substring not found:" + e.error['message']) | |||||
except Exception as e: | |||||
raise AssertionError( | |||||
"Unexpected exception raised: " + type(e).__name__) | |||||
else: | |||||
raise AssertionError("No exception raised") | |||||
def get_rpc_proxy(url, node_number, timeout=None): | |||||
""" | |||||
Args: | |||||
url (str): URL of the RPC server to call | |||||
node_number (int): the node number (or id) that this calls to | |||||
Kwargs: | def assert_raises_jsonrpc(code, message, fun, *args, **kwds): | ||||
timeout (int): HTTP timeout in seconds | """Run an RPC and verify that a specific JSONRPC exception code and message is raised. | ||||
Returns: | Calls function `fun` with arguments `args` and `kwds`. Catches a JSONRPCException | ||||
AuthServiceProxy. convenience object for making RPC calls. | and verifies that the error code and message are as expected. Throws AssertionError if | ||||
no JSONRPCException was returned or if the error code/message are not as expected. | |||||
Args: | |||||
code (int), optional: the error code returned by the RPC call (defined | |||||
in src/rpc/protocol.h). Set to None if checking the error code is not required. | |||||
message (string), optional: [a substring of] the error string returned by the | |||||
RPC call. Set to None if checking the error string is not required | |||||
fun (function): the function to call. This should be the name of an RPC. | |||||
args*: positional arguments for the function. | |||||
kwds**: named arguments for the function. | |||||
""" | """ | ||||
proxy_kwargs = {} | try: | ||||
if timeout is not None: | fun(*args, **kwds) | ||||
proxy_kwargs['timeout'] = timeout | except JSONRPCException as e: | ||||
# JSONRPCException was thrown as expected. Check the code and message values are correct. | |||||
if (code is not None) and (code != e.error["code"]): | |||||
raise AssertionError( | |||||
"Unexpected JSONRPC error code %i" % e.error["code"]) | |||||
if (message is not None) and (message not in e.error['message']): | |||||
raise AssertionError( | |||||
"Expected substring not found:" + e.error['message']) | |||||
except Exception as e: | |||||
raise AssertionError( | |||||
"Unexpected exception raised: " + type(e).__name__) | |||||
else: | |||||
raise AssertionError("No exception raised") | |||||
proxy = AuthServiceProxy(url, **proxy_kwargs) | |||||
proxy.url = url # store URL on proxy for info | |||||
coverage_logfile = coverage.get_filename( | def assert_is_hex_string(string): | ||||
COVERAGE_DIR, node_number) if COVERAGE_DIR else None | try: | ||||
int(string, 16) | |||||
except Exception as e: | |||||
raise AssertionError( | |||||
"Couldn't interpret %r as hexadecimal; raised: %s" % (string, e)) | |||||
return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile) | |||||
def assert_is_hash_string(string, length=64): | |||||
if not isinstance(string, str): | |||||
raise AssertionError("Expected a string, got type %r" % type(string)) | |||||
elif length and len(string) != length: | |||||
raise AssertionError( | |||||
"String of length %d expected; got %d" % (length, len(string))) | |||||
elif not re.match('[abcdef0-9]+$', string): | |||||
raise AssertionError( | |||||
"String %r contains invalid characters for a hash." % string) | |||||
def p2p_port(n): | |||||
assert(n <= MAX_NODES) | |||||
return PORT_MIN + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) | |||||
def assert_array_result(object_array, to_match, expected, should_not_find=False): | |||||
""" | |||||
Pass in array of JSON objects, a dictionary with key/value pairs | |||||
to match against, and another dictionary with expected key/value | |||||
pairs. | |||||
If the should_not_find flag is true, to_match should not be found | |||||
in object_array | |||||
""" | |||||
if should_not_find: | |||||
assert_equal(expected, {}) | |||||
num_matched = 0 | |||||
for item in object_array: | |||||
all_match = True | |||||
for key, value in to_match.items(): | |||||
if item[key] != value: | |||||
all_match = False | |||||
if not all_match: | |||||
continue | |||||
elif should_not_find: | |||||
num_matched = num_matched + 1 | |||||
for key, value in expected.items(): | |||||
if item[key] != value: | |||||
raise AssertionError("%s : expected %s=%s" % | |||||
(str(item), str(key), str(value))) | |||||
num_matched = num_matched + 1 | |||||
if num_matched == 0 and not should_not_find: | |||||
raise AssertionError("No objects matched %s" % (str(to_match))) | |||||
if num_matched > 0 and should_not_find: | |||||
raise AssertionError("Objects were found %s" % (str(to_match))) | |||||
def rpc_port(n): | # Utility functions | ||||
return PORT_MIN + PORT_RANGE + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) | ################### | ||||
def check_json_precision(): | def check_json_precision(): | ||||
"""Make sure json library being used does not lose precision converting BTC values""" | """Make sure json library being used does not lose precision converting BTC values""" | ||||
n = Decimal("20000000.00000003") | n = Decimal("20000000.00000003") | ||||
satoshis = int(json.loads(json.dumps(float(n))) * 1.0e8) | satoshis = int(json.loads(json.dumps(float(n))) * 1.0e8) | ||||
if satoshis != 2000000000000003: | if satoshis != 2000000000000003: | ||||
raise RuntimeError("JSON encode/decode loses precision") | raise RuntimeError("JSON encode/decode loses precision") | ||||
Show All 10 Lines | |||||
def hex_str_to_bytes(hex_str): | def hex_str_to_bytes(hex_str): | ||||
return unhexlify(hex_str.encode('ascii')) | return unhexlify(hex_str.encode('ascii')) | ||||
def str_to_b64str(string): | def str_to_b64str(string): | ||||
return b64encode(string.encode('utf-8')).decode('ascii') | return b64encode(string.encode('utf-8')).decode('ascii') | ||||
def sync_blocks(rpc_connections, *, wait=1, timeout=60): | def satoshi_round(amount): | ||||
""" | return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) | ||||
Wait until everybody has the same tip. | |||||
sync_blocks needs to be called with an rpc_connections set that has least | # RPC/P2P connection constants and functions | ||||
one node already synced to the latest, stable tip, otherwise there's a | ############################################ | ||||
chance it might return before all nodes are stably synced. | |||||
""" | |||||
# Use getblockcount() instead of waitforblockheight() to determine the | |||||
# initial max height because the two RPCs look at different internal global | |||||
# variables (chainActive vs latestBlock) and the former gets updated | |||||
# earlier. | |||||
maxheight = max(x.getblockcount() for x in rpc_connections) | |||||
start_time = cur_time = time.time() | |||||
while cur_time <= start_time + timeout: | |||||
tips = [r.waitforblockheight(maxheight, int(wait * 1000)) | |||||
for r in rpc_connections] | |||||
if all(t["height"] == maxheight for t in tips): | |||||
if all(t["hash"] == tips[0]["hash"] for t in tips): | |||||
return | |||||
raise AssertionError("Block sync failed, mismatched block hashes:{}".format( | |||||
"".join("\n {!r}".format(tip) for tip in tips))) | |||||
cur_time = time.time() | |||||
raise AssertionError("Block sync to height {} timed out:{}".format( | |||||
maxheight, "".join("\n {!r}".format(tip) for tip in tips))) | |||||
def sync_chain(rpc_connections, *, wait=1, timeout=60): | # The maximum number of nodes a single test can spawn | ||||
""" | MAX_NODES = 8 | ||||
Wait until everybody has the same best block | # Don't assign rpc or p2p ports lower than this | ||||
""" | PORT_MIN = 11000 | ||||
while timeout > 0: | # The number of ports to "reserve" for p2p and rpc, each | ||||
best_hash = [x.getbestblockhash() for x in rpc_connections] | PORT_RANGE = 5000 | ||||
if best_hash == [best_hash[0]] * len(best_hash): | |||||
return | |||||
time.sleep(wait) | |||||
timeout -= wait | |||||
raise AssertionError("Chain sync failed: Best block hashes don't match") | |||||
def sync_mempools(rpc_connections, *, wait=1, timeout=60): | class PortSeed: | ||||
# Must be initialized with a unique integer for each process | |||||
n = None | |||||
def get_rpc_proxy(url, node_number, timeout=None, coveragedir=None): | |||||
""" | """ | ||||
Wait until everybody has the same transactions in their memory | Args: | ||||
pools | url (str): URL of the RPC server to call | ||||
node_number (int): the node number (or id) that this calls to | |||||
Kwargs: | |||||
timeout (int): HTTP timeout in seconds | |||||
Returns: | |||||
AuthServiceProxy. convenience object for making RPC calls. | |||||
""" | """ | ||||
while timeout > 0: | proxy_kwargs = {} | ||||
pool = set(rpc_connections[0].getrawmempool()) | if timeout is not None: | ||||
num_match = 1 | proxy_kwargs['timeout'] = timeout | ||||
for i in range(1, len(rpc_connections)): | |||||
if set(rpc_connections[i].getrawmempool()) == pool: | |||||
num_match = num_match + 1 | |||||
if num_match == len(rpc_connections): | |||||
return | |||||
time.sleep(wait) | |||||
timeout -= wait | |||||
raise AssertionError("Mempool sync failed") | |||||
proxy = AuthServiceProxy(url, **proxy_kwargs) | |||||
proxy.url = url # store URL on proxy for info | |||||
coverage_logfile = coverage.get_filename( | |||||
coveragedir, node_number) if coveragedir else None | |||||
return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile) | |||||
bitcoind_processes = {} | |||||
def p2p_port(n): | |||||
assert(n <= MAX_NODES) | |||||
return PORT_MIN + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) | |||||
def rpc_port(n): | |||||
return PORT_MIN + PORT_RANGE + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) | |||||
def rpc_url(datadir, i, rpchost=None): | |||||
rpc_u, rpc_p = get_auth_cookie(datadir, i) | |||||
host = '127.0.0.1' | |||||
port = rpc_port(i) | |||||
if rpchost: | |||||
parts = rpchost.split(':') | |||||
if len(parts) == 2: | |||||
host, port = parts | |||||
else: | |||||
host = rpchost | |||||
return "http://%s:%s@%s:%d" % (rpc_u, rpc_p, host, int(port)) | |||||
# Node functions | |||||
################ | |||||
def initialize_datadir(dirname, n): | def initialize_datadir(dirname, n): | ||||
datadir = os.path.join(dirname, "node" + str(n)) | datadir = os.path.join(dirname, "node" + str(n)) | ||||
if not os.path.isdir(datadir): | if not os.path.isdir(datadir): | ||||
os.makedirs(datadir) | os.makedirs(datadir) | ||||
with open(os.path.join(datadir, "bitcoin.conf"), 'w', encoding='utf8') as f: | with open(os.path.join(datadir, "bitcoin.conf"), 'w', encoding='utf8') as f: | ||||
f.write("regtest=1\n") | f.write("regtest=1\n") | ||||
Show All 26 Lines | if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")): | ||||
split_userpass = userpass.split(':') | split_userpass = userpass.split(':') | ||||
user = split_userpass[0] | user = split_userpass[0] | ||||
password = split_userpass[1] | password = split_userpass[1] | ||||
if user is None or password is None: | if user is None or password is None: | ||||
raise ValueError("No RPC credentials") | raise ValueError("No RPC credentials") | ||||
return user, password | return user, password | ||||
def rpc_url(datadir, i, rpchost=None): | |||||
rpc_u, rpc_p = get_auth_cookie(datadir, i) | |||||
host = '127.0.0.1' | |||||
port = rpc_port(i) | |||||
if rpchost: | |||||
parts = rpchost.split(':') | |||||
if len(parts) == 2: | |||||
host, port = parts | |||||
else: | |||||
host = rpchost | |||||
return "http://%s:%s@%s:%d" % (rpc_u, rpc_p, host, int(port)) | |||||
def wait_for_bitcoind_start(process, datadir, i, rpchost=None): | |||||
''' | |||||
Wait for bitcoind to start. This means that RPC is accessible and fully initialized. | |||||
Raise an exception if bitcoind exits during initialization. | |||||
''' | |||||
while True: | |||||
if process.poll() is not None: | |||||
raise Exception( | |||||
'bitcoind exited with status %i during initialization' % process.returncode) | |||||
try: | |||||
# Check if .cookie file to be created | |||||
rpc = get_rpc_proxy(rpc_url(datadir, i, rpchost), i) | |||||
blocks = rpc.getblockcount() | |||||
break # break out of loop on success | |||||
except IOError as e: | |||||
if e.errno != errno.ECONNREFUSED: # Port not yet open? | |||||
raise # unknown IO error | |||||
except JSONRPCException as e: # Initialization phase | |||||
if e.error['code'] != -28: # RPC in warmup? | |||||
raise # unknown JSON RPC exception | |||||
except ValueError as e: # cookie file not found and no rpcuser or rpcassword. bitcoind still starting | |||||
if "No RPC credentials" not in str(e): | |||||
raise | |||||
time.sleep(0.25) | |||||
def locate_bitcoind_binary(): | |||||
""" | |||||
Find bitcoind binary if possible. | |||||
""" | |||||
bitcoind_binary = os.getenv("BITCOIND", DEFAULT_BITCOIND) | |||||
if os.path.exists(bitcoind_binary): | |||||
return bitcoind_binary | |||||
if os.path.exists(os.path.join('src', DEFAULT_BITCOIND)): | |||||
bitcoind_binary = os.path.abspath( | |||||
os.path.join('src', DEFAULT_BITCOIND)) | |||||
elif bitcoind_binary == 'bitcoind' or not os.path.exists(bitcoind_binary): | |||||
# If BITCOIND was specified and exists, use it, otherwise look for source. | |||||
# get_srcdir() already returns an absolute path | |||||
src_dir_cand = get_srcdir(sys.argv[0]) | |||||
if src_dir_cand and os.path.exists( | |||||
os.path.join(src_dir_cand, 'src', DEFAULT_BITCOIND)): | |||||
bitcoind_binary = os.path.join( | |||||
src_dir_cand, 'src', DEFAULT_BITCOIND) | |||||
else: | |||||
sys.stderr.write("Unable to locate bitcoind for this test.\n") | |||||
sys.exit(1) | |||||
return bitcoind_binary | |||||
def _start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None): | |||||
"""Start a bitcoind and return RPC connection to it | |||||
This function should only be called from within test_framework, not by individual test scripts.""" | |||||
datadir = os.path.join(dirname, "node" + str(i)) | |||||
if binary is None: | |||||
binary = locate_bitcoind_binary() | |||||
args = [binary, "-datadir=" + datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", | |||||
"-debug", "-debugexclude=libevent", "-mocktime=" + str(get_mocktime()), "-uacomment=testnode%d" % i] | |||||
if extra_args is not None: | |||||
args.extend(extra_args) | |||||
bitcoind_processes[i] = subprocess.Popen(args, stderr=stderr) | |||||
logger.debug( | |||||
"initialize_chain: bitcoind started, waiting for RPC to come up") | |||||
wait_for_bitcoind_start(bitcoind_processes[i], datadir, i, rpchost) | |||||
logger.debug("initialize_chain: RPC successfully started") | |||||
proxy = get_rpc_proxy(rpc_url(datadir, i, rpchost), i, timeout=timewait) | |||||
if COVERAGE_DIR: | |||||
coverage.write_all_rpc_commands(COVERAGE_DIR, proxy) | |||||
return proxy | |||||
def assert_start_raises_init_error(i, dirname, extra_args=None, expected_msg=None): | |||||
with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr: | |||||
try: | |||||
node = _start_node(i, dirname, extra_args, stderr=log_stderr) | |||||
_stop_node(node, i) | |||||
except Exception as e: | |||||
assert 'bitcoind exited' in str(e) # node must have shutdown | |||||
if expected_msg is not None: | |||||
log_stderr.seek(0) | |||||
stderr = log_stderr.read().decode('utf-8') | |||||
if expected_msg not in stderr: | |||||
raise AssertionError( | |||||
"Expected error \"" + expected_msg + "\" not found in:\n" + stderr) | |||||
else: | |||||
if expected_msg is None: | |||||
assert_msg = "bitcoind should have exited with an error" | |||||
else: | |||||
assert_msg = "bitcoind should have exited with expected error " + expected_msg | |||||
raise AssertionError(assert_msg) | |||||
def _start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): | |||||
"""Start multiple bitcoinds, return RPC connections to them | |||||
This function should only be called from within test_framework, not by individual test scripts.""" | |||||
if extra_args is None: | |||||
extra_args = [None for _ in range(num_nodes)] | |||||
if binary is None: | |||||
binary = [None for _ in range(num_nodes)] | |||||
assert_equal(len(extra_args), num_nodes) | |||||
if binary is None: | |||||
binary = [None for _ in range(num_nodes)] | |||||
assert_equal(len(binary), num_nodes) | |||||
rpcs = [] | |||||
try: | |||||
for i in range(num_nodes): | |||||
rpcs.append(_start_node( | |||||
i, dirname, extra_args[i], rpchost, timewait=timewait, binary=binary[i])) | |||||
except: # If one node failed to start, stop the others | |||||
_stop_nodes(rpcs) | |||||
raise | |||||
return rpcs | |||||
def log_filename(dirname, n_node, logname): | def log_filename(dirname, n_node, logname): | ||||
return os.path.join(dirname, "node" + str(n_node), "regtest", logname) | return os.path.join(dirname, "node" + str(n_node), "regtest", logname) | ||||
def _stop_node(node, i): | def get_bip9_status(node, key): | ||||
"""Stop a bitcoind test node | info = node.getblockchaininfo() | ||||
return info['bip9_softforks'][key] | |||||
This function should only be called from within test_framework, not by individual test scripts.""" | |||||
logger.debug("Stopping node %d" % i) | |||||
try: | |||||
node.stop() | |||||
except http.client.CannotSendRequest as e: | |||||
logger.exception("Unable to stop node") | |||||
return_code = bitcoind_processes[i].wait( | |||||
timeout=BITCOIND_PROC_WAIT_TIMEOUT) | |||||
assert_equal(return_code, 0) | |||||
del bitcoind_processes[i] | |||||
def _stop_nodes(nodes): | |||||
"""Stop multiple bitcoind test nodes | |||||
This function should only be called from within test_framework, not by individual test scripts.""" | |||||
for i, node in enumerate(nodes): | |||||
_stop_node(node, i) | |||||
assert not bitcoind_processes.values() # All connections must be gone now | |||||
def set_node_times(nodes, t): | def set_node_times(nodes, t): | ||||
for node in nodes: | for node in nodes: | ||||
node.setmocktime(t) | node.setmocktime(t) | ||||
def disconnect_nodes(from_connection, node_num): | def disconnect_nodes(from_connection, node_num): | ||||
Show All 17 Lines | while any(peer['version'] == 0 for peer in from_connection.getpeerinfo()): | ||||
time.sleep(0.1) | time.sleep(0.1) | ||||
def connect_nodes_bi(nodes, a, b): | def connect_nodes_bi(nodes, a, b): | ||||
connect_nodes(nodes[a], b) | connect_nodes(nodes[a], b) | ||||
connect_nodes(nodes[b], a) | connect_nodes(nodes[b], a) | ||||
def sync_blocks(rpc_connections, *, wait=1, timeout=60): | |||||
""" | |||||
Wait until everybody has the same tip. | |||||
sync_blocks needs to be called with an rpc_connections set that has least | |||||
one node already synced to the latest, stable tip, otherwise there's a | |||||
chance it might return before all nodes are stably synced. | |||||
""" | |||||
# Use getblockcount() instead of waitforblockheight() to determine the | |||||
# initial max height because the two RPCs look at different internal global | |||||
# variables (chainActive vs latestBlock) and the former gets updated | |||||
# earlier. | |||||
maxheight = max(x.getblockcount() for x in rpc_connections) | |||||
start_time = cur_time = time.time() | |||||
while cur_time <= start_time + timeout: | |||||
tips = [r.waitforblockheight(maxheight, int(wait * 1000)) | |||||
for r in rpc_connections] | |||||
if all(t["height"] == maxheight for t in tips): | |||||
if all(t["hash"] == tips[0]["hash"] for t in tips): | |||||
return | |||||
raise AssertionError("Block sync failed, mismatched block hashes:{}".format( | |||||
"".join("\n {!r}".format(tip) for tip in tips))) | |||||
cur_time = time.time() | |||||
raise AssertionError("Block sync to height {} timed out:{}".format( | |||||
maxheight, "".join("\n {!r}".format(tip) for tip in tips))) | |||||
def sync_chain(rpc_connections, *, wait=1, timeout=60): | |||||
""" | |||||
Wait until everybody has the same best block | |||||
""" | |||||
while timeout > 0: | |||||
best_hash = [x.getbestblockhash() for x in rpc_connections] | |||||
if best_hash == [best_hash[0]] * len(best_hash): | |||||
return | |||||
time.sleep(wait) | |||||
timeout -= wait | |||||
raise AssertionError("Chain sync failed: Best block hashes don't match") | |||||
def sync_mempools(rpc_connections, *, wait=1, timeout=60): | |||||
""" | |||||
Wait until everybody has the same transactions in their memory | |||||
pools | |||||
""" | |||||
while timeout > 0: | |||||
pool = set(rpc_connections[0].getrawmempool()) | |||||
num_match = 1 | |||||
for i in range(1, len(rpc_connections)): | |||||
if set(rpc_connections[i].getrawmempool()) == pool: | |||||
num_match = num_match + 1 | |||||
if num_match == len(rpc_connections): | |||||
return | |||||
time.sleep(wait) | |||||
timeout -= wait | |||||
raise AssertionError("Mempool sync failed") | |||||
# Transaction/Block functions | |||||
############################# | |||||
def find_output(node, txid, amount): | def find_output(node, txid, amount): | ||||
""" | """ | ||||
Return index to output of txid with value amount | Return index to output of txid with value amount | ||||
Raises exception if there is none. | Raises exception if there is none. | ||||
""" | """ | ||||
txdata = node.getrawtransaction(txid, 1) | txdata = node.getrawtransaction(txid, 1) | ||||
for i in range(len(txdata["vout"])): | for i in range(len(txdata["vout"])): | ||||
if txdata["vout"][i]["value"] == amount: | if txdata["vout"][i]["value"] == amount: | ||||
return i | return i | ||||
raise RuntimeError( | raise RuntimeError("find_output txid %s : %s not found" % | ||||
"find_output txid %s : %s not found" % (txid, str(amount))) | (txid, str(amount))) | ||||
def gather_inputs(from_node, amount_needed, confirmations_required=1): | def gather_inputs(from_node, amount_needed, confirmations_required=1): | ||||
""" | """ | ||||
Return a random set of unspent txouts that are enough to pay amount_needed | Return a random set of unspent txouts that are enough to pay amount_needed | ||||
""" | """ | ||||
assert(confirmations_required >= 0) | assert(confirmations_required >= 0) | ||||
utxo = from_node.listunspent(confirmations_required) | utxo = from_node.listunspent(confirmations_required) | ||||
random.shuffle(utxo) | random.shuffle(utxo) | ||||
inputs = [] | inputs = [] | ||||
total_in = Decimal("0.00000000") | total_in = Decimal("0.00000000") | ||||
while total_in < amount_needed and len(utxo) > 0: | while total_in < amount_needed and len(utxo) > 0: | ||||
t = utxo.pop() | t = utxo.pop() | ||||
total_in += t["amount"] | total_in += t["amount"] | ||||
inputs.append( | inputs.append( | ||||
{"txid": t["txid"], "vout": t["vout"], "address": t["address"]}) | {"txid": t["txid"], "vout": t["vout"], "address": t["address"]}) | ||||
if total_in < amount_needed: | if total_in < amount_needed: | ||||
raise RuntimeError( | raise RuntimeError("Insufficient funds: need %d, have %d" % | ||||
"Insufficient funds: need %d, have %d" % (amount_needed, total_in)) | (amount_needed, total_in)) | ||||
return (total_in, inputs) | return (total_in, inputs) | ||||
def make_change(from_node, amount_in, amount_out, fee): | def make_change(from_node, amount_in, amount_out, fee): | ||||
""" | """ | ||||
Create change output(s), return them | Create change output(s), return them | ||||
""" | """ | ||||
outputs = {} | outputs = {} | ||||
amount = amount_out + fee | amount = amount_out + fee | ||||
change = amount_in - amount | change = amount_in - amount | ||||
if change > amount * 2: | if change > amount * 2: | ||||
# Create an extra change output to break up big inputs | # Create an extra change output to break up big inputs | ||||
change_address = from_node.getnewaddress() | change_address = from_node.getnewaddress() | ||||
# Split change in two, being careful of rounding: | # Split change in two, being careful of rounding: | ||||
outputs[change_address] = Decimal(change / 2).quantize( | outputs[change_address] = Decimal( | ||||
Decimal('0.00000001'), rounding=ROUND_DOWN) | change / 2).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) | ||||
change = amount_in - amount - outputs[change_address] | change = amount_in - amount - outputs[change_address] | ||||
if change > 0: | if change > 0: | ||||
outputs[from_node.getnewaddress()] = change | outputs[from_node.getnewaddress()] = change | ||||
return outputs | return outputs | ||||
def send_zeropri_transaction(from_node, to_node, amount, fee): | def send_zeropri_transaction(from_node, to_node, amount, fee): | ||||
""" | """ | ||||
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants): | ||||
outputs[to_node.getnewaddress()] = float(amount) | outputs[to_node.getnewaddress()] = float(amount) | ||||
rawtx = from_node.createrawtransaction(inputs, outputs) | rawtx = from_node.createrawtransaction(inputs, outputs) | ||||
signresult = from_node.signrawtransaction(rawtx, None, None, "ALL|FORKID") | signresult = from_node.signrawtransaction(rawtx, None, None, "ALL|FORKID") | ||||
txid = from_node.sendrawtransaction(signresult["hex"], True) | txid = from_node.sendrawtransaction(signresult["hex"], True) | ||||
return (txid, signresult["hex"], fee) | return (txid, signresult["hex"], fee) | ||||
def assert_fee_amount(fee, tx_size, fee_per_kB): | |||||
"""Assert the fee was in range""" | |||||
target_fee = tx_size * fee_per_kB / 1000 | |||||
if fee < target_fee: | |||||
raise AssertionError( | |||||
"Fee of %s BTC too low! (Should be %s BTC)" % (str(fee), str(target_fee))) | |||||
# allow the wallet's estimation to be at most 2 bytes off | |||||
if fee > (tx_size + 2) * fee_per_kB / 1000: | |||||
raise AssertionError( | |||||
"Fee of %s BTC too high! (Should be %s BTC)" % (str(fee), str(target_fee))) | |||||
def assert_equal(thing1, thing2, *args): | |||||
if thing1 != thing2 or any(thing1 != arg for arg in args): | |||||
raise AssertionError("not(%s)" % " == ".join(str(arg) | |||||
for arg in (thing1, thing2) + args)) | |||||
def assert_greater_than(thing1, thing2): | |||||
if thing1 <= thing2: | |||||
raise AssertionError("%s <= %s" % (str(thing1), str(thing2))) | |||||
def assert_greater_than_or_equal(thing1, thing2): | |||||
if thing1 < thing2: | |||||
raise AssertionError("%s < %s" % (str(thing1), str(thing2))) | |||||
def assert_raises(exc, fun, *args, **kwds): | |||||
assert_raises_message(exc, None, fun, *args, **kwds) | |||||
def assert_raises_message(exc, message, fun, *args, **kwds): | |||||
try: | |||||
fun(*args, **kwds) | |||||
except exc as e: | |||||
if message is not None and message not in e.error['message']: | |||||
raise AssertionError( | |||||
"Expected substring not found:" + e.error['message']) | |||||
except Exception as e: | |||||
raise AssertionError( | |||||
"Unexpected exception raised: " + type(e).__name__) | |||||
else: | |||||
raise AssertionError("No exception raised") | |||||
def assert_raises_jsonrpc(code, message, fun, *args, **kwds): | |||||
"""Run an RPC and verify that a specific JSONRPC exception code and message is raised. | |||||
Calls function `fun` with arguments `args` and `kwds`. Catches a JSONRPCException | |||||
and verifies that the error code and message are as expected. Throws AssertionError if | |||||
no JSONRPCException was returned or if the error code/message are not as expected. | |||||
Args: | |||||
code (int), optional: the error code returned by the RPC call (defined | |||||
in src/rpc/protocol.h). Set to None if checking the error code is not required. | |||||
message (string), optional: [a substring of] the error string returned by the | |||||
RPC call. Set to None if checking the error string is not required | |||||
fun (function): the function to call. This should be the name of an RPC. | |||||
args*: positional arguments for the function. | |||||
kwds**: named arguments for the function. | |||||
""" | |||||
try: | |||||
fun(*args, **kwds) | |||||
except JSONRPCException as e: | |||||
# JSONRPCException was thrown as expected. Check the code and message values are correct. | |||||
if (code is not None) and (code != e.error["code"]): | |||||
raise AssertionError( | |||||
"Unexpected JSONRPC error code %i" % e.error["code"]) | |||||
if (message is not None) and (message not in e.error['message']): | |||||
raise AssertionError( | |||||
"Expected substring not found:" + e.error['message']) | |||||
except Exception as e: | |||||
raise AssertionError( | |||||
"Unexpected exception raised: " + type(e).__name__) | |||||
else: | |||||
raise AssertionError("No exception raised") | |||||
def assert_is_hex_string(string): | |||||
try: | |||||
int(string, 16) | |||||
except Exception as e: | |||||
raise AssertionError( | |||||
"Couldn't interpret %r as hexadecimal; raised: %s" % (string, e)) | |||||
def assert_is_hash_string(string, length=64): | |||||
if not isinstance(string, str): | |||||
raise AssertionError("Expected a string, got type %r" % type(string)) | |||||
elif length and len(string) != length: | |||||
raise AssertionError( | |||||
"String of length %d expected; got %d" % (length, len(string))) | |||||
elif not re.match('[abcdef0-9]+$', string): | |||||
raise AssertionError( | |||||
"String %r contains invalid characters for a hash." % string) | |||||
def assert_array_result(object_array, to_match, expected, should_not_find=False): | |||||
""" | |||||
Pass in array of JSON objects, a dictionary with key/value pairs | |||||
to match against, and another dictionary with expected key/value | |||||
pairs. | |||||
If the should_not_find flag is true, to_match should not be found | |||||
in object_array | |||||
""" | |||||
if should_not_find == True: | |||||
assert_equal(expected, {}) | |||||
num_matched = 0 | |||||
for item in object_array: | |||||
all_match = True | |||||
for key, value in to_match.items(): | |||||
if item[key] != value: | |||||
all_match = False | |||||
if not all_match: | |||||
continue | |||||
elif should_not_find == True: | |||||
num_matched = num_matched + 1 | |||||
for key, value in expected.items(): | |||||
if item[key] != value: | |||||
raise AssertionError( | |||||
"%s : expected %s=%s" % (str(item), str(key), str(value))) | |||||
num_matched = num_matched + 1 | |||||
if num_matched == 0 and should_not_find != True: | |||||
raise AssertionError("No objects matched %s" % (str(to_match))) | |||||
if num_matched > 0 and should_not_find == True: | |||||
raise AssertionError("Objects were found %s" % (str(to_match))) | |||||
def satoshi_round(amount): | |||||
return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) | |||||
# Helper to create at least "count" utxos | # Helper to create at least "count" utxos | ||||
# Pass in a fee that is sufficient for relay and mining new transactions. | # Pass in a fee that is sufficient for relay and mining new transactions. | ||||
def create_confirmed_utxos(fee, node, count, age=101): | def create_confirmed_utxos(fee, node, count, age=101): | ||||
node.generate(int(0.5 * count) + age) | node.generate(int(0.5 * count) + age) | ||||
utxos = node.listunspent() | utxos = node.listunspent() | ||||
iterations = count - len(utxos) | iterations = count - len(utxos) | ||||
addr1 = node.getnewaddress() | addr1 = node.getnewaddress() | ||||
addr2 = node.getnewaddress() | addr2 = node.getnewaddress() | ||||
if iterations <= 0: | if iterations <= 0: | ||||
return utxos | return utxos | ||||
for i in range(iterations): | for i in range(iterations): | ||||
t = utxos.pop() | t = utxos.pop() | ||||
inputs = [] | inputs = [] | ||||
inputs.append({"txid": t["txid"], "vout": t["vout"]}) | inputs.append({"txid": t["txid"], "vout": t["vout"]}) | ||||
outputs = {} | outputs = {} | ||||
send_value = t['amount'] - fee | send_value = t['amount'] - fee | ||||
outputs[addr1] = satoshi_round(send_value / 2) | outputs[addr1] = satoshi_round(send_value / 2) | ||||
outputs[addr2] = satoshi_round(send_value / 2) | outputs[addr2] = satoshi_round(send_value / 2) | ||||
raw_tx = node.createrawtransaction(inputs, outputs) | raw_tx = node.createrawtransaction(inputs, outputs) | ||||
signed_tx = node.signrawtransaction( | signed_tx = node.signrawtransaction( | ||||
raw_tx, None, None, "ALL|FORKID")["hex"] | raw_tx, None, None, "ALL|FORKID")["hex"] | ||||
txid = node.sendrawtransaction(signed_tx) | node.sendrawtransaction(signed_tx) | ||||
while (node.getmempoolinfo()['size'] > 0): | while (node.getmempoolinfo()['size'] > 0): | ||||
node.generate(1) | node.generate(1) | ||||
utxos = node.listunspent() | utxos = node.listunspent() | ||||
assert(len(utxos) >= count) | assert(len(utxos) >= count) | ||||
return utxos | return utxos | ||||
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines | def mine_large_block(node, utxos=None): | ||||
if len(utxos) < num: | if len(utxos) < num: | ||||
utxos.clear() | utxos.clear() | ||||
utxos.extend(node.listunspent()) | utxos.extend(node.listunspent()) | ||||
fee = 100 * node.getnetworkinfo()["relayfee"] | fee = 100 * node.getnetworkinfo()["relayfee"] | ||||
create_lots_of_big_transactions(node, txouts, utxos, num, fee=fee) | create_lots_of_big_transactions(node, txouts, utxos, num, fee=fee) | ||||
node.generate(1) | node.generate(1) | ||||
def get_bip9_status(node, key): | |||||
info = node.getblockchaininfo() | |||||
return info['bip9_softforks'][key] | |||||
def get_srcdir(calling_script=None): | def get_srcdir(calling_script=None): | ||||
""" | """ | ||||
Try to find out the base folder containing the 'src' folder. | Try to find out the base folder containing the 'src' folder. | ||||
If SRCDIR is set it does a sanity check and returns that. | If SRCDIR is set it does a sanity check and returns that. | ||||
Otherwise it goes on a search and rescue mission. | Otherwise it goes on a search and rescue mission. | ||||
Returns None if it cannot find a suitable folder. | Returns None if it cannot find a suitable folder. | ||||
TODO: This is only used for cdefs, consider moving that there. | |||||
""" | """ | ||||
def contains_src(path_to_check): | def contains_src(path_to_check): | ||||
if not path_to_check: | if not path_to_check: | ||||
return False | return False | ||||
else: | else: | ||||
cand_path = os.path.join(path_to_check, 'src') | cand_path = os.path.join(path_to_check, 'src') | ||||
return os.path.exists(cand_path) and os.path.isdir(cand_path) | return os.path.exists(cand_path) and os.path.isdir(cand_path) | ||||
Show All 25 Lines |