diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 560b3c208..f0f5669d2 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -1,130 +1,137 @@ #!/usr/bin/env python3 # Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node disconnect and ban behavior""" import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, + connect_nodes, connect_nodes_bi, wait_until, ) class DisconnectBanTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 def run_test(self): + self.log.info("Connect nodes both way") + connect_nodes(self.nodes[0], self.nodes[1]) + connect_nodes(self.nodes[1], self.nodes[0]) + self.log.info("Test setban and listbanned RPCs") self.log.info("setban: successfully ban single IP address") # node1 should have 2 connections to node0 at this point assert_equal(len(self.nodes[1].getpeerinfo()), 2) self.nodes[1].setban(subnet="127.0.0.1", command="add") wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0, timeout=10) # all nodes must be disconnected at this point assert_equal(len(self.nodes[1].getpeerinfo()), 0) assert_equal(len(self.nodes[1].listbanned()), 1) self.log.info("clearbanned: successfully clear ban list") self.nodes[1].clearbanned() assert_equal(len(self.nodes[1].listbanned()), 0) self.nodes[1].setban("127.0.0.0/24", "add") self.log.info("setban: fail to ban an already banned subnet") assert_equal(len(self.nodes[1].listbanned()), 1) assert_raises_rpc_error( -23, "IP/Subnet already banned", self.nodes[1].setban, "127.0.0.1", "add") self.log.info("setban: fail to ban an invalid subnet") assert_raises_rpc_error( -30, "Error: Invalid IP/Subnet", self.nodes[1].setban, "127.0.0.1/42", "add") # still only one banned ip because 127.0.0.1 is within the range of # 127.0.0.0/24 assert_equal(len(self.nodes[1].listbanned()), 1) self.log.info("setban remove: fail to unban a non-banned subnet") assert_raises_rpc_error( -30, "Error: Unban failed", self.nodes[1].setban, "127.0.0.1", "remove") assert_equal(len(self.nodes[1].listbanned()), 1) self.log.info("setban remove: successfully unban subnet") self.nodes[1].setban("127.0.0.0/24", "remove") assert_equal(len(self.nodes[1].listbanned()), 0) self.nodes[1].clearbanned() assert_equal(len(self.nodes[1].listbanned()), 0) self.log.info("setban: test persistence across node restart") self.nodes[1].setban("127.0.0.0/32", "add") self.nodes[1].setban("127.0.0.0/24", "add") # Set the mocktime so we can control when bans expire old_time = int(time.time()) self.nodes[1].setmocktime(old_time) # ban for 1 seconds self.nodes[1].setban("192.168.0.1", "add", 1) # ban for 1000 seconds self.nodes[1].setban( "2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) listBeforeShutdown = self.nodes[1].listbanned() assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) # Move time forward by 3 seconds so the third ban has expired self.nodes[1].setmocktime(old_time + 3) assert_equal(len(self.nodes[1].listbanned()), 3) self.stop_node(1) self.start_node(1) listAfterShutdown = self.nodes[1].listbanned() assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) assert_equal("127.0.0.0/32", listAfterShutdown[1]['address']) assert_equal("/19" in listAfterShutdown[2]['address'], True) # Clear ban lists self.nodes[1].clearbanned() - connect_nodes_bi(self.nodes[0], self.nodes[1]) + self.log.info("Connect nodes both way") + connect_nodes(self.nodes[0], self.nodes[1]) + connect_nodes(self.nodes[1], self.nodes[0]) self.log.info("Test disconnectnode RPCs") self.log.info( "disconnectnode: fail to disconnect when calling with address and nodeid") address1 = self.nodes[0].getpeerinfo()[0]['addr'] node1 = self.nodes[0].getpeerinfo()[0]['addr'] assert_raises_rpc_error( -32602, "Only one of address and nodeid should be provided.", self.nodes[0].disconnectnode, address=address1, nodeid=node1) self.log.info( "disconnectnode: fail to disconnect when calling with junk address") assert_raises_rpc_error(-29, "Node not found in connected nodes", self.nodes[0].disconnectnode, address="221B Baker Street") self.log.info( "disconnectnode: successfully disconnect node by address") address1 = self.nodes[0].getpeerinfo()[0]['addr'] self.nodes[0].disconnectnode(address=address1) wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) assert not [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] self.log.info("disconnectnode: successfully reconnect node") # reconnect the node connect_nodes_bi(self.nodes[0], self.nodes[1]) assert_equal(len(self.nodes[0].getpeerinfo()), 2) assert [node for node in self.nodes[0] .getpeerinfo() if node['addr'] == address1] self.log.info( "disconnectnode: successfully disconnect node by node id") id1 = self.nodes[0].getpeerinfo()[0]['id'] self.nodes[0].disconnectnode(nodeid=id1) wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) assert not [node for node in self.nodes[0].getpeerinfo() if node['id'] == id1] if __name__ == '__main__': DisconnectBanTest().main() diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index df29882c7..369a31f47 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -1,189 +1,190 @@ #!/usr/bin/env python3 # Copyright (c) 2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ Test transaction download behavior """ from test_framework.address import ADDRESS_BCHREG_UNSPENDABLE from test_framework.messages import ( CInv, CTransaction, FromHex, MSG_TX, MSG_TYPE_MASK, msg_inv, msg_notfound, ) from test_framework.mininode import ( P2PInterface, mininode_lock, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, wait_until, ) import time class TestP2PConn(P2PInterface): def __init__(self): super().__init__() self.tx_getdata_count = 0 def on_getdata(self, message): for i in message.inv: if i.type & MSG_TYPE_MASK == MSG_TX: self.tx_getdata_count += 1 # Constants from net_processing GETDATA_TX_INTERVAL = 60 # seconds MAX_GETDATA_RANDOM_DELAY = 2 # seconds INBOUND_PEER_TX_DELAY = 2 # seconds MAX_GETDATA_IN_FLIGHT = 100 TX_EXPIRY_INTERVAL = GETDATA_TX_INTERVAL * 10 # Python test constants NUM_INBOUND = 10 MAX_GETDATA_INBOUND_WAIT = GETDATA_TX_INTERVAL + \ MAX_GETDATA_RANDOM_DELAY + INBOUND_PEER_TX_DELAY class TxDownloadTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = False self.num_nodes = 2 def test_tx_requests(self): self.log.info( "Test that we request transactions from all our peers, eventually") txid = 0xdeadbeef self.log.info("Announce the txid from each incoming peer to node 0") msg = msg_inv([CInv(t=1, h=txid)]) for p in self.nodes[0].p2ps: p.send_message(msg) p.sync_with_ping() outstanding_peer_index = [i for i in range(len(self.nodes[0].p2ps))] def getdata_found(peer_index): p = self.nodes[0].p2ps[peer_index] with mininode_lock: return p.last_message.get( "getdata") and p.last_message["getdata"].inv[-1].hash == txid node_0_mocktime = int(time.time()) while outstanding_peer_index: node_0_mocktime += MAX_GETDATA_INBOUND_WAIT self.nodes[0].setmocktime(node_0_mocktime) wait_until(lambda: any(getdata_found(i) for i in outstanding_peer_index)) for i in outstanding_peer_index: if getdata_found(i): outstanding_peer_index.remove(i) self.nodes[0].setmocktime(0) self.log.info("All outstanding peers received a getdata") def test_inv_block(self): self.log.info("Generate a transaction on node 0") tx = self.nodes[0].createrawtransaction( inputs=[{ # coinbase "txid": self.nodes[0].getblock(self.nodes[0].getblockhash(1))['tx'][0], "vout": 0 }], outputs={ADDRESS_BCHREG_UNSPENDABLE: 50 - 0.00025}, ) tx = self.nodes[0].signrawtransactionwithkey( hexstring=tx, privkeys=[self.nodes[0].get_deterministic_priv_key().key], )['hex'] ctx = FromHex(CTransaction(), tx) txid = int(ctx.rehash(), 16) self.log.info( "Announce the transaction to all nodes from all {} incoming peers, but never send it".format(NUM_INBOUND)) msg = msg_inv([CInv(t=1, h=txid)]) for p in self.peers: p.send_message(msg) p.sync_with_ping() self.log.info("Put the tx in node 0's mempool") self.nodes[0].sendrawtransaction(tx) # Since node 1 is connected outbound to an honest peer (node 0), it # should get the tx within a timeout. (Assuming that node 0 # announced the tx within the timeout) # The timeout is the sum of # * the worst case until the tx is first requested from an inbound # peer, plus # * the first time it is re-requested from the outbound peer, plus # * 2 seconds to avoid races + assert self.nodes[1].getpeerinfo()[0]['inbound'] is False timeout = 2 + (MAX_GETDATA_RANDOM_DELAY + INBOUND_PEER_TX_DELAY) + ( GETDATA_TX_INTERVAL + MAX_GETDATA_RANDOM_DELAY) self.log.info( "Tx should be received at node 1 after {} seconds".format(timeout)) self.sync_mempools(timeout=timeout) def test_in_flight_max(self): self.log.info("Test that we don't request more than {} transactions from any peer, every {} minutes".format( MAX_GETDATA_IN_FLIGHT, TX_EXPIRY_INTERVAL / 60)) txids = [i for i in range(MAX_GETDATA_IN_FLIGHT + 2)] p = self.nodes[0].p2ps[0] with mininode_lock: p.tx_getdata_count = 0 p.send_message(msg_inv([CInv(t=1, h=i) for i in txids])) wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT, lock=mininode_lock) with mininode_lock: assert_equal(p.tx_getdata_count, MAX_GETDATA_IN_FLIGHT) self.log.info( "Now check that if we send a NOTFOUND for a transaction, we'll get one more request") p.send_message(msg_notfound(vec=[CInv(t=1, h=txids[0])])) wait_until( lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT + 1, timeout=10, lock=mininode_lock) with mininode_lock: assert_equal(p.tx_getdata_count, MAX_GETDATA_IN_FLIGHT + 1) WAIT_TIME = TX_EXPIRY_INTERVAL // 2 + TX_EXPIRY_INTERVAL self.log.info( "if we wait about {} minutes, we should eventually get more requests".format( WAIT_TIME / 60)) self.nodes[0].setmocktime(int(time.time() + WAIT_TIME)) wait_until(lambda: p.tx_getdata_count == MAX_GETDATA_IN_FLIGHT + 2) self.nodes[0].setmocktime(0) def run_test(self): # Setup the p2p connections self.peers = [] for node in self.nodes: for i in range(NUM_INBOUND): self.peers.append(node.add_p2p_connection(TestP2PConn())) self.log.info( "Nodes are setup with {} incoming connections each".format(NUM_INBOUND)) # Test the in-flight max first, because we want no transactions in # flight ahead of this test. self.test_in_flight_max() self.test_inv_block() self.test_tx_requests() if __name__ == '__main__': TxDownloadTest().main() diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 9896ad06a..c812d190a 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -1,164 +1,171 @@ #!/usr/bin/env python3 # Copyright (c) 2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC calls related to net. Tests correspond to code in rpc/net.cpp. """ from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_greater_than_or_equal, assert_greater_than, assert_raises_rpc_error, - connect_nodes_bi, + connect_nodes, p2p_port, wait_until, ) from test_framework.mininode import P2PInterface from test_framework.messages import CAddress, msg_addr, NODE_NETWORK class NetTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 self.extra_args = [["-minrelaytxfee=0.00001000"], ["-minrelaytxfee=0.00000500"]] def run_test(self): + self.log.info('Connect nodes both way') + connect_nodes(self.nodes[0], self.nodes[1]) + connect_nodes(self.nodes[1], self.nodes[0]) + self._test_connection_count() self._test_getnettotals() self._test_getnetworkinginfo() self._test_getaddednodeinfo() self._test_getpeerinfo() self._test_getnodeaddresses() def _test_connection_count(self): # connect_nodes_bi connects each node to the other assert_equal(self.nodes[0].getconnectioncount(), 2) def _test_getnettotals(self): # getnettotals totalbytesrecv and totalbytessent should be # consistent with getpeerinfo. Since the RPC calls are not atomic, # and messages might have been recvd or sent between RPC calls, call # getnettotals before and after and verify that the returned values # from getpeerinfo are bounded by those values. net_totals_before = self.nodes[0].getnettotals() peer_info = self.nodes[0].getpeerinfo() net_totals_after = self.nodes[0].getnettotals() assert_equal(len(peer_info), 2) peers_recv = sum([peer['bytesrecv'] for peer in peer_info]) peers_sent = sum([peer['bytessent'] for peer in peer_info]) assert_greater_than_or_equal( peers_recv, net_totals_before['totalbytesrecv']) assert_greater_than_or_equal( net_totals_after['totalbytesrecv'], peers_recv) assert_greater_than_or_equal( peers_sent, net_totals_before['totalbytessent']) assert_greater_than_or_equal( net_totals_after['totalbytessent'], peers_sent) # test getnettotals and getpeerinfo by doing a ping # the bytes sent/received should change # note ping and pong are 32 bytes each self.nodes[0].ping() wait_until(lambda: (self.nodes[0].getnettotals()[ 'totalbytessent'] >= net_totals_after['totalbytessent'] + 32 * 2), timeout=1) wait_until(lambda: (self.nodes[0].getnettotals()[ 'totalbytesrecv'] >= net_totals_after['totalbytesrecv'] + 32 * 2), timeout=1) peer_info_after_ping = self.nodes[0].getpeerinfo() for before, after in zip(peer_info, peer_info_after_ping): assert_greater_than_or_equal( after['bytesrecv_per_msg'].get( 'pong', 0), before['bytesrecv_per_msg'].get( 'pong', 0) + 32) assert_greater_than_or_equal( after['bytessent_per_msg'].get( 'ping', 0), before['bytessent_per_msg'].get( 'ping', 0) + 32) def _test_getnetworkinginfo(self): assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) self.nodes[0].setnetworkactive(state=False) assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], False) # Wait a bit for all sockets to close wait_until(lambda: self.nodes[0].getnetworkinfo()[ 'connections'] == 0, timeout=3) self.nodes[0].setnetworkactive(state=True) - connect_nodes_bi(self.nodes[0], self.nodes[1]) + self.log.info('Connect nodes both way') + connect_nodes(self.nodes[0], self.nodes[1]) + connect_nodes(self.nodes[1], self.nodes[0]) + assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) def _test_getaddednodeinfo(self): assert_equal(self.nodes[0].getaddednodeinfo(), []) # add a node (node2) to node0 ip_port = "127.0.0.1:{}".format(p2p_port(2)) self.nodes[0].addnode(node=ip_port, command='add') # check that the node has indeed been added added_nodes = self.nodes[0].getaddednodeinfo(ip_port) assert_equal(len(added_nodes), 1) assert_equal(added_nodes[0]['addednode'], ip_port) # check that a non-existent node returns an error assert_raises_rpc_error(-24, "Node has not been added", self.nodes[0].getaddednodeinfo, '1.1.1.1') def _test_getpeerinfo(self): peer_info = [x.getpeerinfo() for x in self.nodes] # check both sides of bidirectional connection between nodes # the address bound to on one side will be the source address for the # other node assert_equal(peer_info[0][0]['addrbind'], peer_info[1][0]['addr']) assert_equal(peer_info[1][0]['addrbind'], peer_info[0][0]['addr']) assert_equal(peer_info[0][0]['minfeefilter'], Decimal("0.00000500")) assert_equal(peer_info[1][0]['minfeefilter'], Decimal("0.00001000")) def _test_getnodeaddresses(self): self.nodes[0].add_p2p_connection(P2PInterface()) # send some addresses to the node via the p2p message addr msg = msg_addr() imported_addrs = [] for i in range(256): a = "123.123.123.{}".format(i) imported_addrs.append(a) addr = CAddress() addr.time = 100000000 addr.nServices = NODE_NETWORK addr.ip = a addr.port = 8333 msg.addrs.append(addr) self.nodes[0].p2p.send_and_ping(msg) # obtain addresses via rpc call and check they were ones sent in before REQUEST_COUNT = 10 node_addresses = self.nodes[0].getnodeaddresses(REQUEST_COUNT) assert_equal(len(node_addresses), REQUEST_COUNT) for a in node_addresses: assert_greater_than(a["time"], 1527811200) # 1st June 2018 assert_equal(a["services"], NODE_NETWORK) assert a["address"] in imported_addrs assert_equal(a["port"], 8333) assert_raises_rpc_error(-8, "Address count out of range", self.nodes[0].getnodeaddresses, -1) # addrman's size cannot be known reliably after insertion, as hash collisions may occur # so only test that requesting a large number of addresses returns less # than that LARGE_REQUEST_COUNT = 10000 node_addresses = self.nodes[0].getnodeaddresses(LARGE_REQUEST_COUNT) assert_greater_than(LARGE_REQUEST_COUNT, len(node_addresses)) if __name__ == '__main__': NetTest().main() diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 53be4a31b..86792e49d 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,615 +1,626 @@ #!/usr/bin/env python3 # Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" import argparse import configparser from enum import Enum import logging import os import pdb import shutil import sys import tempfile import time from .authproxy import JSONRPCException from . import coverage from .test_node import TestNode from .mininode import NetworkThread from .util import ( assert_equal, check_json_precision, + connect_nodes, connect_nodes_bi, disconnect_nodes, get_datadir_path, initialize_datadir, MAX_NODES, p2p_port, PortSeed, rpc_port, sync_blocks, sync_mempools, ) class TestStatus(Enum): PASSED = 1 FAILED = 2 SKIPPED = 3 TEST_EXIT_PASSED = 0 TEST_EXIT_FAILED = 1 TEST_EXIT_SKIPPED = 77 # Timestamp is Dec. 1st, 2019 at 00:00:00 TIMESTAMP_IN_THE_PAST = 1575158400 class SkipTest(Exception): """This exception is raised to skip a test""" def __init__(self, message): self.message = message class BitcoinTestMetaClass(type): """Metaclass for BitcoinTestFramework. Ensures that any attempt to register a subclass of `BitcoinTestFramework` adheres to a standard whereby the subclass overrides `set_test_params` and `run_test` but DOES NOT override either `__init__` or `main`. If any of those standards are violated, a ``TypeError`` is raised.""" def __new__(cls, clsname, bases, dct): if not clsname == 'BitcoinTestFramework': if not ('run_test' in dct and 'set_test_params' in dct): raise TypeError("BitcoinTestFramework subclasses must override " "'run_test' and 'set_test_params'") if '__init__' in dct or 'main' in dct: raise TypeError("BitcoinTestFramework subclasses may not override " "'__init__' or 'main'") return super().__new__(cls, clsname, bases, dct) class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """Base class for a bitcoin test script. Individual bitcoin test scripts should subclass this class and override the set_test_params() and run_test() methods. Individual tests can also override the following methods to customize the test setup: - add_options() - setup_chain() - setup_network() - setup_nodes() The __init__() and main() methods should not be overridden. This class also contains various public and private helper methods.""" def __init__(self): """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method""" self.chain = 'regtest' self.setup_clean_chain = False self.nodes = [] self.network_thread = None # Wait for up to 60 seconds for the RPC server to respond self.rpc_timeout = 60 self.supports_cli = False self.bind_to_localhost_only = True def main(self): """Main function. This should not be overridden by the subclass test scripts.""" parser = argparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") parser.add_argument("--noshutdown", dest="noshutdown", default=False, action="store_true", help="Don't stop bitcoinds after the test execution") parser.add_argument("--cachedir", dest="cachedir", default=os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../../cache"), help="Directory for caching pregenerated datadirs (default: %(default)s)") parser.add_argument("--tmpdir", dest="tmpdir", help="Root directory for datadirs") parser.add_argument("-l", "--loglevel", dest="loglevel", default="INFO", help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory.") parser.add_argument("--tracerpc", dest="trace_rpc", default=False, action="store_true", help="Print out all RPC calls as they are made") parser.add_argument("--portseed", dest="port_seed", default=os.getpid(), type=int, help="The seed to use for assigning port numbers (default: current process id)") parser.add_argument("--coveragedir", dest="coveragedir", help="Write tested RPC commands into this directory") parser.add_argument("--configfile", dest="configfile", default=os.path.abspath(os.path.dirname(os.path.realpath( __file__)) + "/../../config.ini"), help="Location of the test framework config file (default: %(default)s)") parser.add_argument("--pdbonfailure", dest="pdbonfailure", default=False, action="store_true", help="Attach a python debugger if test fails") parser.add_argument("--usecli", dest="usecli", default=False, action="store_true", help="use bitcoin-cli instead of RPC for all commands") parser.add_argument("--with-phononactivation", dest="phononactivation", default=False, action="store_true", help="Activate phonon update on timestamp {}".format(TIMESTAMP_IN_THE_PAST)) self.add_options(parser) self.options = parser.parse_args() self.set_test_params() assert hasattr( self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" PortSeed.n = self.options.port_seed check_json_precision() self.options.cachedir = os.path.abspath(self.options.cachedir) config = configparser.ConfigParser() config.read_file(open(self.options.configfile, encoding='utf-8')) self.config = config self.options.bitcoind = os.getenv( "BITCOIND", default=config["environment"]["BUILDDIR"] + '/src/bitcoind' + config["environment"]["EXEEXT"]) self.options.bitcoincli = os.getenv( "BITCOINCLI", default=config["environment"]["BUILDDIR"] + '/src/bitcoin-cli' + config["environment"]["EXEEXT"]) self.options.emulator = config["environment"]["EMULATOR"] or None os.environ['PATH'] = config['environment']['BUILDDIR'] + os.pathsep + \ config['environment']['BUILDDIR'] + os.path.sep + "qt" + os.pathsep + \ os.environ['PATH'] # Set up temp directory and start logging if self.options.tmpdir: self.options.tmpdir = os.path.abspath(self.options.tmpdir) os.makedirs(self.options.tmpdir, exist_ok=False) else: self.options.tmpdir = tempfile.mkdtemp(prefix="test") self._start_logging() self.log.debug('Setting up network thread') self.network_thread = NetworkThread() self.network_thread.start() success = TestStatus.FAILED try: if self.options.usecli: if not self.supports_cli: raise SkipTest( "--usecli specified but test does not support using CLI") self.skip_if_no_cli() self.skip_test_if_missing_module() self.setup_chain() self.setup_network() self.run_test() success = TestStatus.PASSED except JSONRPCException: self.log.exception("JSONRPC error") except SkipTest as e: self.log.warning("Test Skipped: {}".format(e.message)) success = TestStatus.SKIPPED except AssertionError: self.log.exception("Assertion failed") except KeyError: self.log.exception("Key error") except Exception: self.log.exception("Unexpected exception caught during testing") except KeyboardInterrupt: self.log.warning("Exiting after keyboard interrupt") if success == TestStatus.FAILED and self.options.pdbonfailure: print("Testcase failed. Attaching python debugger. Enter ? for help") pdb.set_trace() self.log.debug('Closing down network thread') self.network_thread.close() if not self.options.noshutdown: self.log.info("Stopping nodes") if self.nodes: self.stop_nodes() else: for node in self.nodes: node.cleanup_on_exit = False self.log.info( "Note: bitcoinds were not stopped and may still be running") if not self.options.nocleanup and not self.options.noshutdown and success != TestStatus.FAILED: self.log.info("Cleaning up {} on exit".format(self.options.tmpdir)) cleanup_tree_on_exit = True else: self.log.warning( "Not cleaning up dir {}".format(self.options.tmpdir)) cleanup_tree_on_exit = False if success == TestStatus.PASSED: self.log.info("Tests successful") exit_code = TEST_EXIT_PASSED elif success == TestStatus.SKIPPED: self.log.info("Test skipped") exit_code = TEST_EXIT_SKIPPED else: self.log.error( "Test failed. Test logging available at {}/test_framework.log".format(self.options.tmpdir)) self.log.error("Hint: Call {} '{}' to consolidate all logs".format(os.path.normpath( os.path.dirname(os.path.realpath(__file__)) + "/../combine_logs.py"), self.options.tmpdir)) exit_code = TEST_EXIT_FAILED logging.shutdown() if cleanup_tree_on_exit: shutil.rmtree(self.options.tmpdir) sys.exit(exit_code) # Methods to override in subclass test scripts. def set_test_params(self): """Tests must this method to change default values for number of nodes, topology, etc""" raise NotImplementedError def add_options(self, parser): """Override this method to add command-line options to the test""" pass def skip_test_if_missing_module(self): """Override this method to skip a test if a module is not compiled""" pass def setup_chain(self): """Override this method to customize blockchain setup""" self.log.info("Initializing test directory " + self.options.tmpdir) if self.setup_clean_chain: self._initialize_chain_clean() else: self._initialize_chain() def setup_network(self): """Override this method to customize test network topology""" self.setup_nodes() # Connect the nodes as a "chain". This allows us # to split the network between nodes 1 and 2 to get # two halves that can work on competing chains. + # + # Topology looks like this: + # node0 <-- node1 <-- node2 <-- node3 + # + # If all nodes are in IBD (clean chain from genesis), node0 is assumed to be the source of blocks (miner). To + # ensure block propagation, all nodes will establish outgoing connections toward node0. + # See fPreferredDownload in net_processing. + # + # If further outbound connections are needed, they can be added at the beginning of the test with e.g. + # connect_nodes(self.nodes[1], 2) for i in range(self.num_nodes - 1): - connect_nodes_bi(self.nodes[i], self.nodes[i + 1]) + connect_nodes(self.nodes[i + 1], self.nodes[i]) self.sync_all() def setup_nodes(self): """Override this method to customize test node setup""" extra_args = None if hasattr(self, "extra_args"): extra_args = self.extra_args self.add_nodes(self.num_nodes, extra_args) self.start_nodes() self.import_deterministic_coinbase_privkeys() if not self.setup_clean_chain: for n in self.nodes: assert_equal(n.getblockchaininfo()["blocks"], 199) # To ensure that all nodes are out of IBD, the most recent block # must have a timestamp not too old (see IsInitialBlockDownload()). self.log.debug('Generate a block with current time') block_hash = self.nodes[0].generate(1)[0] block = self.nodes[0].getblock(blockhash=block_hash, verbosity=0) for n in self.nodes: n.submitblock(block) chain_info = n.getblockchaininfo() assert_equal(chain_info["blocks"], 200) assert_equal(chain_info["initialblockdownload"], False) def import_deterministic_coinbase_privkeys(self): for n in self.nodes: try: n.getwalletinfo() except JSONRPCException as e: assert str(e).startswith('Method not found') continue n.importprivkey( privkey=n.get_deterministic_priv_key().key, label='coinbase') def run_test(self): """Tests must override this method to define test logic""" raise NotImplementedError # Public helper methods. These can be accessed by the subclass test # scripts. def add_nodes(self, num_nodes, extra_args=None, *, host=None, binary=None): """Instantiate TestNode objects. Should only be called once after the nodes have been specified in set_test_params().""" if self.bind_to_localhost_only: extra_confs = [["bind=127.0.0.1"]] * num_nodes else: extra_confs = [[]] * num_nodes if extra_args is None: extra_args = [[]] * num_nodes if binary is None: binary = [self.options.bitcoind] * num_nodes assert_equal(len(extra_confs), num_nodes) assert_equal(len(extra_args), num_nodes) assert_equal(len(binary), num_nodes) for i in range(num_nodes): self.nodes.append(TestNode( i, get_datadir_path(self.options.tmpdir, i), chain=self.chain, host=host, rpc_port=rpc_port(i), p2p_port=p2p_port(i), timewait=self.rpc_timeout, bitcoind=binary[i], bitcoin_cli=self.options.bitcoincli, coverage_dir=self.options.coveragedir, extra_conf=extra_confs[i], extra_args=extra_args[i], use_cli=self.options.usecli, emulator=self.options.emulator, )) if self.options.phononactivation: self.nodes[i].extend_default_args( ["-phononactivationtime={}".format(TIMESTAMP_IN_THE_PAST)]) def start_node(self, i, *args, **kwargs): """Start a bitcoind""" node = self.nodes[i] node.start(*args, **kwargs) node.wait_for_rpc_connection() if self.options.coveragedir is not None: coverage.write_all_rpc_commands(self.options.coveragedir, node.rpc) def start_nodes(self, extra_args=None, *args, **kwargs): """Start multiple bitcoinds""" if extra_args is None: extra_args = [None] * self.num_nodes assert_equal(len(extra_args), self.num_nodes) try: for i, node in enumerate(self.nodes): node.start(extra_args[i], *args, **kwargs) for node in self.nodes: node.wait_for_rpc_connection() except BaseException: # If one node failed to start, stop the others self.stop_nodes() raise if self.options.coveragedir is not None: for node in self.nodes: coverage.write_all_rpc_commands( self.options.coveragedir, node.rpc) def stop_node(self, i, expected_stderr='', wait=0): """Stop a bitcoind test node""" self.nodes[i].stop_node(expected_stderr, wait=wait) self.nodes[i].wait_until_stopped() def stop_nodes(self, wait=0): """Stop multiple bitcoind test nodes""" for node in self.nodes: # Issue RPC to stop nodes node.stop_node(wait=wait) for node in self.nodes: # Wait for nodes to stop node.wait_until_stopped() def restart_node(self, i, extra_args=None): """Stop and start a test node""" self.stop_node(i) self.start_node(i, extra_args) def wait_for_node_exit(self, i, timeout): self.nodes[i].process.wait(timeout) def split_network(self): """ Split the network of four nodes into nodes 0/1 and 2/3. """ disconnect_nodes(self.nodes[1], self.nodes[2]) disconnect_nodes(self.nodes[2], self.nodes[1]) self.sync_all(self.nodes[:2]) self.sync_all(self.nodes[2:]) def join_network(self): """ Join the (previously split) network halves together. """ connect_nodes_bi(self.nodes[1], self.nodes[2]) self.sync_all() def sync_blocks(self, nodes=None, **kwargs): sync_blocks(nodes or self.nodes, **kwargs) def sync_mempools(self, nodes=None, **kwargs): sync_mempools(nodes or self.nodes, **kwargs) def sync_all(self, nodes=None, **kwargs): self.sync_blocks(nodes, **kwargs) self.sync_mempools(nodes, **kwargs) # Private helper methods. These should not be accessed by the subclass # test scripts. def _start_logging(self): # Add logger and logging handlers self.log = logging.getLogger('TestFramework') self.log.setLevel(logging.DEBUG) # Create file handler to log all messages fh = logging.FileHandler( self.options.tmpdir + '/test_framework.log', encoding='utf-8') fh.setLevel(logging.DEBUG) # Create console handler to log messages to stderr. By default this # logs only error messages, but can be configured with --loglevel. ch = logging.StreamHandler(sys.stdout) # User can provide log level as a number or string (eg DEBUG). loglevel # was caught as a string, so try to convert it to an int ll = int(self.options.loglevel) if self.options.loglevel.isdigit( ) else self.options.loglevel.upper() ch.setLevel(ll) # Format logs the same as bitcoind's debug.log with microprecision (so # log files can be concatenated and sorted) formatter = logging.Formatter( fmt='%(asctime)s.%(msecs)03d000Z %(name)s (%(levelname)s): %(message)s', datefmt='%Y-%m-%dT%H:%M:%S') formatter.converter = time.gmtime fh.setFormatter(formatter) ch.setFormatter(formatter) # add the handlers to the logger self.log.addHandler(fh) self.log.addHandler(ch) if self.options.trace_rpc: rpc_logger = logging.getLogger("BitcoinRPC") rpc_logger.setLevel(logging.DEBUG) rpc_handler = logging.StreamHandler(sys.stdout) rpc_handler.setLevel(logging.DEBUG) rpc_logger.addHandler(rpc_handler) def _initialize_chain(self): """Initialize a pre-mined blockchain for use by the test. Create a cache of a 199-block-long chain Afterward, create num_nodes copies from the cache.""" # Use node 0 to create the cache for all other nodes CACHE_NODE_ID = 0 cache_node_dir = get_datadir_path(self.options.cachedir, CACHE_NODE_ID) assert self.num_nodes <= MAX_NODES if not os.path.isdir(cache_node_dir): self.log.debug( "Creating cache directory {}".format(cache_node_dir)) initialize_datadir( self.options.cachedir, CACHE_NODE_ID, self.chain) self.nodes.append( TestNode( CACHE_NODE_ID, cache_node_dir, chain=self.chain, extra_conf=["bind=127.0.0.1"], extra_args=['-disablewallet'], host=None, rpc_port=rpc_port(CACHE_NODE_ID), p2p_port=p2p_port(CACHE_NODE_ID), timewait=self.rpc_timeout, bitcoind=self.options.bitcoind, bitcoin_cli=self.options.bitcoincli, coverage_dir=None, emulator=self.options.emulator, )) if self.options.phononactivation: self.nodes[CACHE_NODE_ID].extend_default_args( ["-phononactivationtime={}".format(TIMESTAMP_IN_THE_PAST)]) self.start_node(CACHE_NODE_ID) cache_node = self.nodes[CACHE_NODE_ID] # Wait for RPC connections to be ready cache_node.wait_for_rpc_connection() # Set a time in the past, so that blocks don't end up in the future cache_node.setmocktime( cache_node.getblockheader( cache_node.getbestblockhash())['time']) # Create a 199-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. # The 4th node gets only 24 immature blocks so that the very last # block in the cache does not age too much (have an old tip age). # This is needed so that we are out of IBD when the test starts, # see the tip age check in IsInitialBlockDownload(). for i in range(8): cache_node.generatetoaddress( nblocks=25 if i != 7 else 24, address=TestNode.PRIV_KEYS[i % 4].address, ) assert_equal(cache_node.getblockchaininfo()["blocks"], 199) # Shut it down, and clean up cache directories: self.stop_nodes() self.nodes = [] def cache_path(*paths): return os.path.join(cache_node_dir, "regtest", *paths) # Remove empty wallets dir os.rmdir(cache_path('wallets')) for entry in os.listdir(cache_path()): # Only keep chainstate and blocks folder if entry not in ['chainstate', 'blocks']: os.remove(cache_path(entry)) for i in range(self.num_nodes): self.log.debug( "Copy cache directory {} to node {}".format( cache_node_dir, i)) to_dir = get_datadir_path(self.options.tmpdir, i) shutil.copytree(cache_node_dir, to_dir) # Overwrite port/rpcport in bitcoin.conf initialize_datadir(self.options.tmpdir, i, self.chain) def _initialize_chain_clean(self): """Initialize empty blockchain for use by the test. Create an empty blockchain and num_nodes wallets. Useful if a test case wants complete control over initialization.""" for i in range(self.num_nodes): initialize_datadir(self.options.tmpdir, i, self.chain) def skip_if_no_py3_zmq(self): """Attempt to import the zmq package and skip the test if the import fails.""" try: import zmq # noqa except ImportError: raise SkipTest("python3-zmq module not available.") def skip_if_no_bitcoind_zmq(self): """Skip the running test if bitcoind has not been compiled with zmq support.""" if not self.is_zmq_compiled(): raise SkipTest("bitcoind has not been built with zmq enabled.") def skip_if_no_wallet(self): """Skip the running test if wallet has not been compiled.""" if not self.is_wallet_compiled(): raise SkipTest("wallet has not been compiled.") def skip_if_no_wallet_tool(self): """Skip the running test if bitcoin-wallet has not been compiled.""" if not self.is_wallet_tool_compiled(): raise SkipTest("bitcoin-wallet has not been compiled") def skip_if_no_cli(self): """Skip the running test if bitcoin-cli has not been compiled.""" if not self.is_cli_compiled(): raise SkipTest("bitcoin-cli has not been compiled.") def is_cli_compiled(self): """Checks whether bitcoin-cli was compiled.""" config = configparser.ConfigParser() config.read_file(open(self.options.configfile, encoding='utf-8')) return config["components"].getboolean("ENABLE_CLI") def is_wallet_compiled(self): """Checks whether the wallet module was compiled.""" config = configparser.ConfigParser() config.read_file(open(self.options.configfile, encoding='utf-8')) return config["components"].getboolean("ENABLE_WALLET") def is_wallet_tool_compiled(self): """Checks whether bitcoin-wallet was compiled.""" return self.config["components"].getboolean("ENABLE_WALLET_TOOL") def is_zmq_compiled(self): """Checks whether the zmq module was compiled.""" config = configparser.ConfigParser() config.read_file(open(self.options.configfile, encoding='utf-8')) return config["components"].getboolean("ENABLE_ZMQ") diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index 85401f5a4..5081de5e6 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -1,292 +1,300 @@ #!/usr/bin/env python3 # Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listsincelast RPC.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_array_result, assert_raises_rpc_error +from test_framework.util import ( + assert_array_result, + assert_equal, + assert_raises_rpc_error, + connect_nodes, +) -class ListSinceBlockTest (BitcoinTestFramework): +class ListSinceBlockTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.setup_clean_chain = True self.extra_args = [["-noparkdeepreorg"], ["-noparkdeepreorg"], [], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): + # All nodes are in IBD from genesis, so they'll need the miner (node2) to be an outbound connection, or have + # only one connection. (See fPreferredDownload in net_processing) + connect_nodes(self.nodes[1], self.nodes[2]) self.nodes[2].generate(101) self.sync_all() self.test_no_blockhash() self.test_invalid_blockhash() self.test_reorg() self.test_double_spend() self.test_double_send() def test_no_blockhash(self): txid = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) blockhash, = self.nodes[2].generate(1) self.sync_all() txs = self.nodes[0].listtransactions() assert_array_result(txs, {"txid": txid}, { "category": "receive", "amount": 1, "blockhash": blockhash, "confirmations": 1, }) assert_equal( self.nodes[0].listsinceblock(), {"lastblock": blockhash, "removed": [], "transactions": txs}) assert_equal( self.nodes[0].listsinceblock(""), {"lastblock": blockhash, "removed": [], "transactions": txs}) def test_invalid_blockhash(self): assert_raises_rpc_error(-5, "Block not found", self.nodes[0].listsinceblock, "42759cde25462784395a337460bde75f58e73d3f08bd31fdc3507cbac856a2c4") assert_raises_rpc_error(-5, "Block not found", self.nodes[0].listsinceblock, "0000000000000000000000000000000000000000000000000000000000000000") assert_raises_rpc_error(-8, "blockhash must be of length 64 (not 11, for 'invalid-hex')", self.nodes[0].listsinceblock, "invalid-hex") assert_raises_rpc_error(-8, "blockhash must be hexadecimal string (not 'Z000000000000000000000000000000000000000000000000000000000000000')", self.nodes[0].listsinceblock, "Z000000000000000000000000000000000000000000000000000000000000000") def test_reorg(self): ''' `listsinceblock` did not behave correctly when handed a block that was no longer in the main chain: ab0 / \ aa1 [tx0] bb1 | | aa2 bb2 | | aa3 bb3 | bb4 Consider a client that has only seen block `aa3` above. It asks the node to `listsinceblock aa3`. But at some point prior the main chain switched to the bb chain. Previously: listsinceblock would find height=4 for block aa3 and compare this to height=5 for the tip of the chain (bb4). It would then return results restricted to bb3-bb4. Now: listsinceblock finds the fork at ab0 and returns results in the range bb1-bb4. This test only checks that [tx0] is present. ''' # Split network into two self.split_network() # send to nodes[0] from nodes[2] senttx = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) # generate on both sides lastblockhash = self.nodes[1].generate(6)[5] self.nodes[2].generate(7) self.log.info('lastblockhash={}'.format(lastblockhash)) self.sync_all(self.nodes[:2]) self.sync_all(self.nodes[2:]) self.join_network() # listsinceblock(lastblockhash) should now include tx, as seen from # nodes[0] lsbres = self.nodes[0].listsinceblock(lastblockhash) found = False for tx in lsbres['transactions']: if tx['txid'] == senttx: found = True break assert found def test_double_spend(self): ''' This tests the case where the same UTXO is spent twice on two separate blocks as part of a reorg. ab0 / \ aa1 [tx1] bb1 [tx2] | | aa2 bb2 | | aa3 bb3 | bb4 Problematic case: 1. User 1 receives BCH in tx1 from utxo1 in block aa1. 2. User 2 receives BCH in tx2 from utxo1 (same) in block bb1 3. User 1 sees 2 confirmations at block aa3. 4. Reorg into bb chain. 5. User 1 asks `listsinceblock aa3` and does not see that tx1 is now invalidated. Currently the solution to this is to detect that a reorg'd block is asked for in listsinceblock, and to iterate back over existing blocks up until the fork point, and to include all transactions that relate to the node wallet. ''' self.sync_all() # Split network into two self.split_network() # share utxo between nodes[1] and nodes[2] utxos = self.nodes[2].listunspent() utxo = utxos[0] privkey = self.nodes[2].dumpprivkey(utxo['address']) self.nodes[1].importprivkey(privkey) # send from nodes[1] using utxo to nodes[0] change = '{:.8f}'.format(float(utxo['amount']) - 1.0003) recipientDict = { self.nodes[0].getnewaddress(): 1, self.nodes[1].getnewaddress(): change, } utxoDicts = [{ 'txid': utxo['txid'], 'vout': utxo['vout'], }] txid1 = self.nodes[1].sendrawtransaction( self.nodes[1].signrawtransactionwithwallet( self.nodes[1].createrawtransaction(utxoDicts, recipientDict))['hex']) # send from nodes[2] using utxo to nodes[3] recipientDict2 = { self.nodes[3].getnewaddress(): 1, self.nodes[2].getnewaddress(): change, } self.nodes[2].sendrawtransaction( self.nodes[2].signrawtransactionwithwallet( self.nodes[2].createrawtransaction(utxoDicts, recipientDict2))['hex']) # generate on both sides lastblockhash = self.nodes[1].generate(3)[2] self.nodes[2].generate(4) self.join_network() self.sync_all() # gettransaction should work for txid1 assert self.nodes[0].gettransaction( txid1)['txid'] == txid1, "gettransaction failed to find txid1" # listsinceblock(lastblockhash) should now include txid1, as seen from # nodes[0] lsbres = self.nodes[0].listsinceblock(lastblockhash) assert any(tx['txid'] == txid1 for tx in lsbres['removed']) # but it should not include 'removed' if include_removed=false lsbres2 = self.nodes[0].listsinceblock( blockhash=lastblockhash, include_removed=False) assert 'removed' not in lsbres2 def test_double_send(self): ''' This tests the case where the same transaction is submitted twice on two separate blocks as part of a reorg. The former will vanish and the latter will appear as the true transaction (with confirmations dropping as a result). ab0 / \ aa1 [tx1] bb1 | | aa2 bb2 | | aa3 bb3 [tx1] | bb4 Asserted: 1. tx1 is listed in listsinceblock. 2. It is included in 'removed' as it was removed, even though it is now present in a different block. 3. It is listed with a confirmation count of 2 (bb3, bb4), not 3 (aa1, aa2, aa3). ''' self.sync_all() # Split network into two self.split_network() # create and sign a transaction utxos = self.nodes[2].listunspent() utxo = utxos[0] change = '{:.8f}'.format(float(utxo['amount']) - 1.0003) recipientDict = { self.nodes[0].getnewaddress(): 1, self.nodes[2].getnewaddress(): change, } utxoDicts = [{ 'txid': utxo['txid'], 'vout': utxo['vout'], }] signedtxres = self.nodes[2].signrawtransactionwithwallet( self.nodes[2].createrawtransaction(utxoDicts, recipientDict)) assert signedtxres['complete'] signedtx = signedtxres['hex'] # send from nodes[1]; this will end up in aa1 txid1 = self.nodes[1].sendrawtransaction(signedtx) # generate bb1-bb2 on right side self.nodes[2].generate(2) # send from nodes[2]; this will end up in bb3 txid2 = self.nodes[2].sendrawtransaction(signedtx) assert_equal(txid1, txid2) # generate on both sides lastblockhash = self.nodes[1].generate(3)[2] self.nodes[2].generate(2) self.join_network() self.sync_all() # gettransaction should work for txid1 self.nodes[0].gettransaction(txid1) # listsinceblock(lastblockhash) should now include txid1 in transactions # as well as in removed lsbres = self.nodes[0].listsinceblock(lastblockhash) assert any(tx['txid'] == txid1 for tx in lsbres['transactions']) assert any(tx['txid'] == txid1 for tx in lsbres['removed']) # find transaction and ensure confirmations is valid for tx in lsbres['transactions']: if tx['txid'] == txid1: assert_equal(tx['confirmations'], 2) # the same check for the removed array; confirmations should STILL be 2 for tx in lsbres['removed']: if tx['txid'] == txid1: assert_equal(tx['confirmations'], 2) if __name__ == '__main__': ListSinceBlockTest().main()