diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 6f08ebc36..bac1ecfd7 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -1,204 +1,204 @@ #!/usr/bin/env python3 # Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool persistence. By default, bitcoind will dump mempool on shutdown and then reload it on startup. This can be overridden with the -persistmempool=0 command line option. Test is as follows: - start node0, node1 and node2. node1 has -persistmempool=0 - create 5 transactions on node2 to its own address. Note that these are not sent to node0 or node1 addresses because we don't want them to be saved in the wallet. - check that node0 and node1 have 5 transactions in their mempools - shutdown all nodes. - startup node0. Verify that it still has 5 transactions in its mempool. Shutdown node0. This tests that by default the mempool is persistent. - startup node1. Verify that its mempool is empty. Shutdown node1. This tests that with -persistmempool=0, the mempool is not dumped to disk when the node is shut down. - Restart node0 with -persistmempool=0. Verify that its mempool is empty. Shutdown node0. This tests that with -persistmempool=0, the mempool is not loaded from disk on start up. - Restart node0 with -persistmempool. Verify that it has 5 transactions in its mempool. This tests that -persistmempool=0 does not overwrite a previously valid mempool stored on disk. - Remove node0 mempool.dat and verify savemempool RPC recreates it and verify that node1 can load it and has 5 transactions in its mempool. - Verify that savemempool throws when the RPC is called if node1 can't write to disk. """ from decimal import Decimal import os import time -from test_framework.test_framework import BitcoinTestFramework from test_framework.p2p import P2PTxInvStore +from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_greater_than_or_equal, assert_raises_rpc_error, connect_nodes, disconnect_nodes, wait_until, ) class MempoolPersistTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 3 self.extra_args = [[], ["-persistmempool=0"], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): self.log.debug("Send 5 transactions from node2 (to its own address)") tx_creation_time_lower = int(time.time()) for i in range(5): last_txid = self.nodes[2].sendtoaddress( self.nodes[2].getnewaddress(), Decimal("10")) node2_balance = self.nodes[2].getbalance() self.sync_all() tx_creation_time_higher = int(time.time()) self.log.debug( "Verify that node0 and node1 have 5 transactions in their mempools") assert_equal(len(self.nodes[0].getrawmempool()), 5) assert_equal(len(self.nodes[1].getrawmempool()), 5) self.log.debug("Prioritize a transaction on node0") fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] assert_equal(fees['base'], fees['modified']) self.nodes[0].prioritisetransaction(txid=last_txid, fee_delta=1000) fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified']) tx_creation_time = self.nodes[0].getmempoolentry(txid=last_txid)[ 'time'] assert_greater_than_or_equal(tx_creation_time, tx_creation_time_lower) assert_greater_than_or_equal(tx_creation_time_higher, tx_creation_time) # disconnect nodes & make a txn that remains in the unbroadcast set. disconnect_nodes(self.nodes[0], self.nodes[1]) assert(len(self.nodes[0].getpeerinfo()) == 0) assert(len(self.nodes[0].p2ps) == 0) self.nodes[0].sendtoaddress( self.nodes[2].getnewaddress(), Decimal("12")) connect_nodes(self.nodes[0], self.nodes[2]) self.log.debug("Stop-start the nodes. Verify that node0 has the " "transactions in its mempool and node1 does not. " "Verify that node2 calculates its balance correctly " "after loading wallet transactions.") self.stop_nodes() # Give this one a head-start, so we can be "extra-sure" that it didn't # load anything later # Also don't store the mempool, to keep the datadir clean self.start_node(1, extra_args=["-persistmempool=0"]) self.start_node(0) self.start_node(2) # start_node is blocking on the mempool being loaded assert self.nodes[0].getmempoolinfo()["loaded"] assert self.nodes[2].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[0].getrawmempool()), 6) assert_equal(len(self.nodes[2].getrawmempool()), 5) # The others have loaded their mempool. If node_1 loaded anything, we'd # probably notice by now: assert_equal(len(self.nodes[1].getrawmempool()), 0) self.log.debug('Verify prioritization is loaded correctly') fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified']) self.log.debug('Verify time is loaded correctly') assert_equal( tx_creation_time, self.nodes[0].getmempoolentry( txid=last_txid)['time']) # Verify accounting of mempool transactions after restart is correct # Flush mempool to wallet self.nodes[2].syncwithvalidationinterfacequeue() assert_equal(node2_balance, self.nodes[2].getbalance()) # start node0 with wallet disabled so wallet transactions don't get # resubmitted self.log.debug( "Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") self.stop_nodes() self.start_node(0, extra_args=["-persistmempool=0", "-disablewallet"]) assert self.nodes[0].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[0].getrawmempool()), 0) self.log.debug( "Stop-start node0. Verify that it has the transactions in its mempool.") self.stop_nodes() self.start_node(0) assert self.nodes[0].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[0].getrawmempool()), 6) mempooldat0 = os.path.join( self.nodes[0].datadir, self.chain, 'mempool.dat') mempooldat1 = os.path.join( self.nodes[1].datadir, self.chain, 'mempool.dat') self.log.debug( "Remove the mempool.dat file. Verify that savemempool to disk via RPC re-creates it") os.remove(mempooldat0) self.nodes[0].savemempool() assert os.path.isfile(mempooldat0) self.log.debug( "Stop nodes, make node1 use mempool.dat from node0. Verify it has 6 transactions") os.rename(mempooldat0, mempooldat1) self.stop_nodes() self.start_node(1, extra_args=[]) assert self.nodes[1].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[1].getrawmempool()), 6) self.log.debug( "Prevent bitcoind from writing mempool.dat to disk. Verify that `savemempool` fails") # to test the exception we are creating a tmp folder called mempool.dat.new # which is an implementation detail that could change and break this # test mempooldotnew1 = mempooldat1 + '.new' os.mkdir(mempooldotnew1) assert_raises_rpc_error(-1, "Unable to dump mempool to disk", self.nodes[1].savemempool) os.rmdir(mempooldotnew1) self.test_persist_unbroadcast() def test_persist_unbroadcast(self): node0 = self.nodes[0] self.start_node(0) # clear out mempool node0.generate(1) # ensure node0 doesn't have any connections # make a transaction that will remain in the unbroadcast set assert(len(node0.getpeerinfo()) == 0) assert(len(node0.p2ps) == 0) node0.sendtoaddress(self.nodes[1].getnewaddress(), Decimal("12")) # shutdown, then startup with wallet disabled self.stop_nodes() self.start_node(0, extra_args=["-disablewallet"]) # check that txn gets broadcast due to unbroadcast logic conn = node0.add_p2p_connection(P2PTxInvStore()) # 15 min + 1 for buffer node0.mockscheduler(16 * 60) wait_until(lambda: len(conn.get_invs()) == 1) if __name__ == '__main__': MempoolPersistTest().main() diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index f279bf9ee..36136aba7 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -1,71 +1,69 @@ #!/usr/bin/env python3 # Copyright (c) 2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ Test addr relay """ from test_framework.messages import ( CAddress, NODE_NETWORK, msg_addr, ) -from test_framework.p2p import ( - P2PInterface, -) +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, ) import time ADDRS = [] for i in range(10): addr = CAddress() addr.time = int(time.time()) + i addr.nServices = NODE_NETWORK addr.ip = "123.123.123.{}".format(i % 256) addr.port = 8333 + i ADDRS.append(addr) class AddrReceiver(P2PInterface): def on_addr(self, message): for addr in message.addrs: assert_equal(addr.nServices, NODE_NETWORK) assert addr.ip.startswith('123.123.123.') assert (8333 <= addr.port < 8343) class AddrTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = False self.num_nodes = 1 def run_test(self): self.log.info('Create connection that sends addr messages') addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) msg = msg_addr() self.log.info('Send too-large addr message') msg.addrs = ADDRS * 101 with self.nodes[0].assert_debug_log(['addr message size = 1010']): addr_source.send_and_ping(msg) self.log.info( 'Check that addr message content is relayed and added to addrman') addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver()) msg.addrs = ADDRS with self.nodes[0].assert_debug_log([ 'Added 10 addresses from 127.0.0.1: 0 tried', 'received: addr (301 bytes) peer=0', 'sending addr (301 bytes) peer=1', ]): addr_source.send_and_ping(msg) self.nodes[0].setmocktime(int(time.time()) + 30 * 60) addr_receiver.sync_with_ping() if __name__ == '__main__': AddrTest().main() diff --git a/test/functional/p2p_eviction.py b/test/functional/p2p_eviction.py index 4ff953b9d..9e5587994 100755 --- a/test/functional/p2p_eviction.py +++ b/test/functional/p2p_eviction.py @@ -1,150 +1,150 @@ #!/usr/bin/env python3 # Copyright (c) 2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ Test node eviction logic When the number of peers has reached the limit of maximum connections, the next connecting inbound peer will trigger the eviction mechanism. We cannot currently test the parts of the eviction logic that are based on address/netgroup since in the current framework, all peers are connecting from the same local address. See Issue #14210 for more info. Therefore, this test is limited to the remaining protection criteria. """ import time -from test_framework.test_framework import BitcoinTestFramework -from test_framework.p2p import P2PInterface, P2PDataStore -from test_framework.util import assert_equal, wait_until from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import CTransaction, FromHex, msg_pong, msg_tx +from test_framework.p2p import P2PDataStore, P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, wait_until class SlowP2PDataStore(P2PDataStore): def on_ping(self, message): time.sleep(0.1) self.send_message(msg_pong(message.nonce)) class SlowP2PInterface(P2PInterface): def on_ping(self, message): time.sleep(0.1) self.send_message(msg_pong(message.nonce)) class P2PEvict(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 # The choice of maxconnections=32 results in a maximum of 21 inbound connections # (32 - 10 outbound - 1 feeler). 20 inbound peers are protected from eviction: # 4 by netgroup, 4 that sent us blocks, 4 that sent us transactions and # 8 via lowest ping time self.extra_args = [['-maxconnections=32']] def run_test(self): protected_peers = set() # peers that we expect to be protected from eviction current_peer = -1 node = self.nodes[0] node.generatetoaddress(101, node.get_deterministic_priv_key().address) self.log.info( "Create 4 peers and protect them from eviction by sending us a block") for _ in range(4): block_peer = node.add_p2p_connection(SlowP2PDataStore()) current_peer += 1 block_peer.sync_with_ping() best_block = node.getbestblockhash() tip = int(best_block, 16) best_block_time = node.getblock(best_block)['time'] block = create_block( tip, create_coinbase( node.getblockcount() + 1), best_block_time + 1) block.solve() block_peer.send_blocks_and_test([block], node, success=True) protected_peers.add(current_peer) self.log.info( "Create 5 slow-pinging peers, making them eviction candidates") for _ in range(5): node.add_p2p_connection(SlowP2PInterface()) current_peer += 1 self.log.info( "Create 4 peers and protect them from eviction by sending us a tx") for i in range(4): txpeer = node.add_p2p_connection(SlowP2PInterface()) current_peer += 1 txpeer.sync_with_ping() prevtx = node.getblock(node.getblockhash(i + 1), 2)['tx'][0] rawtx = node.createrawtransaction( inputs=[{'txid': prevtx['txid'], 'vout': 0}], outputs=[ {node.get_deterministic_priv_key().address: 50 - 0.00125}], ) sigtx = node.signrawtransactionwithkey( hexstring=rawtx, privkeys=[node.get_deterministic_priv_key().key], prevtxs=[{ 'txid': prevtx['txid'], 'vout': 0, 'amount': prevtx['vout'][0]['value'], 'scriptPubKey': prevtx['vout'][0]['scriptPubKey']['hex'], }], )['hex'] txpeer.send_message(msg_tx(FromHex(CTransaction(), sigtx))) protected_peers.add(current_peer) self.log.info( "Create 8 peers and protect them from eviction by having faster pings") for _ in range(8): fastpeer = node.add_p2p_connection(P2PInterface()) current_peer += 1 wait_until(lambda: "ping" in fastpeer.last_message, timeout=10) # Make sure by asking the node what the actual min pings are peerinfo = node.getpeerinfo() pings = {} for i in range(len(peerinfo)): pings[i] = peerinfo[i]['minping'] if 'minping' in peerinfo[i] else 1000000 sorted_pings = sorted(pings.items(), key=lambda x: x[1]) # Usually the 8 fast peers are protected. In rare case of unreliable pings, # one of the slower peers might have a faster min ping though. for i in range(8): protected_peers.add(sorted_pings[i][0]) self.log.info("Create peer that triggers the eviction mechanism") node.add_p2p_connection(SlowP2PInterface()) # One of the non-protected peers must be evicted. We can't be sure which one because # 4 peers are protected via netgroup, which is identical for all peers, # and the eviction mechanism doesn't preserve the order of identical # elements. evicted_peers = [] for i in range(len(node.p2ps)): if not node.p2ps[i].is_connected: evicted_peers.append(i) self.log.info("Test that one peer was evicted") self.log.debug( "{} evicted peer: {}".format( len(evicted_peers), set(evicted_peers))) assert_equal(len(evicted_peers), 1) self.log.info("Test that no peer expected to be protected was evicted") self.log.debug( "{} protected peers: {}".format( len(protected_peers), protected_peers)) assert evicted_peers[0] not in protected_peers if __name__ == '__main__': P2PEvict().main() diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index 766a2c761..29e3e5306 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -1,146 +1,146 @@ #!/usr/bin/env python3 # Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of feefilter messages.""" from decimal import Decimal import time from test_framework.messages import MSG_TX, msg_feefilter from test_framework.p2p import ( - p2p_lock, P2PInterface, + p2p_lock, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal def hashToHex(hash): return format(hash, '064x') # Wait up to 60 secs to see if the testnode has received all the expected invs def allInvsMatch(invsExpected, testnode): for x in range(60): with p2p_lock: if (sorted(invsExpected) == sorted(testnode.txinvs)): return True time.sleep(1) return False class FeefilterConn(P2PInterface): feefilter_received = False def on_feefilter(self, message): self.feefilter_received = True def assert_feefilter_received(self, recv: bool): with p2p_lock: assert_equal(self.feefilter_received, recv) class TestP2PConn(P2PInterface): def __init__(self): super().__init__() self.txinvs = [] def on_inv(self, message): for i in message.inv: if (i.type == MSG_TX): self.txinvs.append(hashToHex(i.hash)) def clear_invs(self): with p2p_lock: self.txinvs = [] class FeeFilterTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 # We lower the various required feerates for this test # to catch a corner-case where feefilter used to slightly undercut # mempool and wallet feerate calculation based on GetFee # rounding down 3 places, leading to stranded transactions. # See issue #16499 self.extra_args = [["-minrelaytxfee=0.00000100", "-mintxfee=0.00000100"]] * self.num_nodes def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): self.test_feefilter_forcerelay() self.test_feefilter() def test_feefilter_forcerelay(self): self.log.info( 'Check that peers without forcerelay permission (default) get a feefilter message') self.nodes[0].add_p2p_connection( FeefilterConn()).assert_feefilter_received(True) self.log.info( 'Check that peers with forcerelay permission do not get a feefilter message') self.restart_node(0, extra_args=['-whitelist=forcerelay@127.0.0.1']) self.nodes[0].add_p2p_connection( FeefilterConn()).assert_feefilter_received(False) # Restart to disconnect peers and load default extra_args self.restart_node(0) self.connect_nodes(1, 0) def test_feefilter(self): node1 = self.nodes[1] node0 = self.nodes[0] conn = self.nodes[0].add_p2p_connection(TestP2PConn()) # Test that invs are received by test connection for all txs at # feerate of .2 sat/byte node1.settxfee(Decimal("0.00000200")) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] assert allInvsMatch(txids, conn) conn.clear_invs() # Set a filter of .15 sat/byte on test connection conn.send_and_ping(msg_feefilter(150)) # Test that txs are still being received by test connection # (paying .15 sat/byte) node1.settxfee(Decimal("0.00000150")) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] assert allInvsMatch(txids, conn) conn.clear_invs() # Change tx fee rate to .1 sat/byte and test they are no longer received # by the test connection node1.settxfee(Decimal("0.00000100")) [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] self.sync_mempools() # must be sure node 0 has received all txs # Send one transaction from node0 that should be received, so that we # we can sync the test on receipt (if node1's txs were relayed, they'd # be received by the time this node0 tx is received). This is # unfortunately reliant on the current relay behavior where we batch up # to 35 entries in an inv, which means that when this next transaction # is eligible for relay, the prior transactions from node1 are eligible # as well. node0.settxfee(Decimal("0.00020000")) txids = [node0.sendtoaddress(node0.getnewaddress(), 1)] assert allInvsMatch(txids, conn) conn.clear_invs() # Remove fee filter and check that txs are received again conn.send_and_ping(msg_feefilter(0)) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] assert allInvsMatch(txids, conn) conn.clear_invs() if __name__ == '__main__': FeeFilterTest().main() diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 00f5935c2..289a26bdd 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -1,416 +1,414 @@ #!/usr/bin/env python3 # Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPCs related to blockchainstate. Test the following RPCs: - getblockchaininfo - gettxoutsetinfo - getdifficulty - getbestblockhash - getblockhash - getblockheader - getchaintxstats - getnetworkhashps - verifychain Tests correspond to code in rpc/blockchain.cpp. """ from decimal import Decimal import http.client import subprocess -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - assert_greater_than, - assert_greater_than_or_equal, - assert_raises, - assert_raises_rpc_error, - assert_is_hash_string, - assert_is_hex_string, -) from test_framework.blocktools import ( create_block, create_coinbase, TIME_GENESIS_BLOCK, ) from test_framework.messages import ( msg_block, ) -from test_framework.p2p import ( - P2PInterface, +from test_framework.p2p import P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_greater_than_or_equal, + assert_raises, + assert_raises_rpc_error, + assert_is_hex_string, + assert_is_hash_string, ) class BlockchainTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 # TODO: remove -txindex. Currently required for getrawtransaction call. self.extra_args = [["-txindex"]] self.supports_cli = False def run_test(self): self.mine_chain() # Set extra args with pruning after rescan is complete self.restart_node(0, extra_args=['-stopatheight=207', '-prune=1']) self._test_getblockchaininfo() self._test_getchaintxstats() self._test_gettxoutsetinfo() self._test_getblockheader() self._test_getdifficulty() self._test_getnetworkhashps() self._test_stopatheight() self._test_waitforblockheight() if self.is_wallet_compiled(): self._test_getblock() assert self.nodes[0].verifychain(4, 0) def mine_chain(self): self.log.info('Create some old blocks') address = self.nodes[0].get_deterministic_priv_key().address for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600): # ten-minute steps from genesis block time self.nodes[0].setmocktime(t) self.nodes[0].generatetoaddress(1, address) assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200) def _test_getblockchaininfo(self): self.log.info("Test getblockchaininfo") keys = [ 'bestblockhash', 'blocks', 'chain', 'chainwork', 'difficulty', 'headers', 'initialblockdownload', 'mediantime', 'pruned', 'size_on_disk', 'softforks', 'verificationprogress', 'warnings', ] res = self.nodes[0].getblockchaininfo() # result should have these additional pruning keys if manual pruning is # enabled assert_equal(sorted(res.keys()), sorted( ['pruneheight', 'automatic_pruning'] + keys)) # size_on_disk should be > 0 assert_greater_than(res['size_on_disk'], 0) assert_equal(res['softforks'], { 'testdummy': { 'type': 'bip9', 'bip9': { 'status': 'started', 'bit': 28, 'start_time': 0, # testdummy does not have a timeout so is set to the max # int64 value 'timeout': 0x7fffffffffffffff, 'since': 144, 'statistics': { 'period': 144, 'threshold': 108, 'elapsed': 57, 'count': 57, 'possible': True, }, }, 'active': False, }, }) # pruneheight should be greater or equal to 0 assert_greater_than_or_equal(res['pruneheight'], 0) # check other pruning fields given that prune=1 assert res['pruned'] assert not res['automatic_pruning'] self.restart_node(0, ['-stopatheight=207']) res = self.nodes[0].getblockchaininfo() # should have exact keys assert_equal(sorted(res.keys()), keys) self.restart_node(0, ['-stopatheight=207', '-prune=550']) res = self.nodes[0].getblockchaininfo() # result should have these additional pruning keys if prune=550 assert_equal(sorted(res.keys()), sorted( ['pruneheight', 'automatic_pruning', 'prune_target_size'] + keys)) # check related fields assert res['pruned'] assert_equal(res['pruneheight'], 0) assert res['automatic_pruning'] assert_equal(res['prune_target_size'], 576716800) assert_greater_than(res['size_on_disk'], 0) def _test_getchaintxstats(self): self.log.info("Test getchaintxstats") # Test `getchaintxstats` invalid extra parameters assert_raises_rpc_error( -1, 'getchaintxstats', self.nodes[0].getchaintxstats, 0, '', 0) # Test `getchaintxstats` invalid `nblocks` assert_raises_rpc_error( -1, "JSON value is not an integer as expected", self.nodes[0].getchaintxstats, '') assert_raises_rpc_error( -8, "Invalid block count: should be between 0 and the block's height - 1", self.nodes[0].getchaintxstats, -1) assert_raises_rpc_error(-8, "Invalid block count: should be between 0 and the block's height - 1", self.nodes[ 0].getchaintxstats, self.nodes[0].getblockcount()) # Test `getchaintxstats` invalid `blockhash` assert_raises_rpc_error( -1, "JSON value is not a string as expected", self.nodes[0].getchaintxstats, blockhash=0) assert_raises_rpc_error(-8, "blockhash must be of length 64 (not 1, for '0')", self.nodes[0].getchaintxstats, blockhash='0') assert_raises_rpc_error( -8, "blockhash must be hexadecimal string (not 'ZZZ0000000000000000000000000000000000000000000000000000000000000')", self.nodes[0].getchaintxstats, blockhash='ZZZ0000000000000000000000000000000000000000000000000000000000000') assert_raises_rpc_error( -5, "Block not found", self.nodes[0].getchaintxstats, blockhash='0000000000000000000000000000000000000000000000000000000000000000') blockhash = self.nodes[0].getblockhash(200) self.nodes[0].invalidateblock(blockhash) assert_raises_rpc_error( -8, "Block is not in main chain", self.nodes[0].getchaintxstats, blockhash=blockhash) self.nodes[0].reconsiderblock(blockhash) chaintxstats = self.nodes[0].getchaintxstats(nblocks=1) # 200 txs plus genesis tx assert_equal(chaintxstats['txcount'], 201) # tx rate should be 1 per 10 minutes, or 1/600 # we have to round because of binary math assert_equal(round(chaintxstats['txrate'] * 600, 10), Decimal(1)) b1_hash = self.nodes[0].getblockhash(1) b1 = self.nodes[0].getblock(b1_hash) b200_hash = self.nodes[0].getblockhash(200) b200 = self.nodes[0].getblock(b200_hash) time_diff = b200['mediantime'] - b1['mediantime'] chaintxstats = self.nodes[0].getchaintxstats() assert_equal(chaintxstats['time'], b200['time']) assert_equal(chaintxstats['txcount'], 201) assert_equal(chaintxstats['window_final_block_hash'], b200_hash) assert_equal(chaintxstats['window_final_block_height'], 200) assert_equal(chaintxstats['window_block_count'], 199) assert_equal(chaintxstats['window_tx_count'], 199) assert_equal(chaintxstats['window_interval'], time_diff) assert_equal( round(chaintxstats['txrate'] * time_diff, 10), Decimal(199)) chaintxstats = self.nodes[0].getchaintxstats(blockhash=b1_hash) assert_equal(chaintxstats['time'], b1['time']) assert_equal(chaintxstats['txcount'], 2) assert_equal(chaintxstats['window_final_block_hash'], b1_hash) assert_equal(chaintxstats['window_final_block_height'], 1) assert_equal(chaintxstats['window_block_count'], 0) assert 'window_tx_count' not in chaintxstats assert 'window_interval' not in chaintxstats assert 'txrate' not in chaintxstats def _test_gettxoutsetinfo(self): node = self.nodes[0] res = node.gettxoutsetinfo() assert_equal(res['total_amount'], Decimal('8725.00000000')) assert_equal(res['transactions'], 200) assert_equal(res['height'], 200) assert_equal(res['txouts'], 200) assert_equal(res['bogosize'], 15000), assert_equal(res['bestblock'], node.getblockhash(200)) size = res['disk_size'] assert size > 6400 assert size < 64000 assert_equal(len(res['bestblock']), 64) assert_equal(len(res['hash_serialized']), 64) self.log.info( "Test that gettxoutsetinfo() works for blockchain with just the genesis block") b1hash = node.getblockhash(1) node.invalidateblock(b1hash) res2 = node.gettxoutsetinfo() assert_equal(res2['transactions'], 0) assert_equal(res2['total_amount'], Decimal('0')) assert_equal(res2['height'], 0) assert_equal(res2['txouts'], 0) assert_equal(res2['bogosize'], 0), assert_equal(res2['bestblock'], node.getblockhash(0)) assert_equal(len(res2['hash_serialized']), 64) self.log.info( "Test that gettxoutsetinfo() returns the same result after invalidate/reconsider block") node.reconsiderblock(b1hash) res3 = node.gettxoutsetinfo() # The field 'disk_size' is non-deterministic and can thus not be # compared between res and res3. Everything else should be the same. del res['disk_size'], res3['disk_size'] assert_equal(res, res3) def _test_getblockheader(self): node = self.nodes[0] assert_raises_rpc_error(-8, "hash must be of length 64 (not 8, for 'nonsense')", node.getblockheader, "nonsense") assert_raises_rpc_error( -8, "hash must be hexadecimal string (not 'ZZZ7bb8b1697ea987f3b223ba7819250cae33efacb068d23dc24859824a77844')", node.getblockheader, "ZZZ7bb8b1697ea987f3b223ba7819250cae33efacb068d23dc24859824a77844") assert_raises_rpc_error(-5, "Block not found", node.getblockheader, "0cf7bb8b1697ea987f3b223ba7819250cae33efacb068d23dc24859824a77844") besthash = node.getbestblockhash() secondbesthash = node.getblockhash(199) header = node.getblockheader(blockhash=besthash) assert_equal(header['hash'], besthash) assert_equal(header['height'], 200) assert_equal(header['confirmations'], 1) assert_equal(header['previousblockhash'], secondbesthash) assert_is_hex_string(header['chainwork']) assert_equal(header['nTx'], 1) assert_is_hash_string(header['hash']) assert_is_hash_string(header['previousblockhash']) assert_is_hash_string(header['merkleroot']) assert_is_hash_string(header['bits'], length=None) assert isinstance(header['time'], int) assert isinstance(header['mediantime'], int) assert isinstance(header['nonce'], int) assert isinstance(header['version'], int) assert isinstance(int(header['versionHex'], 16), int) assert isinstance(header['difficulty'], Decimal) def _test_getdifficulty(self): difficulty = self.nodes[0].getdifficulty() # 1 hash in 2 should be valid, so difficulty should be 1/2**31 # binary => decimal => binary math is why we do this check assert abs(difficulty * 2**31 - 1) < 0.0001 def _test_getnetworkhashps(self): hashes_per_second = self.nodes[0].getnetworkhashps() # This should be 2 hashes every 10 minutes or 1/300 assert abs(hashes_per_second * 300 - 1) < 0.0001 def _test_stopatheight(self): assert_equal(self.nodes[0].getblockcount(), 200) self.nodes[0].generatetoaddress( 6, self.nodes[0].get_deterministic_priv_key().address) assert_equal(self.nodes[0].getblockcount(), 206) self.log.debug('Node should not stop at this height') assert_raises(subprocess.TimeoutExpired, lambda: self.nodes[0].process.wait(timeout=3)) try: self.nodes[0].generatetoaddress( 1, self.nodes[0].get_deterministic_priv_key().address) except (ConnectionError, http.client.BadStatusLine): pass # The node already shut down before response self.log.debug('Node should stop at this height...') self.nodes[0].wait_until_stopped() self.start_node(0) assert_equal(self.nodes[0].getblockcount(), 207) def _test_waitforblockheight(self): self.log.info("Test waitforblockheight") node = self.nodes[0] node.add_p2p_connection(P2PInterface()) current_height = node.getblock(node.getbestblockhash())['height'] # Create a fork somewhere below our current height, invalidate the tip # of that fork, and then ensure that waitforblockheight still # works as expected. # # (Previously this was broken based on setting # `rpc/blockchain.cpp:latestblock` incorrectly.) # b20hash = node.getblockhash(20) b20 = node.getblock(b20hash) def solve_and_send_block(prevhash, height, time): b = create_block(prevhash, create_coinbase(height), time) b.solve() node.p2p.send_and_ping(msg_block(b)) return b b21f = solve_and_send_block(int(b20hash, 16), 21, b20['time'] + 1) b22f = solve_and_send_block(b21f.sha256, 22, b21f.nTime + 1) node.invalidateblock(b22f.hash) def assert_waitforheight(height, timeout=2): assert_equal( node.waitforblockheight( height=height, timeout=timeout)['height'], current_height) assert_waitforheight(0) assert_waitforheight(current_height - 1) assert_waitforheight(current_height) assert_waitforheight(current_height + 1) def _test_getblock(self): # Checks for getblock verbose outputs node = self.nodes[0] (blockhash, nextblockhash) = node.generate(2) blockinfo = node.getblock(blockhash, 2) transactioninfo = node.gettransaction(blockinfo['tx'][0]['txid']) blockheaderinfo = node.getblockheader(blockhash, True) assert_equal(blockinfo['hash'], transactioninfo['blockhash']) assert_equal( blockinfo['confirmations'], transactioninfo['confirmations']) assert_equal(blockinfo['height'], blockheaderinfo['height']) assert_equal(blockinfo['versionHex'], blockheaderinfo['versionHex']) assert_equal(blockinfo['version'], blockheaderinfo['version']) assert_equal(blockinfo['size'], 181) assert_equal(blockinfo['merkleroot'], blockheaderinfo['merkleroot']) # Verify transaction data by check the hex values for tx in blockinfo['tx']: rawtransaction = node.getrawtransaction(tx['txid'], True) assert_equal(tx['hex'], rawtransaction['hex']) assert_equal(blockinfo['time'], blockheaderinfo['time']) assert_equal(blockinfo['mediantime'], blockheaderinfo['mediantime']) assert_equal(blockinfo['nonce'], blockheaderinfo['nonce']) assert_equal(blockinfo['bits'], blockheaderinfo['bits']) assert_equal(blockinfo['difficulty'], blockheaderinfo['difficulty']) assert_equal(blockinfo['chainwork'], blockheaderinfo['chainwork']) assert_equal( blockinfo['previousblockhash'], blockheaderinfo['previousblockhash']) assert_equal(blockinfo['nextblockhash'], nextblockhash) assert_equal( blockinfo['nextblockhash'], blockheaderinfo['nextblockhash']) if __name__ == '__main__': BlockchainTest().main() diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 454733925..a72f76a92 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -1,235 +1,235 @@ #!/usr/bin/env python3 # Copyright (c) 2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC calls related to net. Tests correspond to code in rpc/net.cpp. """ from decimal import Decimal from itertools import product import time +from test_framework.p2p import P2PInterface +import test_framework.messages +from test_framework.messages import ( + CAddress, + msg_addr, + NODE_NETWORK, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, assert_equal, assert_greater_than_or_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, p2p_port, wait_until, ) -from test_framework.p2p import P2PInterface -import test_framework.messages -from test_framework.messages import ( - CAddress, - msg_addr, - NODE_NETWORK, -) def assert_net_servicesnames(servicesflag, servicenames): """Utility that checks if all flags are correctly decoded in `getpeerinfo` and `getnetworkinfo`. :param servicesflag: The services as an integer. :param servicenames: The list of decoded services names, as strings. """ servicesflag_generated = 0 for servicename in servicenames: servicesflag_generated |= getattr( test_framework.messages, 'NODE_' + servicename) assert servicesflag_generated == servicesflag class NetTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 self.extra_args = [["-minrelaytxfee=0.00001000"], ["-minrelaytxfee=0.00000500"]] self.supports_cli = False def run_test(self): # Get out of IBD for the minfeefilter and getpeerinfo tests. self.nodes[0].generate(101) # Connect nodes both ways. connect_nodes(self.nodes[0], self.nodes[1]) connect_nodes(self.nodes[1], self.nodes[0]) self.test_connection_count() self.test_getpeerinfo() self.test_getnettotals() self.test_getnetworkinfo() self.test_getaddednodeinfo() self.test_service_flags() self.test_getnodeaddresses() def test_connection_count(self): self.log.info("Test getconnectioncount") # After using `connect_nodes` to connect nodes 0 and 1 to each other. assert_equal(self.nodes[0].getconnectioncount(), 2) def test_getnettotals(self): self.log.info("Test getnettotals") # getnettotals totalbytesrecv and totalbytessent should be # consistent with getpeerinfo. Since the RPC calls are not atomic, # and messages might have been recvd or sent between RPC calls, call # getnettotals before and after and verify that the returned values # from getpeerinfo are bounded by those values. net_totals_before = self.nodes[0].getnettotals() peer_info = self.nodes[0].getpeerinfo() net_totals_after = self.nodes[0].getnettotals() assert_equal(len(peer_info), 2) peers_recv = sum([peer['bytesrecv'] for peer in peer_info]) peers_sent = sum([peer['bytessent'] for peer in peer_info]) assert_greater_than_or_equal( peers_recv, net_totals_before['totalbytesrecv']) assert_greater_than_or_equal( net_totals_after['totalbytesrecv'], peers_recv) assert_greater_than_or_equal( peers_sent, net_totals_before['totalbytessent']) assert_greater_than_or_equal( net_totals_after['totalbytessent'], peers_sent) # test getnettotals and getpeerinfo by doing a ping # the bytes sent/received should change # note ping and pong are 32 bytes each self.nodes[0].ping() wait_until(lambda: (self.nodes[0].getnettotals()[ 'totalbytessent'] >= net_totals_after['totalbytessent'] + 32 * 2), timeout=1) wait_until(lambda: (self.nodes[0].getnettotals()[ 'totalbytesrecv'] >= net_totals_after['totalbytesrecv'] + 32 * 2), timeout=1) peer_info_after_ping = self.nodes[0].getpeerinfo() for before, after in zip(peer_info, peer_info_after_ping): assert_greater_than_or_equal( after['bytesrecv_per_msg'].get( 'pong', 0), before['bytesrecv_per_msg'].get( 'pong', 0) + 32) assert_greater_than_or_equal( after['bytessent_per_msg'].get( 'ping', 0), before['bytessent_per_msg'].get( 'ping', 0) + 32) def test_getnetworkinfo(self): self.log.info("Test getnetworkinfo") assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) self.nodes[0].setnetworkactive(state=False) assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], False) # Wait a bit for all sockets to close wait_until(lambda: self.nodes[0].getnetworkinfo()[ 'connections'] == 0, timeout=3) self.nodes[0].setnetworkactive(state=True) # Connect nodes both ways. connect_nodes(self.nodes[0], self.nodes[1]) connect_nodes(self.nodes[1], self.nodes[0]) assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) # check the `servicesnames` field network_info = [node.getnetworkinfo() for node in self.nodes] for info in network_info: assert_net_servicesnames(int(info["localservices"], 0x10), info["localservicesnames"]) def test_getaddednodeinfo(self): self.log.info("Test getaddednodeinfo") assert_equal(self.nodes[0].getaddednodeinfo(), []) # add a node (node2) to node0 ip_port = "127.0.0.1:{}".format(p2p_port(2)) self.nodes[0].addnode(node=ip_port, command='add') # check that the node has indeed been added added_nodes = self.nodes[0].getaddednodeinfo(ip_port) assert_equal(len(added_nodes), 1) assert_equal(added_nodes[0]['addednode'], ip_port) # check that a non-existent node returns an error assert_raises_rpc_error(-24, "Node has not been added", self.nodes[0].getaddednodeinfo, '1.1.1.1') def test_getpeerinfo(self): self.log.info("Test getpeerinfo") # Create a few getpeerinfo last_block/last_transaction values. if self.is_wallet_compiled(): self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) self.nodes[1].generate(1) self.sync_all() time_now = int(time.time()) peer_info = [x.getpeerinfo() for x in self.nodes] # Verify last_block and last_transaction keys/values. for node, peer, field in product(range(self.num_nodes), range(2), [ 'last_block', 'last_transaction']): assert field in peer_info[node][peer].keys() if peer_info[node][peer][field] != 0: assert_approx(peer_info[node][peer][field], time_now, vspan=60) # check both sides of bidirectional connection between nodes # the address bound to on one side will be the source address for the # other node assert_equal(peer_info[0][0]['addrbind'], peer_info[1][0]['addr']) assert_equal(peer_info[1][0]['addrbind'], peer_info[0][0]['addr']) assert_equal(peer_info[0][0]['minfeefilter'], Decimal("0.00000500")) assert_equal(peer_info[1][0]['minfeefilter'], Decimal("0.00001000")) # check the `servicesnames` field for info in peer_info: assert_net_servicesnames(int(info[0]["services"], 0x10), info[0]["servicesnames"]) def test_service_flags(self): self.log.info("Test service flags") self.nodes[0].add_p2p_connection( P2PInterface(), services=( 1 << 5) | ( 1 << 63)) assert_equal(['UNKNOWN[2^5]', 'UNKNOWN[2^63]'], self.nodes[0].getpeerinfo()[-1]['servicesnames']) self.nodes[0].disconnect_p2ps() def test_getnodeaddresses(self): self.log.info("Test getnodeaddresses") self.nodes[0].add_p2p_connection(P2PInterface()) # send some addresses to the node via the p2p message addr msg = msg_addr() imported_addrs = [] for i in range(256): a = "123.123.123.{}".format(i) imported_addrs.append(a) addr = CAddress() addr.time = 100000000 addr.nServices = NODE_NETWORK addr.ip = a addr.port = 8333 msg.addrs.append(addr) self.nodes[0].p2p.send_and_ping(msg) # obtain addresses via rpc call and check they were ones sent in before REQUEST_COUNT = 10 node_addresses = self.nodes[0].getnodeaddresses(REQUEST_COUNT) assert_equal(len(node_addresses), REQUEST_COUNT) for a in node_addresses: assert_greater_than(a["time"], 1527811200) # 1st June 2018 assert_equal(a["services"], NODE_NETWORK) assert a["address"] in imported_addrs assert_equal(a["port"], 8333) assert_raises_rpc_error(-8, "Address count out of range", self.nodes[0].getnodeaddresses, -1) # addrman's size cannot be known reliably after insertion, as hash collisions may occur # so only test that requesting a large number of addresses returns less # than that LARGE_REQUEST_COUNT = 10000 node_addresses = self.nodes[0].getnodeaddresses(LARGE_REQUEST_COUNT) assert_greater_than(LARGE_REQUEST_COUNT, len(node_addresses)) if __name__ == '__main__': NetTest().main() diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index ef2193f84..88d6bd5cb 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,761 +1,761 @@ #!/usr/bin/env python3 # Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" import argparse import configparser from enum import Enum import logging import os import pdb import random import shutil import sys import tempfile import time from typing import Optional from .authproxy import JSONRPCException from . import coverage -from .test_node import TestNode from .p2p import NetworkThread +from .test_node import TestNode from .util import ( assert_equal, check_json_precision, connect_nodes, disconnect_nodes, get_datadir_path, initialize_datadir, MAX_NODES, p2p_port, PortSeed, rpc_port, ) class TestStatus(Enum): PASSED = 1 FAILED = 2 SKIPPED = 3 TEST_EXIT_PASSED = 0 TEST_EXIT_FAILED = 1 TEST_EXIT_SKIPPED = 77 # Timestamp is Dec. 1st, 2019 at 00:00:00 TIMESTAMP_IN_THE_PAST = 1575158400 TMPDIR_PREFIX = "bitcoin_func_test_" class SkipTest(Exception): """This exception is raised to skip a test""" def __init__(self, message): self.message = message class BitcoinTestMetaClass(type): """Metaclass for BitcoinTestFramework. Ensures that any attempt to register a subclass of `BitcoinTestFramework` adheres to a standard whereby the subclass overrides `set_test_params` and `run_test` but DOES NOT override either `__init__` or `main`. If any of those standards are violated, a ``TypeError`` is raised.""" def __new__(cls, clsname, bases, dct): if not clsname == 'BitcoinTestFramework': if not ('run_test' in dct and 'set_test_params' in dct): raise TypeError("BitcoinTestFramework subclasses must override " "'run_test' and 'set_test_params'") if '__init__' in dct or 'main' in dct: raise TypeError("BitcoinTestFramework subclasses may not override " "'__init__' or 'main'") return super().__new__(cls, clsname, bases, dct) class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """Base class for a bitcoin test script. Individual bitcoin test scripts should subclass this class and override the set_test_params() and run_test() methods. Individual tests can also override the following methods to customize the test setup: - add_options() - setup_chain() - setup_network() - setup_nodes() The __init__() and main() methods should not be overridden. This class also contains various public and private helper methods.""" chain: Optional[str] = None setup_clean_chain: Optional[bool] = None def __init__(self): """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method""" self.chain = 'regtest' self.setup_clean_chain = False self.nodes = [] self.network_thread = None # Wait for up to 60 seconds for the RPC server to respond self.rpc_timeout = 60 self.supports_cli = True self.bind_to_localhost_only = True # We run parse_args before set_test_params for tests who need to # know the parser options during setup. self.parse_args() self.set_test_params() if self.options.timeout_factor == 0: self.options.timeout_factor = 99999 # optionally, increase timeout by a factor self.rpc_timeout = int(self.rpc_timeout * self.options.timeout_factor) def main(self): """Main function. This should not be overridden by the subclass test scripts.""" assert hasattr( self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" try: self.setup() self.run_test() except JSONRPCException: self.log.exception("JSONRPC error") self.success = TestStatus.FAILED except SkipTest as e: self.log.warning("Test Skipped: {}".format(e.message)) self.success = TestStatus.SKIPPED except AssertionError: self.log.exception("Assertion failed") self.success = TestStatus.FAILED except KeyError: self.log.exception("Key error") self.success = TestStatus.FAILED except Exception: self.log.exception("Unexpected exception caught during testing") self.success = TestStatus.FAILED except KeyboardInterrupt: self.log.warning("Exiting after keyboard interrupt") self.success = TestStatus.FAILED finally: exit_code = self.shutdown() sys.exit(exit_code) def parse_args(self): parser = argparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") parser.add_argument("--noshutdown", dest="noshutdown", default=False, action="store_true", help="Don't stop bitcoinds after the test execution") parser.add_argument("--cachedir", dest="cachedir", default=os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../../cache"), help="Directory for caching pregenerated datadirs (default: %(default)s)") parser.add_argument("--tmpdir", dest="tmpdir", help="Root directory for datadirs") parser.add_argument("-l", "--loglevel", dest="loglevel", default="INFO", help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory.") parser.add_argument("--tracerpc", dest="trace_rpc", default=False, action="store_true", help="Print out all RPC calls as they are made") parser.add_argument("--portseed", dest="port_seed", default=os.getpid(), type=int, help="The seed to use for assigning port numbers (default: current process id)") parser.add_argument("--coveragedir", dest="coveragedir", help="Write tested RPC commands into this directory") parser.add_argument("--configfile", dest="configfile", default=os.path.abspath(os.path.dirname(os.path.realpath( __file__)) + "/../../config.ini"), help="Location of the test framework config file (default: %(default)s)") parser.add_argument("--pdbonfailure", dest="pdbonfailure", default=False, action="store_true", help="Attach a python debugger if test fails") parser.add_argument("--usecli", dest="usecli", default=False, action="store_true", help="use bitcoin-cli instead of RPC for all commands") parser.add_argument("--perf", dest="perf", default=False, action="store_true", help="profile running nodes with perf for the duration of the test") parser.add_argument("--valgrind", dest="valgrind", default=False, action="store_true", help="run nodes under the valgrind memory error detector: expect at least a ~10x slowdown, valgrind 3.14 or later required") parser.add_argument("--randomseed", type=int, help="set a random seed for deterministically reproducing a previous test run") parser.add_argument("--with-axionactivation", dest="axionactivation", default=False, action="store_true", help="Activate axion update on timestamp {}".format(TIMESTAMP_IN_THE_PAST)) parser.add_argument( '--timeout-factor', dest="timeout_factor", type=float, default=1.0, help='adjust test timeouts by a factor. ' 'Setting it to 0 disables all timeouts') self.add_options(parser) self.options = parser.parse_args() def setup(self): """Call this method to start up the test framework object with options set.""" PortSeed.n = self.options.port_seed check_json_precision() self.options.cachedir = os.path.abspath(self.options.cachedir) config = configparser.ConfigParser() config.read_file(open(self.options.configfile, encoding='utf-8')) self.config = config fname_bitcoind = os.path.join( config["environment"]["BUILDDIR"], "src", "bitcoind" + config["environment"]["EXEEXT"] ) fname_bitcoincli = os.path.join( config["environment"]["BUILDDIR"], "src", "bitcoin-cli" + config["environment"]["EXEEXT"] ) self.options.bitcoind = os.getenv("BITCOIND", default=fname_bitcoind) self.options.bitcoincli = os.getenv( "BITCOINCLI", default=fname_bitcoincli) self.options.emulator = config["environment"]["EMULATOR"] or None os.environ['PATH'] = config['environment']['BUILDDIR'] + os.pathsep + \ config['environment']['BUILDDIR'] + os.path.sep + "qt" + os.pathsep + \ os.environ['PATH'] # Set up temp directory and start logging if self.options.tmpdir: self.options.tmpdir = os.path.abspath(self.options.tmpdir) os.makedirs(self.options.tmpdir, exist_ok=False) else: self.options.tmpdir = tempfile.mkdtemp(prefix=TMPDIR_PREFIX) self._start_logging() # Seed the PRNG. Note that test runs are reproducible if and only if # a single thread accesses the PRNG. For more information, see # https://docs.python.org/3/library/random.html#notes-on-reproducibility. # The network thread shouldn't access random. If we need to change the # network thread to access randomness, it should instantiate its own # random.Random object. seed = self.options.randomseed if seed is None: seed = random.randrange(sys.maxsize) else: self.log.debug("User supplied random seed {}".format(seed)) random.seed(seed) self.log.debug("PRNG seed is: {}".format(seed)) self.log.debug('Setting up network thread') self.network_thread = NetworkThread() self.network_thread.start() if self.options.usecli: if not self.supports_cli: raise SkipTest( "--usecli specified but test does not support using CLI") self.skip_if_no_cli() self.skip_test_if_missing_module() self.setup_chain() self.setup_network() self.success = TestStatus.PASSED def shutdown(self): """Call this method to shut down the test framework object.""" if self.success == TestStatus.FAILED and self.options.pdbonfailure: print("Testcase failed. Attaching python debugger. Enter ? for help") pdb.set_trace() self.log.debug('Closing down network thread') self.network_thread.close() if not self.options.noshutdown: self.log.info("Stopping nodes") if self.nodes: self.stop_nodes() else: for node in self.nodes: node.cleanup_on_exit = False self.log.info( "Note: bitcoinds were not stopped and may still be running") should_clean_up = ( not self.options.nocleanup and not self.options.noshutdown and self.success != TestStatus.FAILED and not self.options.perf ) if should_clean_up: self.log.info("Cleaning up {} on exit".format(self.options.tmpdir)) cleanup_tree_on_exit = True elif self.options.perf: self.log.warning( "Not cleaning up dir {} due to perf data".format( self.options.tmpdir)) cleanup_tree_on_exit = False else: self.log.warning( "Not cleaning up dir {}".format(self.options.tmpdir)) cleanup_tree_on_exit = False if self.success == TestStatus.PASSED: self.log.info("Tests successful") exit_code = TEST_EXIT_PASSED elif self.success == TestStatus.SKIPPED: self.log.info("Test skipped") exit_code = TEST_EXIT_SKIPPED else: self.log.error( "Test failed. Test logging available at {}/test_framework.log".format(self.options.tmpdir)) self.log.error("") self.log.error("Hint: Call {} '{}' to consolidate all logs".format(os.path.normpath( os.path.dirname(os.path.realpath(__file__)) + "/../combine_logs.py"), self.options.tmpdir)) self.log.error("") self.log.error( "If this failure happened unexpectedly or intermittently, please" " file a bug and provide a link or upload of the combined log.") self.log.error(self.config['environment']['PACKAGE_BUGREPORT']) self.log.error("") exit_code = TEST_EXIT_FAILED # Logging.shutdown will not remove stream- and filehandlers, so we must # do it explicitly. Handlers are removed so the next test run can apply # different log handler settings. # See: https://docs.python.org/3/library/logging.html#logging.shutdown for h in list(self.log.handlers): h.flush() h.close() self.log.removeHandler(h) rpc_logger = logging.getLogger("BitcoinRPC") for h in list(rpc_logger.handlers): h.flush() rpc_logger.removeHandler(h) if cleanup_tree_on_exit: shutil.rmtree(self.options.tmpdir) self.nodes.clear() return exit_code # Methods to override in subclass test scripts. def set_test_params(self): """Tests must this method to change default values for number of nodes, topology, etc""" raise NotImplementedError def add_options(self, parser): """Override this method to add command-line options to the test""" pass def skip_test_if_missing_module(self): """Override this method to skip a test if a module is not compiled""" pass def setup_chain(self): """Override this method to customize blockchain setup""" self.log.info("Initializing test directory " + self.options.tmpdir) if self.setup_clean_chain: self._initialize_chain_clean() else: self._initialize_chain() def setup_network(self): """Override this method to customize test network topology""" self.setup_nodes() # Connect the nodes as a "chain". This allows us # to split the network between nodes 1 and 2 to get # two halves that can work on competing chains. # # Topology looks like this: # node0 <-- node1 <-- node2 <-- node3 # # If all nodes are in IBD (clean chain from genesis), node0 is assumed to be the source of blocks (miner). To # ensure block propagation, all nodes will establish outgoing connections toward node0. # See fPreferredDownload in net_processing. # # If further outbound connections are needed, they can be added at the beginning of the test with e.g. # self.connect_nodes(1, 2) for i in range(self.num_nodes - 1): self.connect_nodes(i + 1, i) self.sync_all() def setup_nodes(self): """Override this method to customize test node setup""" extra_args = None if hasattr(self, "extra_args"): extra_args = self.extra_args self.add_nodes(self.num_nodes, extra_args) self.start_nodes() self.import_deterministic_coinbase_privkeys() if not self.setup_clean_chain: for n in self.nodes: assert_equal(n.getblockchaininfo()["blocks"], 199) # To ensure that all nodes are out of IBD, the most recent block # must have a timestamp not too old (see IsInitialBlockDownload()). self.log.debug('Generate a block with current time') block_hash = self.nodes[0].generate(1)[0] block = self.nodes[0].getblock(blockhash=block_hash, verbosity=0) for n in self.nodes: n.submitblock(block) chain_info = n.getblockchaininfo() assert_equal(chain_info["blocks"], 200) assert_equal(chain_info["initialblockdownload"], False) def import_deterministic_coinbase_privkeys(self): for n in self.nodes: try: n.getwalletinfo() except JSONRPCException as e: assert str(e).startswith('Method not found') continue n.importprivkey( privkey=n.get_deterministic_priv_key().key, label='coinbase') def run_test(self): """Tests must override this method to define test logic""" raise NotImplementedError # Public helper methods. These can be accessed by the subclass test # scripts. def add_nodes(self, num_nodes: int, extra_args=None, *, host=None, binary=None): """Instantiate TestNode objects. Should only be called once after the nodes have been specified in set_test_params().""" if self.bind_to_localhost_only: extra_confs = [["bind=127.0.0.1"]] * num_nodes else: extra_confs = [[]] * num_nodes if extra_args is None: extra_args = [[]] * num_nodes if binary is None: binary = [self.options.bitcoind] * num_nodes assert_equal(len(extra_confs), num_nodes) assert_equal(len(extra_args), num_nodes) assert_equal(len(binary), num_nodes) for i in range(num_nodes): self.nodes.append(TestNode( i, get_datadir_path(self.options.tmpdir, i), chain=self.chain, host=host, rpc_port=rpc_port(i), p2p_port=p2p_port(i), timewait=self.rpc_timeout, timeout_factor=self.options.timeout_factor, bitcoind=binary[i], bitcoin_cli=self.options.bitcoincli, coverage_dir=self.options.coveragedir, cwd=self.options.tmpdir, extra_conf=extra_confs[i], extra_args=extra_args[i], use_cli=self.options.usecli, emulator=self.options.emulator, start_perf=self.options.perf, use_valgrind=self.options.valgrind, )) if self.options.axionactivation: self.nodes[i].extend_default_args( ["-axionactivationtime={}".format(TIMESTAMP_IN_THE_PAST)]) def start_node(self, i, *args, **kwargs): """Start a bitcoind""" node = self.nodes[i] node.start(*args, **kwargs) node.wait_for_rpc_connection() if self.options.coveragedir is not None: coverage.write_all_rpc_commands(self.options.coveragedir, node.rpc) def start_nodes(self, extra_args=None, *args, **kwargs): """Start multiple bitcoinds""" if extra_args is None: extra_args = [None] * self.num_nodes assert_equal(len(extra_args), self.num_nodes) try: for i, node in enumerate(self.nodes): node.start(extra_args[i], *args, **kwargs) for node in self.nodes: node.wait_for_rpc_connection() except BaseException: # If one node failed to start, stop the others self.stop_nodes() raise if self.options.coveragedir is not None: for node in self.nodes: coverage.write_all_rpc_commands( self.options.coveragedir, node.rpc) def stop_node(self, i, expected_stderr='', wait=0): """Stop a bitcoind test node""" self.nodes[i].stop_node(expected_stderr, wait=wait) def stop_nodes(self, wait=0): """Stop multiple bitcoind test nodes""" for node in self.nodes: # Issue RPC to stop nodes node.stop_node(wait=wait, wait_until_stopped=False) for node in self.nodes: # Wait for nodes to stop node.wait_until_stopped() def restart_node(self, i, extra_args=None): """Stop and start a test node""" self.stop_node(i) self.start_node(i, extra_args) def wait_for_node_exit(self, i, timeout): self.nodes[i].process.wait(timeout) def connect_nodes(self, a, b): connect_nodes(self.nodes[a], self.nodes[b]) def disconnect_nodes(self, a, b): disconnect_nodes(self.nodes[a], self.nodes[b]) def split_network(self): """ Split the network of four nodes into nodes 0/1 and 2/3. """ self.disconnect_nodes(1, 2) self.sync_all(self.nodes[:2]) self.sync_all(self.nodes[2:]) def join_network(self): """ Join the (previously split) network halves together. """ self.connect_nodes(1, 2) self.sync_all() def sync_blocks(self, nodes=None, wait=1, timeout=60): """ Wait until everybody has the same tip. sync_blocks needs to be called with an rpc_connections set that has least one node already synced to the latest, stable tip, otherwise there's a chance it might return before all nodes are stably synced. """ rpc_connections = nodes or self.nodes timeout = int(timeout * self.options.timeout_factor) stop_time = time.time() + timeout while time.time() <= stop_time: best_hash = [x.getbestblockhash() for x in rpc_connections] if best_hash.count(best_hash[0]) == len(rpc_connections): return # Check that each peer has at least one connection assert (all([len(x.getpeerinfo()) for x in rpc_connections])) time.sleep(wait) raise AssertionError("Block sync timed out after {}s:{}".format( timeout, "".join("\n {!r}".format(b) for b in best_hash), )) def sync_mempools(self, nodes=None, wait=1, timeout=60, flush_scheduler=True): """ Wait until everybody has the same transactions in their memory pools """ rpc_connections = nodes or self.nodes timeout = int(timeout * self.options.timeout_factor) stop_time = time.time() + timeout while time.time() <= stop_time: pool = [set(r.getrawmempool()) for r in rpc_connections] if pool.count(pool[0]) == len(rpc_connections): if flush_scheduler: for r in rpc_connections: r.syncwithvalidationinterfacequeue() return # Check that each peer has at least one connection assert (all([len(x.getpeerinfo()) for x in rpc_connections])) time.sleep(wait) raise AssertionError("Mempool sync timed out after {}s:{}".format( timeout, "".join("\n {!r}".format(m) for m in pool), )) def sync_all(self, nodes=None): self.sync_blocks(nodes) self.sync_mempools(nodes) # Private helper methods. These should not be accessed by the subclass # test scripts. def _start_logging(self): # Add logger and logging handlers self.log = logging.getLogger('TestFramework') self.log.setLevel(logging.DEBUG) # Create file handler to log all messages fh = logging.FileHandler( self.options.tmpdir + '/test_framework.log', encoding='utf-8') fh.setLevel(logging.DEBUG) # Create console handler to log messages to stderr. By default this # logs only error messages, but can be configured with --loglevel. ch = logging.StreamHandler(sys.stdout) # User can provide log level as a number or string (eg DEBUG). loglevel # was caught as a string, so try to convert it to an int ll = int(self.options.loglevel) if self.options.loglevel.isdigit( ) else self.options.loglevel.upper() ch.setLevel(ll) # Format logs the same as bitcoind's debug.log with microprecision (so # log files can be concatenated and sorted) formatter = logging.Formatter( fmt='%(asctime)s.%(msecs)03d000Z %(name)s (%(levelname)s): %(message)s', datefmt='%Y-%m-%dT%H:%M:%S') formatter.converter = time.gmtime fh.setFormatter(formatter) ch.setFormatter(formatter) # add the handlers to the logger self.log.addHandler(fh) self.log.addHandler(ch) if self.options.trace_rpc: rpc_logger = logging.getLogger("BitcoinRPC") rpc_logger.setLevel(logging.DEBUG) rpc_handler = logging.StreamHandler(sys.stdout) rpc_handler.setLevel(logging.DEBUG) rpc_logger.addHandler(rpc_handler) def _initialize_chain(self): """Initialize a pre-mined blockchain for use by the test. Create a cache of a 199-block-long chain Afterward, create num_nodes copies from the cache.""" # Use node 0 to create the cache for all other nodes CACHE_NODE_ID = 0 cache_node_dir = get_datadir_path(self.options.cachedir, CACHE_NODE_ID) assert self.num_nodes <= MAX_NODES if not os.path.isdir(cache_node_dir): self.log.debug( "Creating cache directory {}".format(cache_node_dir)) initialize_datadir( self.options.cachedir, CACHE_NODE_ID, self.chain) self.nodes.append( TestNode( CACHE_NODE_ID, cache_node_dir, chain=self.chain, extra_conf=["bind=127.0.0.1"], extra_args=['-disablewallet'], host=None, rpc_port=rpc_port(CACHE_NODE_ID), p2p_port=p2p_port(CACHE_NODE_ID), timewait=self.rpc_timeout, timeout_factor=self.options.timeout_factor, bitcoind=self.options.bitcoind, bitcoin_cli=self.options.bitcoincli, coverage_dir=None, cwd=self.options.tmpdir, emulator=self.options.emulator, )) if self.options.axionactivation: self.nodes[CACHE_NODE_ID].extend_default_args( ["-axionactivationtime={}".format(TIMESTAMP_IN_THE_PAST)]) self.start_node(CACHE_NODE_ID) cache_node = self.nodes[CACHE_NODE_ID] # Wait for RPC connections to be ready cache_node.wait_for_rpc_connection() # Set a time in the past, so that blocks don't end up in the future cache_node.setmocktime( cache_node.getblockheader( cache_node.getbestblockhash())['time']) # Create a 199-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. # The 4th node gets only 24 immature blocks so that the very last # block in the cache does not age too much (have an old tip age). # This is needed so that we are out of IBD when the test starts, # see the tip age check in IsInitialBlockDownload(). for i in range(8): cache_node.generatetoaddress( nblocks=25 if i != 7 else 24, address=TestNode.PRIV_KEYS[i % 4].address, ) assert_equal(cache_node.getblockchaininfo()["blocks"], 199) # Shut it down, and clean up cache directories: self.stop_nodes() self.nodes = [] def cache_path(*paths): return os.path.join(cache_node_dir, self.chain, *paths) # Remove empty wallets dir os.rmdir(cache_path('wallets')) for entry in os.listdir(cache_path()): # Only keep chainstate and blocks folder if entry not in ['chainstate', 'blocks']: os.remove(cache_path(entry)) for i in range(self.num_nodes): self.log.debug( "Copy cache directory {} to node {}".format( cache_node_dir, i)) to_dir = get_datadir_path(self.options.tmpdir, i) shutil.copytree(cache_node_dir, to_dir) # Overwrite port/rpcport in bitcoin.conf initialize_datadir(self.options.tmpdir, i, self.chain) def _initialize_chain_clean(self): """Initialize empty blockchain for use by the test. Create an empty blockchain and num_nodes wallets. Useful if a test case wants complete control over initialization.""" for i in range(self.num_nodes): initialize_datadir(self.options.tmpdir, i, self.chain) def skip_if_no_py3_zmq(self): """Attempt to import the zmq package and skip the test if the import fails.""" try: import zmq # noqa except ImportError: raise SkipTest("python3-zmq module not available.") def skip_if_no_bitcoind_zmq(self): """Skip the running test if bitcoind has not been compiled with zmq support.""" if not self.is_zmq_compiled(): raise SkipTest("bitcoind has not been built with zmq enabled.") def skip_if_no_wallet(self): """Skip the running test if wallet has not been compiled.""" if not self.is_wallet_compiled(): raise SkipTest("wallet has not been compiled.") def skip_if_no_wallet_tool(self): """Skip the running test if bitcoin-wallet has not been compiled.""" if not self.is_wallet_tool_compiled(): raise SkipTest("bitcoin-wallet has not been compiled") def skip_if_no_cli(self): """Skip the running test if bitcoin-cli has not been compiled.""" if not self.is_cli_compiled(): raise SkipTest("bitcoin-cli has not been compiled.") def is_cli_compiled(self): """Checks whether bitcoin-cli was compiled.""" return self.config["components"].getboolean("ENABLE_CLI") def is_wallet_compiled(self): """Checks whether the wallet module was compiled.""" return self.config["components"].getboolean("ENABLE_WALLET") def is_wallet_tool_compiled(self): """Checks whether bitcoin-wallet was compiled.""" return self.config["components"].getboolean("ENABLE_WALLET_TOOL") def is_zmq_compiled(self): """Checks whether the zmq module was compiled.""" return self.config["components"].getboolean("ENABLE_ZMQ")