Page MenuHomePhabricator
Differential D4561 Diff 14465 test/functional/feature_pruning.py

Changeset View

Standalone View

View Options

test/functional/feature_pruning.py

Show All 26 Lines
# Rescans start at the earliest block up to 2 hours before a key timestamp, so # Rescans start at the earliest block up to 2 hours before a key timestamp, so
# the manual prune RPC avoids pruning blocks in the same window to be # the manual prune RPC avoids pruning blocks in the same window to be
# compatible with pruning based on key creation time. # compatible with pruning based on key creation time.
TIMESTAMP_WINDOW = 2 * 60 * 60 TIMESTAMP_WINDOW = 2 * 60 * 60
def calc_usage(blockdir): def calc_usage(blockdir):
return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.) return sum(os.path.getsize(blockdir + f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.)
class PruneTest(BitcoinTestFramework): class PruneTest(BitcoinTestFramework):
def set_test_params(self): def set_test_params(self):
self.setup_clean_chain = True self.setup_clean_chain = True
self.num_nodes = 6 self.num_nodes = 6
self.rpc_timeout = 900 self.rpc_timeout = 900
Show All 18 Linesclass PruneTest(BitcoinTestFramework):
def setup_network(self): def setup_network(self):
self.setup_nodes() self.setup_nodes()
self.prunedir = os.path.join( self.prunedir = os.path.join(
self.nodes[2].datadir, 'regtest', 'blocks', '') self.nodes[2].datadir, 'regtest', 'blocks', '')
connect_nodes(self.nodes[0], self.nodes[1]) connect_nodes(self.nodes[0], self.nodes[1])
connect_nodes(self.nodes[1], self.nodes[2]) connect_nodes(self.nodes[1], self.nodes[2])
connect_nodes(self.nodes[2], self.nodes[0]) connect_nodes(self.nodes[0], self.nodes[2])
connect_nodes(self.nodes[0], self.nodes[3]) connect_nodes(self.nodes[0], self.nodes[3])
connect_nodes(self.nodes[0], self.nodes[4]) connect_nodes(self.nodes[0], self.nodes[4])
sync_blocks(self.nodes[0:5]) sync_blocks(self.nodes[0:5])
def setup_nodes(self): def setup_nodes(self):
self.add_nodes(self.num_nodes, self.extra_args) self.add_nodes(self.num_nodes, self.extra_args)
self.start_nodes() self.start_nodes()
def create_big_chain(self): def create_big_chain(self):
# Start by creating some coinbases we can spend later # Start by creating some coinbases we can spend later
self.nodes[1].generate(200) self.nodes[1].generate(200)
sync_blocks(self.nodes[0:2]) sync_blocks(self.nodes[0:2])
self.nodes[0].generate(150) self.nodes[0].generate(150)
# Then mine enough full blocks to create more than 550MiB of data # Then mine enough full blocks to create more than 550MiB of data
for i in range(645): for i in range(645):
mine_big_block(self.nodes[0], self.utxo_cache_0) mine_big_block(self.nodes[0], self.utxo_cache_0)
sync_blocks(self.nodes[0:5]) sync_blocks(self.nodes[0:5])
def test_height_min(self): def test_height_min(self):
if not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")): assert os.path.isfile(os.path.join(
raise AssertionError("blk00000.dat is missing, pruning too early") self.prunedir, "blk00000.dat")), "blk00000.dat is missing, pruning too early"
self.log.info("Success") self.log.info("Success")
self.log.info("Though we're already using more than 550MiB, current usage: {}".format( self.log.info("Though we're already using more than 550MiB, current usage: {}".format(
calc_usage(self.prunedir))) calc_usage(self.prunedir)))
self.log.info( self.log.info(
"Mining 25 more blocks should cause the first block file to be pruned") "Mining 25 more blocks should cause the first block file to be pruned")
# Pruning doesn't run until we're allocating another chunk, 20 full # Pruning doesn't run until we're allocating another chunk, 20 full
# blocks past the height cutoff will ensure this # blocks past the height cutoff will ensure this
for i in range(25): for i in range(25):
mine_big_block(self.nodes[0], self.utxo_cache_0) mine_big_block(self.nodes[0], self.utxo_cache_0)
# Wait for blk00000.dat to be pruned # Wait for blk00000.dat to be pruned
wait_until(lambda: not os.path.isfile( wait_until(lambda: not os.path.isfile(
os.path.join(self.prunedir, "blk00000.dat")), timeout=30) os.path.join(self.prunedir, "blk00000.dat")), timeout=30)
self.log.info("Success") self.log.info("Success")
usage = calc_usage(self.prunedir) usage = calc_usage(self.prunedir)
self.log.info("Usage should be below target: {}".format(usage)) self.log.info("Usage should be below target: {}".format(usage))
if (usage > 550): assert_greater_than(550, usage)
raise AssertionError("Pruning target not being met")
def create_chain_with_staleblocks(self): def create_chain_with_staleblocks(self):
# Create stale blocks in manageable sized chunks # Create stale blocks in manageable sized chunks
self.log.info( self.log.info(
"Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds") "Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds")
for j in range(12): for j in range(12):
# Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain
Show All 14 Lines def create_chain_with_staleblocks(self):
self.nodes[1].generate(1) self.nodes[1].generate(1)
# Reorg back with 25 block chain from node 0 # Reorg back with 25 block chain from node 0
for i in range(25): for i in range(25):
mine_big_block(self.nodes[0], self.utxo_cache_0) mine_big_block(self.nodes[0], self.utxo_cache_0)
# Create connections in the order so both nodes can see the reorg # Create connections in the order so both nodes can see the reorg
# at the same time # at the same time
connect_nodes(self.nodes[1], self.nodes[0]) connect_nodes(self.nodes[0], self.nodes[1])
connect_nodes(self.nodes[2], self.nodes[0]) connect_nodes(self.nodes[0], self.nodes[2])
sync_blocks(self.nodes[0:3]) sync_blocks(self.nodes[0:3])
self.log.info("Usage can be over target because of high stale rate: {}".format( self.log.info("Usage can be over target because of high stale rate: {}".format(
calc_usage(self.prunedir))) calc_usage(self.prunedir)))
def reorg_test(self): def reorg_test(self):
# Node 1 will mine a 300 block chain starting 287 blocks back from Node # Node 1 will mine a 300 block chain starting 287 blocks back from Node
# 0 and Node 2's tip. This will cause Node 2 to do a reorg requiring # 0 and Node 2's tip. This will cause Node 2 to do a reorg requiring
# 288 blocks of undo data to the reorg_test chain. Reboot node 1 to # 288 blocks of undo data to the reorg_test chain. Reboot node 1 to
# clear its mempool (hopefully make the invalidate faster). Lower the # clear its mempool (hopefully make the invalidate faster). Lower the
# block max size so we don't keep mining all our big mempool # block max size so we don't keep mining all our big mempool
# transactions (from disconnected blocks) # transactions (from disconnected blocks)
self.stop_node(1) self.stop_node(1)
self.start_node(1, extra_args=[ self.start_node(1, extra_args=[
"-maxreceivebuffer=20000", "-blockmaxsize=5000", "-checkblocks=5", "-maxreceivebuffer=20000", "-blockmaxsize=5000", "-checkblocks=5",
"-noparkdeepreorg", "-maxreorgdepth=-1"]) "-noparkdeepreorg", "-maxreorgdepth=-1"])
height = self.nodes[1].getblockcount() height = self.nodes[1].getblockcount()
self.log.info("Current block height: {}".format(height)) self.log.info("Current block height: {}".format(height))
invalidheight = height - 287 self.forkheight = height - 287
badhash = self.nodes[1].getblockhash(invalidheight) self.forkhash = self.nodes[1].getblockhash(self.forkheight)
self.log.info("Invalidating block {} at height {}".format( self.log.info("Invalidating block {} at height {}".format(
badhash, invalidheight)) self.forkhash, self.forkheight))
self.nodes[1].invalidateblock(badhash) self.nodes[1].invalidateblock(self.forkhash)
# We've now switched to our previously mined-24 block fork on node 1, but that's not what we want. # We've now switched to our previously mined-24 block fork on node 1, but that's not what we want.
# So invalidate that fork as well, until we're on the same chain as # So invalidate that fork as well, until we're on the same chain as
# node 0/2 (but at an ancestor 288 blocks ago) # node 0/2 (but at an ancestor 288 blocks ago)
mainchainhash = self.nodes[0].getblockhash(invalidheight - 1) mainchainhash = self.nodes[0].getblockhash(self.forkheight - 1)
curhash = self.nodes[1].getblockhash(invalidheight - 1) curhash = self.nodes[1].getblockhash(self.forkheight - 1)
while curhash != mainchainhash: while curhash != mainchainhash:
self.nodes[1].invalidateblock(curhash) self.nodes[1].invalidateblock(curhash)
curhash = self.nodes[1].getblockhash(invalidheight - 1) curhash = self.nodes[1].getblockhash(self.forkheight - 1)
assert self.nodes[1].getblockcount() == invalidheight - 1 assert self.nodes[1].getblockcount() == self.forkheight - 1
self.log.info("New best height: {}".format( self.log.info("New best height: {}".format(
self.nodes[1].getblockcount())) self.nodes[1].getblockcount()))
# Reboot node1 to clear those giant tx's from mempool # Reboot node1 to clear those giant tx's from mempool
self.stop_node(1) self.stop_node(1)
self.start_node(1, extra_args=[ self.start_node(1, extra_args=[
"-maxreceivebuffer=20000", "-blockmaxsize=5000", "-checkblocks=5", "-maxreceivebuffer=20000", "-blockmaxsize=5000", "-checkblocks=5",
"-noparkdeepreorg", "-maxreorgdepth=-1"]) "-noparkdeepreorg", "-maxreorgdepth=-1"])
self.log.info("Generating new longer chain of 300 more blocks") self.log.info("Generating new longer chain of 300 more blocks")
self.nodes[1].generate(300) self.nodes[1].generate(300)
self.log.info("Reconnect nodes") self.log.info("Reconnect nodes")
connect_nodes(self.nodes[0], self.nodes[1]) connect_nodes(self.nodes[0], self.nodes[1])
connect_nodes(self.nodes[2], self.nodes[1]) connect_nodes(self.nodes[1], self.nodes[2])
sync_blocks(self.nodes[0:3], timeout=120) sync_blocks(self.nodes[0:3], timeout=120)
self.log.info("Verify height on node 2: {}".format( self.log.info("Verify height on node 2: {}".format(
self.nodes[2].getblockcount())) self.nodes[2].getblockcount()))
self.log.info("Usage possibly still high bc of stale blocks in block files: {}".format( self.log.info("Usage possibly still high because of stale blocks in block files: {}".format(
calc_usage(self.prunedir))) calc_usage(self.prunedir)))
self.log.info( self.log.info("Mine 220 more blocks so we have requisite history")
"Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)")
# Get node0's wallet transactions back in its mempool, to avoid the # Get node0's wallet transactions back in its mempool, to avoid the
# mined blocks from being too small. # mined blocks from being too small.
self.nodes[0].resendwallettransactions() self.nodes[0].resendwallettransactions()
for i in range(22): for i in range(22):
# This can be slow, so do this in multiple RPC calls to avoid # This can be slow, so do this in multiple RPC calls to avoid
# RPC timeouts. # RPC timeouts.
# node 0 has many large tx's in its mempool from the disconnects # node 0 has many large tx's in its mempool from the disconnects
self.nodes[0].generate(10) self.nodes[0].generate(10)
sync_blocks(self.nodes[0:3], timeout=300) sync_blocks(self.nodes[0:3], timeout=300)
usage = calc_usage(self.prunedir) usage = calc_usage(self.prunedir)
self.log.info("Usage should be below target: {}".format(usage)) self.log.info("Usage should be below target: {}".format(usage))
if (usage > 550): assert_greater_than(550, usage)
raise AssertionError("Pruning target not being met")
return invalidheight, badhash
def reorg_back(self): def reorg_back(self):
# Verify that a block on the old main chain fork has been pruned away # Verify that a block on the old main chain fork has been pruned away
assert_raises_rpc_error( assert_raises_rpc_error(
-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash) -1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash)
self.log.info( self.log.info(
"Will need to redownload block {}".format(self.forkheight)) "Will need to redownload block {}".format(self.forkheight))
Show All 20 Lines def reorg_back(self):
# 288 blocks of the fork point so it will preserve its ability to # 288 blocks of the fork point so it will preserve its ability to
# reorg. # reorg.
if self.nodes[2].getblockcount() < self.mainchainheight: if self.nodes[2].getblockcount() < self.mainchainheight:
blocks_to_mine = first_reorg_height + 1 - self.mainchainheight blocks_to_mine = first_reorg_height + 1 - self.mainchainheight
self.log.info( self.log.info(
"Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: {}".format( "Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: {}".format(
blocks_to_mine)) blocks_to_mine))
self.nodes[0].invalidateblock(curchainhash) self.nodes[0].invalidateblock(curchainhash)
assert self.nodes[0].getblockcount() == self.mainchainheight assert_equal(self.nodes[0].getblockcount(), self.mainchainheight)
assert self.nodes[0].getbestblockhash() == self.mainchainhash2 assert_equal(self.nodes[0].getbestblockhash(), self.mainchainhash2)
goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1] goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1]
goalbestheight = first_reorg_height + 1 goalbestheight = first_reorg_height + 1
self.log.info( self.log.info(
"Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") "Verify node 2 reorged back to the main chain, some blocks of which it had to redownload")
# Wait for Node 2 to reorg to proper height # Wait for Node 2 to reorg to proper height
wait_until(lambda: self.nodes[2].getblockcount( wait_until(lambda: self.nodes[2].getblockcount(
) >= goalbestheight, timeout=900) ) >= goalbestheight, timeout=900)
assert self.nodes[2].getbestblockhash() == goalbesthash assert_equal(self.nodes[2].getbestblockhash(), goalbesthash)
# Verify we can now have the data for a block previously pruned # Verify we can now have the data for a block previously pruned
assert self.nodes[2].getblock( assert_equal(self.nodes[2].getblock(
self.forkhash)["height"] == self.forkheight self.forkhash)["height"], self.forkheight)
def manual_test(self, node_number, use_timestamp): def manual_test(self, node_number, use_timestamp):
# at this point, node has 995 blocks and has not yet run in prune mode # at this point, node has 995 blocks and has not yet run in prune mode
self.start_node(node_number) self.start_node(node_number)
node = self.nodes[node_number] node = self.nodes[node_number]
assert_equal(node.getblockcount(), 995) assert_equal(node.getblockcount(), 995)
assert_raises_rpc_error(-1, "not in prune mode", assert_raises_rpc_error(-1, "not in prune mode",
node.pruneblockchain, 500) node.pruneblockchain, 500)
Show All 37 Lines def manual_test(self, node_number, use_timestamp):
node.generate(6) node.generate(6)
assert_equal(node.getblockchaininfo()["blocks"], 1001) assert_equal(node.getblockchaininfo()["blocks"], 1001)
# negative heights should raise an exception # negative heights should raise an exception
assert_raises_rpc_error(-8, "Negative", node.pruneblockchain, -10) assert_raises_rpc_error(-8, "Negative", node.pruneblockchain, -10)
# height=100 too low to prune first block file so this is a no-op # height=100 too low to prune first block file so this is a no-op
prune(100) prune(100)
if not has_block(0): assert has_block(
raise AssertionError( 0), "blk00000.dat is missing when should still be there"
"blk00000.dat is missing when should still be there")
# Does nothing # Does nothing
node.pruneblockchain(height(0)) node.pruneblockchain(height(0))
if not has_block(0): assert has_block(
raise AssertionError( 0), "blk00000.dat is missing when should still be there"
"blk00000.dat is missing when should still be there")
# height=500 should prune first file # height=500 should prune first file
prune(500) prune(500)
if has_block(0): assert not has_block(
raise AssertionError( 0), "blk00000.dat is still there, should be pruned by now"
"blk00000.dat is still there, should be pruned by now") assert has_block(
if not has_block(1): 1), "blk00001.dat is missing when should still be there"
raise AssertionError(
"blk00001.dat is missing when should still be there")
# height=650 should prune second file # height=650 should prune second file
prune(650) prune(650)
if has_block(1): assert not has_block(
raise AssertionError( 1), "blk00001.dat is still there, should be pruned by now"
"blk00001.dat is still there, should be pruned by now")
# height=1000 should not prune anything more, because tip-288 is in # height=1000 should not prune anything more, because tip-288 is in
# blk00002.dat. # blk00002.dat.
prune(1000, 1001 - MIN_BLOCKS_TO_KEEP) prune(1000, 1001 - MIN_BLOCKS_TO_KEEP)
if not has_block(2): assert has_block(
raise AssertionError( 2), "blk00002.dat is still there, should be pruned by now"
"blk00002.dat is still there, should be pruned by now")
# advance the tip so blk00002.dat and blk00003.dat can be pruned (the # advance the tip so blk00002.dat and blk00003.dat can be pruned (the
# last 288 blocks should now be in blk00004.dat) # last 288 blocks should now be in blk00004.dat)
node.generate(288) node.generate(288)
prune(1000) prune(1000)
if has_block(2): assert not has_block(
raise AssertionError( 2), "blk00002.dat is still there, should be pruned by now"
"blk00002.dat is still there, should be pruned by now") assert not has_block(
if has_block(3): 3), "blk00003.dat is still there, should be pruned by now"
raise AssertionError(
"blk00003.dat is still there, should be pruned by now")
# stop node, start back up with auto-prune at 550MB, make sure still # stop node, start back up with auto-prune at 550MB, make sure still
# runs # runs
self.stop_node(node_number) self.stop_node(node_number)
self.start_node(node_number, extra_args=["-prune=550"]) self.start_node(node_number, extra_args=["-prune=550"])
self.log.info("Success") self.log.info("Success")
def wallet_test(self): def wallet_test(self):
# check that the pruning node's wallet is still in good shape # check that the pruning node's wallet is still in good shape
self.log.info("Stop and start pruning node to trigger wallet rescan") self.log.info("Stop and start pruning node to trigger wallet rescan")
self.stop_node(2) self.stop_node(2)
self.start_node( self.start_node(
2, extra_args=["-prune=550", "-noparkdeepreorg", "-maxreorgdepth=-1"]) 2, extra_args=["-prune=550", "-noparkdeepreorg", "-maxreorgdepth=-1"])
self.log.info("Success") self.log.info("Success")
# check that wallet loads successfully when restarting a pruned node after IBD. # check that wallet loads successfully when restarting a pruned node after IBD.
# this was reported to fail in #7494. # this was reported to fail in #7494.
self.log.info("Syncing node 5 to test wallet") self.log.info("Syncing node 5 to test wallet")
connect_nodes(self.nodes[0], self.nodes[5]) connect_nodes(self.nodes[0], self.nodes[5])
nds = [self.nodes[0], self.nodes[5]] nds = [self.nodes[0], self.nodes[5]]
sync_blocks(nds, wait=5, timeout=300) sync_blocks(nds, wait=5, timeout=300)
self.stop_node(5) # stop and start to trigger rescan # Stop and start to trigger rescan
self.stop_node(5)
self.start_node( self.start_node(
5, extra_args=["-prune=550", "-noparkdeepreorg", "-maxreorgdepth=-1"]) 5, extra_args=["-prune=550", "-noparkdeepreorg", "-maxreorgdepth=-1"])
self.log.info("Success") self.log.info("Success")
def run_test(self): def run_test(self):
self.log.info( self.log.info(
"Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)") "Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)")
self.log.info("Mining a big blockchain of 995 blocks") self.log.info("Mining a big blockchain of 995 blocks")
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines def run_test(self):
# \ \ \ # \ \ \
# +...+(1044) &.. $...$(1319) # +...+(1044) &.. $...$(1319)
# Save some current chain state for later use # Save some current chain state for later use
self.mainchainheight = self.nodes[2].getblockcount() # 1320 self.mainchainheight = self.nodes[2].getblockcount() # 1320
self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight) self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight)
self.log.info("Check that we can survive a 288 block reorg still") self.log.info("Check that we can survive a 288 block reorg still")
(self.forkheight, self.forkhash) = self.reorg_test() # (1033, ) self.reorg_test() # (1033, )
# Now create a 288 block reorg by mining a longer chain on N1 # Now create a 288 block reorg by mining a longer chain on N1
# First disconnect N1 # First disconnect N1
# Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain # Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain
# N1 **...*(1020) **...**(1032)X.. # N1 **...*(1020) **...**(1032)X..
# \ # \
# ++...+(1031)X.. # ++...+(1031)X..
# #
# Now mine 300 more blocks on N1 # Now mine 300 more blocks on N1
▲ Show 20 Lines • Show All 58 Lines • Show Last 20 Lines