diff --git a/src/blockdb.cpp b/src/blockdb.cpp --- a/src/blockdb.cpp +++ b/src/blockdb.cpp @@ -10,7 +10,10 @@ extern RecursiveMutex cs_main; FlatFileSeq BlockFileSeq() { - return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", BLOCKFILE_CHUNK_SIZE); + return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", + gArgs.GetBoolArg("-fastprune", false) + ? 0x4000 /* 16kb */ + : BLOCKFILE_CHUNK_SIZE); } FlatFileSeq UndoFileSeq() { diff --git a/src/chainparams.cpp b/src/chainparams.cpp --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -456,7 +456,7 @@ netMagic[2] = 0xbf; netMagic[3] = 0xfa; nDefaultPort = 18444; - nPruneAfterHeight = 1000; + nPruneAfterHeight = gArgs.GetBoolArg("-fastprune", false) ? 100 : 1000; m_assumed_blockchain_size = 0; m_assumed_chain_state_size = 0; diff --git a/src/index/base.cpp b/src/index/base.cpp --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -67,6 +67,48 @@ ::ChainActive(), locator); } m_synced = m_best_block_index.load() == ::ChainActive().Tip(); + if (!m_synced) { + bool prune_violation = false; + if (!m_best_block_index) { + // index is not built yet + // make sure we have all block data back to the genesis + const CBlockIndex *block = ::ChainActive().Tip(); + while (block->pprev && block->pprev->nStatus.hasData()) { + block = block->pprev; + } + prune_violation = block != ::ChainActive().Genesis(); + } + // in case the index has a best block set and is not fully synced + // check if we have the required blocks to continue building the index + else { + const CBlockIndex *block_to_test = m_best_block_index.load(); + if (!ChainActive().Contains(block_to_test)) { + // if the bestblock is not part of the mainchain, find the fork + // and make sure we have all data down to the fork + block_to_test = ::ChainActive().FindFork(block_to_test); + } + const CBlockIndex *block = ::ChainActive().Tip(); + prune_violation = true; + // check backwards from the tip if we have all block data until we + // reach the indexes bestblock + while (block_to_test && block && block->nStatus.hasData()) { + if (block_to_test == block) { + prune_violation = false; + break; + } + assert(block->pprev); + block = block->pprev; + } + } + if (prune_violation) { + // throw error and graceful shutdown if we can't build the index + FatalError("%s: %s best block of the index goes beyond pruned " + "data. Please disable the index or reindex (which will " + "download the whole blockchain again)", + __func__, GetName()); + return false; + } + } return true; } @@ -182,6 +224,10 @@ assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); // In the case of a reorg, ensure persisted block locator is not stale. + // Pruning has a minimum of 288 blocks-to-keep and getting the index + // out of sync may be possible but a users fault. + // In case we reorg beyond the pruned depth, ReadBlockFromDisk would + // throw and lead to a graceful shutdown m_best_block_index = new_tip; if (!Commit()) { // If commit fails, revert the best block index to avoid corruption. @@ -334,6 +380,7 @@ IndexSummary summary{}; summary.name = GetName(); summary.synced = m_synced; - summary.best_block_height = m_best_block_index.load()->nHeight; + summary.best_block_height = + m_best_block_index ? m_best_block_index.load()->nHeight : 0; return summary; } diff --git a/src/init.cpp b/src/init.cpp --- a/src/init.cpp +++ b/src/init.cpp @@ -443,6 +443,11 @@ "Specify directory to hold blocks subdirectory for *.dat " "files (default: )", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-fastprune", + "Use smaller block files and lower minimum prune height for " + "testing purposes", + ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, + OptionsCategory::DEBUG_TEST); #if defined(HAVE_SYSTEM) argsman.AddArg("-blocknotify=", "Execute command when the best block changes (%s in cmd is " @@ -1905,10 +1910,6 @@ if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { return InitError(_("Prune mode is incompatible with -txindex.")); } - if (!g_enabled_filter_types.empty()) { - return InitError( - _("Prune mode is incompatible with -blockfilterindex.")); - } } // -bind and -whitebind can't be set when not listening diff --git a/src/validation.h b/src/validation.h --- a/src/validation.h +++ b/src/validation.h @@ -564,7 +564,7 @@ */ void FindFilesToPrune(std::set &setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, - bool is_ibd); + int prune_height, bool is_ibd); public: BlockMap m_block_index GUARDED_BY(cs_main); diff --git a/src/validation.cpp b/src/validation.cpp --- a/src/validation.cpp +++ b/src/validation.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -2156,22 +2157,34 @@ { bool fFlushForPrune = false; bool fDoFullFlush = false; + CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(&m_mempool); LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { + // Make sure we don't prune above the blockfilterindexes + // bestblocks. Pruning is height-based. + int last_prune = m_chain.Height(); + ForEachBlockFilterIndex([&](BlockFilterIndex &index) { + last_prune = std::max( + 1, std::min(last_prune, + index.GetSummary().best_block_height)); + }); + if (nManualPruneHeight > 0) { LOG_TIME_MILLIS_WITH_CATEGORY( "find files to prune (manual)", BCLog::BENCH); m_blockman.FindFilesToPruneManual( - setFilesToPrune, nManualPruneHeight, m_chain.Height()); + setFilesToPrune, + std::min(last_prune, nManualPruneHeight), + m_chain.Height()); } else { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCH); m_blockman.FindFilesToPrune( setFilesToPrune, chainparams.PruneAfterHeight(), - m_chain.Height(), IsInitialBlockDownload()); + m_chain.Height(), last_prune, IsInitialBlockDownload()); fCheckForPruning = false; } if (!setFilesToPrune.empty()) { @@ -3687,7 +3700,9 @@ bool finalize_undo = false; if (!fKnown) { - while (vinfoBlockFile[nFile].nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { + while (vinfoBlockFile[nFile].nSize + nAddSize >= + (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ + : MAX_BLOCKFILE_SIZE)) { // when the undo file is keeping up with the block file, we want to // flush it explicitly when it is lagging behind (more blocks arrive // than are being connected), we let the undo block write case @@ -4651,7 +4666,8 @@ void BlockManager::FindFilesToPrune(std::set &setFilesToPrune, uint64_t nPruneAfterHeight, - int chain_tip_height, bool is_ibd) { + int chain_tip_height, int prune_height, + bool is_ibd) { LOCK2(cs_main, cs_LastBlockFile); if (chain_tip_height < 0 || nPruneTarget == 0) { return; @@ -4660,7 +4676,8 @@ return; } - unsigned int nLastBlockWeCanPrune = chain_tip_height - MIN_BLOCKS_TO_KEEP; + unsigned int nLastBlockWeCanPrune = std::min( + prune_height, chain_tip_height - static_cast(MIN_BLOCKS_TO_KEEP)); uint64_t nCurrentUsage = CalculateCurrentUsage(); // We don't check to prune until after we've allocated new space for files, // so we should leave a buffer under our target to account for another diff --git a/test/functional/feature_blockfilterindex_prune.py b/test/functional/feature_blockfilterindex_prune.py new file mode 100755 --- /dev/null +++ b/test/functional/feature_blockfilterindex_prune.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test blockfilterindex in conjunction with prune.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, +) + + +class FeatureBlockfilterindexPruneTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [["-fastprune", "-prune=1", "-blockfilterindex=1"]] + + def sync_index(self, height): + expected = { + 'basic block filter index': { + 'synced': True, + 'best_block_height': height + } + } + self.wait_until(lambda: self.nodes[0].getindexinfo() == expected) + + def run_test(self): + node = self.nodes[0] + + self.log.info("check if we can access a blockfilter when pruning is " + "enabled but no blocks are actually pruned") + self.sync_index(200) + assert_greater_than( + len(node.getblockfilter(node.getbestblockhash())['filter']), + 0) + node.generate(500) + self.sync_index(height=700) + + self.log.info("prune some blocks") + pruneheight = node.pruneblockchain(400) + # The difference in number of blocks per block file between Bitcoin ABC + # and Bitcoin Core is caused by additional witness data in coinbase + # transactions for core. + assert_equal(pruneheight, 346) + + self.log.info("check if we can access the tips blockfilter when we have" + " pruned some blocks") + assert_greater_than( + len(node.getblockfilter(node.getbestblockhash())['filter']), + 0) + + self.log.info("check if we can access the blockfilter of a pruned " + "block") + assert_greater_than( + len(node.getblockfilter(node.getblockhash(2))['filter']), + 0) + + # mine and sync index up to a height that will later be the pruneheight + node.generate(338) + # Backport note: 3 blk?????.dat file with 346 blocks each makes 1038. + self.sync_index(height=1038) + + self.log.info("start node without blockfilterindex") + self.restart_node(0, extra_args=["-fastprune", "-prune=1"]) + + self.log.info("make sure accessing the blockfilters throws an error") + assert_raises_rpc_error( + -1, "Index is not enabled for filtertype basic", + node.getblockfilter, node.getblockhash(2)) + node.generate(462) + + self.log.info("prune exactly up to the blockfilterindexes best block " + "while blockfilters are disabled") + pruneheight_2 = self.nodes[0].pruneblockchain(1040) + assert_equal(pruneheight_2, 1038) + self.restart_node( + 0, extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1"]) + self.log.info("make sure that we can continue with the partially synced" + " index after having pruned up to the index height") + self.sync_index(height=1500) + + self.log.info("prune below the blockfilterindexes best block while " + "blockfilters are disabled") + self.restart_node( + 0, + extra_args=["-fastprune", "-prune=1"]) + node.generate(1000) + pruneheight_3 = self.nodes[0].pruneblockchain(2000) + assert_greater_than(pruneheight_3, pruneheight_2) + self.stop_node(0) + + self.log.info("make sure we get an init error when starting the node " + "again with block filters") + with node.assert_debug_log( + ["basic block filter index best block of the index goes beyond " + "pruned data. Please disable the index or reindex (which will " + "download the whole blockchain again)"]): + node.assert_start_raises_init_error( + extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1"]) + self.log.info("make sure the node starts again with the -reindex arg") + self.start_node( + 0, + extra_args=["-fastprune", "-prune=1", "-blockfilterindex", + "-reindex"]) + + +if __name__ == '__main__': + FeatureBlockfilterindexPruneTest().main() diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -14,6 +14,7 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "index/txindex -> validation -> index/txindex" + "index/blockfilterindex -> validation -> index/blockfilterindex" "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" "qt/bitcoingui -> qt/walletframe -> qt/bitcoingui" "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel"