diff --git a/src/node/blockmanager_args.cpp b/src/node/blockmanager_args.cpp index 26adb4314..72fe916c7 100644 --- a/src/node/blockmanager_args.cpp +++ b/src/node/blockmanager_args.cpp @@ -1,34 +1,34 @@ // Copyright (c) 2023 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include namespace node { std::optional ApplyArgsManOptions(const ArgsManager &args, BlockManager::Options &opts) { // block pruning; get the amount of disk space (in MiB) to allot for block & // undo files int64_t nPruneArg{args.GetIntArg("-prune", opts.prune_target)}; if (nPruneArg < 0) { return _("Prune cannot be configured with a negative value."); } uint64_t nPruneTarget{uint64_t(nPruneArg) * 1024 * 1024}; // manual pruning: -prune=1 if (nPruneArg == 1) { - nPruneTarget = std::numeric_limits::max(); + nPruneTarget = BlockManager::PRUNE_TARGET_MANUAL; } else if (nPruneTarget) { if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) { return strprintf(_("Prune configured below the minimum of %d MiB. " "Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024); } } opts.prune_target = nPruneTarget; return std::nullopt; } } // namespace node diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 4aa66810a..0ecad4db3 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -1,257 +1,259 @@ // Copyright (c) 2011-2021 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_NODE_BLOCKSTORAGE_H #define BITCOIN_NODE_BLOCKSTORAGE_H #include #include #include #include #include #include #include // For CMessageHeader::MessageStartChars #include #include class ArgsManager; class BlockValidationState; class CBlock; class CBlockFileInfo; class CBlockHeader; class CBlockUndo; class CChain; class CChainParams; class CTxUndo; class Chainstate; class ChainstateManager; struct CCheckpointData; class Config; struct FlatFilePos; namespace Consensus { struct Params; } namespace node { static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ static constexpr unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB /** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** Size of header written by WriteBlockToDisk before a serialized CBlock */ static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = CMessageHeader::MESSAGE_START_SIZE + sizeof(unsigned int); extern std::atomic_bool fReindex; // Because validation code takes pointers to the map's CBlockIndex objects, if // we ever switch to another associative container, we need to either use a // container that has stable addressing (true of all std associative // containers), or make the key a `std::unique_ptr` using BlockMap = std::unordered_map; /** * Maintains a tree of blocks (stored in `m_block_index`) which is consulted * to determine where the most-work tip is. * * This data is used mostly in `Chainstate` - information about, e.g., * candidate tips is not maintained here. */ class BlockManager { friend Chainstate; friend ChainstateManager; private: /** * Load the blocktree off disk and into memory. Populate certain metadata * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as * peripheral collections like m_dirty_blockindex. */ bool LoadBlockIndex(const Consensus::Params &consensus_params) EXCLUSIVE_LOCKS_REQUIRED(cs_main); void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); void FlushUndoFile(int block_file, bool finalize = false); bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int nHeight, CChain &active_chain, uint64_t nTime, bool fKnown); bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); /** * Calculate the block/rev files to delete based on height specified * by user with RPC command pruneblockchain */ void FindFilesToPruneManual(std::set &setFilesToPrune, int nManualPruneHeight, int chain_tip_height); /** * Prune block and undo files (blk???.dat and undo???.dat) so that the disk * space used is less than a user-defined target. The user sets the target * (in MB) on the command line or in config file. This will be run on * startup and whenever new space is allocated in a block or undo file, * staying below the target. Changing back to unpruned requires a reindex * (which in this case means the blockchain must be re-downloaded.) * * Pruning functions are called from FlushStateToDisk when the * m_check_for_pruning flag has been set. Block and undo files are deleted * in lock-step (when blk00003.dat is deleted, so is rev00003.dat.) Pruning * cannot take place until the longest chain is at least a certain length * (CChainParams::nPruneAfterHeight). Pruning will never delete a block * within a defined distance (currently 288) from the active chain's tip. * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any * blocks that were stored in the deleted files. A db flag records the fact * that at least some block files have been pruned. * * @param[out] setFilesToPrune The set of file indices that can be * unlinked will be returned */ void FindFilesToPrune(std::set &setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd); RecursiveMutex cs_LastBlockFile; std::vector m_blockfile_info; int m_last_blockfile = 0; /** * Global flag to indicate we should check to see if there are * block/undo files that should be deleted. Set on startup * or if we allocate more file space when we're in prune mode */ bool m_check_for_pruning = false; const bool m_prune_mode; /** Dirty block index entries. */ std::set m_dirty_blockindex; /** Dirty block file entries. */ std::set m_dirty_fileinfo; const kernel::BlockManagerOpts m_opts; public: using Options = kernel::BlockManagerOpts; explicit BlockManager(Options opts) : m_prune_mode{opts.prune_target > 0}, m_opts{std::move(opts)} {}; std::atomic m_importing{false}; BlockMap m_block_index GUARDED_BY(cs_main); std::vector GetAllBlockIndices() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** * All pairs A->B, where A (or one of its ancestors) misses transactions, * but B has transactions. Pruned nodes may have entries where B is missing * data. */ std::multimap m_blocks_unlinked; std::unique_ptr m_block_tree_db GUARDED_BY(::cs_main); bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); bool LoadBlockIndexDB(const Consensus::Params &consensus_params) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); CBlockIndex *AddToBlockIndex(const CBlockHeader &block, CBlockIndex *&best_header) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Create a new block index entry for a given block hash */ CBlockIndex *InsertBlockIndex(const BlockHash &hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); //! Mark one block file as pruned (modify associated database entries) void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main); CBlockIndex *LookupBlockIndex(const BlockHash &hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); const CBlockIndex *LookupBlockIndex(const BlockHash &hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Get block file info entry for one block file */ CBlockFileInfo *GetBlockFileInfo(size_t n); bool WriteUndoDataForBlock(const CBlockUndo &blockundo, BlockValidationState &state, CBlockIndex *pindex, const CChainParams &chainparams) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** * Store block on disk. If dbp is not nullptr, then it provides the known * position of the block within a block file on disk. */ FlatFilePos SaveBlockToDisk(const CBlock &block, int nHeight, CChain &active_chain, const CChainParams &chainparams, const FlatFilePos *dbp); /** Whether running in -prune mode. */ [[nodiscard]] bool IsPruneMode() const { return m_prune_mode; } /** Attempt to stay below this number of bytes of block files. */ [[nodiscard]] uint64_t GetPruneTarget() const { return m_opts.prune_target; } + static constexpr auto PRUNE_TARGET_MANUAL{ + std::numeric_limits::max()}; [[nodiscard]] bool LoadingBlocks() const { return m_importing || fReindex; } /** * Calculate the amount of disk space the block & undo files currently use */ uint64_t CalculateCurrentUsage(); //! Returns last CBlockIndex* that is a checkpoint const CBlockIndex *GetLastCheckpoint(const CCheckpointData &data) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** True if any block files have ever been pruned. */ bool m_have_pruned = false; //! Check whether the block associated with this index entry is pruned or //! not. bool IsBlockPruned(const CBlockIndex *pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); }; //! Find the first block that is not pruned const CBlockIndex *GetFirstStoredBlock(const CBlockIndex *start_block) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); void CleanupBlockRevFiles(); /** Open a block file (blk?????.dat) */ FILE *OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); /** Translation to a filesystem path. */ fs::path GetBlockPosFilename(const FlatFilePos &pos); /** * Actually unlink the specified files */ void UnlinkPrunedFiles(const std::set &setFilesToPrune); /** Functions for disk access for blocks */ bool ReadBlockFromDisk(CBlock &block, const FlatFilePos &pos, const Consensus::Params &consensusParams); bool ReadBlockFromDisk(CBlock &block, const CBlockIndex *pindex, const Consensus::Params &consensusParams); bool UndoReadFromDisk(CBlockUndo &blockundo, const CBlockIndex *pindex); /** Functions for disk access for txs */ bool ReadTxFromDisk(CMutableTransaction &tx, const FlatFilePos &pos); bool ReadTxUndoFromDisk(CTxUndo &tx, const FlatFilePos &pos); void ThreadImport(ChainstateManager &chainman, std::vector vImportFiles, const ArgsManager &args, const fs::path &mempool_path); } // namespace node #endif // BITCOIN_NODE_BLOCKSTORAGE_H diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index b31250a20..d5a27b61c 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -1,305 +1,305 @@ // Copyright (c) 2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include namespace node { // Complete initialization of chainstates after the initial call has been made // to ChainstateManager::InitializeChainstate(). static ChainstateLoadResult CompleteChainstateInitialization( ChainstateManager &chainman, const CacheSizes &cache_sizes, const ChainstateLoadOptions &options) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { auto &pblocktree{chainman.m_blockman.m_block_tree_db}; // new CBlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: pblocktree.reset(); pblocktree.reset(new CBlockTreeDB(cache_sizes.block_tree_db, options.block_tree_db_in_memory, options.reindex)); if (options.reindex) { pblocktree->WriteReindexing(true); // If we're reindexing in prune mode, wipe away unusable block // files and all undo data files if (options.prune) { CleanupBlockRevFiles(); } } // If necessary, upgrade from older database format. // This is a no-op if we cleared the block tree db with -reindex // or -reindex-chainstate if (!pblocktree->Upgrade()) { return {ChainstateLoadStatus::FAILURE, _("Error upgrading block index database")}; } if (options.check_interrupt && options.check_interrupt()) { return {ChainstateLoadStatus::INTERRUPTED, {}}; } // LoadBlockIndex will load m_have_pruned if we've ever removed a // block file from disk. // Note that it also sets fReindex global based on the disk flag! // From here on, fReindex and options.reindex values may be different! if (!chainman.LoadBlockIndex()) { if (options.check_interrupt && options.check_interrupt()) { return {ChainstateLoadStatus::INTERRUPTED, {}}; } return {ChainstateLoadStatus::FAILURE, _("Error loading block database")}; } if (!chainman.BlockIndex().empty() && !chainman.m_blockman.LookupBlockIndex( chainman.GetConsensus().hashGenesisBlock)) { // If the loaded chain has a wrong genesis, bail out immediately // (we're likely using a testnet datadir, or the other way around). return {ChainstateLoadStatus::FAILURE_INCOMPATIBLE_DB, _("Incorrect or no genesis block found. Wrong datadir for " "network?")}; } // Check for changed -prune state. What we are concerned about is a // user who has pruned blocks in the past, but is now trying to run // unpruned. if (chainman.m_blockman.m_have_pruned && !options.prune) { return { ChainstateLoadStatus::FAILURE, _("You need to rebuild the database using -reindex to go back to " "unpruned mode. This will redownload the entire blockchain")}; } // At this point blocktree args are consistent with what's on disk. // If we're not mid-reindex (based on disk + args), add a genesis // block on disk (otherwise we use the one already on disk). This is // called again in ThreadImport after the reindex completes. if (!fReindex && !chainman.ActiveChainstate().LoadGenesisBlock()) { return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } auto is_coinsview_empty = [&](Chainstate *chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); }; assert(chainman.m_total_coinstip_cache > 0); assert(chainman.m_total_coinsdb_cache > 0); // Conservative value which is arbitrarily chosen, as it will ultimately be // changed by a call to `chainman.MaybeRebalanceCaches()`. We just need to // make sure that the sum of the two caches (40%) does not exceed the // allowable amount during this temporary initialization state. double init_cache_fraction = 0.2; // At this point we're either in reindex or we've loaded a useful // block tree into BlockIndex()! for (Chainstate *chainstate : chainman.GetAll()) { LogPrintf("Initializing chainstate %s\n", chainstate->ToString()); chainstate->InitCoinsDB( /* cache_size_bytes */ chainman.m_total_coinsdb_cache * init_cache_fraction, /* in_memory */ options.coins_db_in_memory, /* should_wipe */ options.reindex || options.reindex_chainstate); if (options.coins_error_cb) { chainstate->CoinsErrorCatcher().AddReadErrCallback( options.coins_error_cb); } // If necessary, upgrade from older database format. // This is a no-op if we cleared the coinsviewdb with -reindex // or -reindex-chainstate if (!chainstate->CoinsDB().Upgrade()) { return {ChainstateLoadStatus::FAILURE_INCOMPATIBLE_DB, _("Error upgrading chainstate database")}; } // ReplayBlocks is a no-op if we cleared the coinsviewdb with // -reindex or -reindex-chainstate if (!chainstate->ReplayBlocks()) { return {ChainstateLoadStatus::FAILURE, _("Unable to replay blocks. You will need to rebuild the " "database using -reindex-chainstate.")}; } // The on-disk coinsdb is now in a good state, create the cache chainstate->InitCoinsCache(chainman.m_total_coinstip_cache * init_cache_fraction); assert(chainstate->CanFlushToDisk()); if (!is_coinsview_empty(chainstate)) { // LoadChainTip initializes the chain based on CoinsTip()'s // best block if (!chainstate->LoadChainTip()) { return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } assert(chainstate->m_chain.Tip() != nullptr); } } // Now that chainstates are loaded and we're able to flush to // disk, rebalance the coins caches to desired levels based // on the condition of each chainstate. chainman.MaybeRebalanceCaches(); return {ChainstateLoadStatus::SUCCESS, {}}; } ChainstateLoadResult LoadChainstate(ChainstateManager &chainman, const CacheSizes &cache_sizes, const ChainstateLoadOptions &options) { if (!chainman.AssumedValidBlock().IsNull()) { LogPrintf("Assuming ancestors of block %s have valid signatures.\n", chainman.AssumedValidBlock().GetHex()); } else { LogPrintf("Validating signatures for all blocks.\n"); } LogPrintf("Setting nMinimumChainWork=%s\n", chainman.MinimumChainWork().GetHex()); if (chainman.MinimumChainWork() < UintToArith256(chainman.GetConsensus().nMinimumChainWork)) { LogPrintf("Warning: nMinimumChainWork set below default value of %s\n", chainman.GetConsensus().nMinimumChainWork.GetHex()); } if (chainman.m_blockman.GetPruneTarget() == - std::numeric_limits::max()) { + BlockManager::PRUNE_TARGET_MANUAL) { LogPrintf( "Block pruning enabled. Use RPC call pruneblockchain(height) to " "manually prune block and undo files.\n"); } else if (chainman.m_blockman.GetPruneTarget()) { LogPrintf("Prune configured to target %u MiB on disk for block and " "undo files.\n", chainman.m_blockman.GetPruneTarget() / 1024 / 1024); } LOCK(cs_main); chainman.m_total_coinstip_cache = cache_sizes.coins; chainman.m_total_coinsdb_cache = cache_sizes.coins_db; // Load the fully validated chainstate. chainman.InitializeChainstate(options.mempool); // Load a chain created from a UTXO snapshot, if any exist. chainman.DetectSnapshotChainstate(options.mempool); { auto [init_status, init_error] = CompleteChainstateInitialization(chainman, cache_sizes, options); if (init_status != ChainstateLoadStatus::SUCCESS) { return {init_status, init_error}; } } // If a snapshot chainstate was fully validated by a background chainstate // during the last run, detect it here and clean up the now-unneeded // background chainstate. // // Why is this cleanup done here (on subsequent restart) and not just when // the snapshot is actually validated? Because this entails unusual // filesystem operations to move leveldb data directories around, and that // seems too risky to do in the middle of normal runtime. auto snapshot_completion = chainman.MaybeCompleteSnapshotValidation(); if (snapshot_completion == SnapshotCompletionResult::SKIPPED) { // do nothing; expected case } else if (snapshot_completion == SnapshotCompletionResult::SUCCESS) { LogPrintf("[snapshot] cleaning up unneeded background chainstate, then " "reinitializing\n"); if (!chainman.ValidatedSnapshotCleanup()) { AbortNode("Background chainstate cleanup failed unexpectedly."); } // Because ValidatedSnapshotCleanup() has torn down chainstates with // ChainstateManager::ResetChainstates(), reinitialize them here without // duplicating the blockindex work above. assert(chainman.GetAll().empty()); assert(!chainman.IsSnapshotActive()); assert(!chainman.IsSnapshotValidated()); chainman.InitializeChainstate(options.mempool); // A reload of the block index is required to recompute // setBlockIndexCandidates for the fully validated chainstate. chainman.ActiveChainstate().UnloadBlockIndex(); auto [init_status, init_error] = CompleteChainstateInitialization(chainman, cache_sizes, options); if (init_status != ChainstateLoadStatus::SUCCESS) { return {init_status, init_error}; } } else { return {ChainstateLoadStatus::FAILURE, _("UTXO snapshot failed to validate. " "Restart to resume normal initial block download, or try " "loading a different snapshot.")}; } return {ChainstateLoadStatus::SUCCESS, {}}; } ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager &chainman, const ChainstateLoadOptions &options) { auto is_coinsview_empty = [&](Chainstate *chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); }; LOCK(cs_main); for (Chainstate *chainstate : chainman.GetAll()) { if (!is_coinsview_empty(chainstate)) { const CBlockIndex *tip = chainstate->m_chain.Tip(); if (tip && tip->nTime > GetTime() + MAX_FUTURE_BLOCK_TIME) { return {ChainstateLoadStatus::FAILURE, _("The block database contains a block which appears " "to be from the future. " "This may be due to your computer's date and time " "being set incorrectly. " "Only rebuild the block database if you are sure " "that your computer's date and time are correct")}; } VerifyDBResult result = CVerifyDB().VerifyDB(*chainstate, chainstate->CoinsDB(), options.check_level, options.check_blocks); switch (result) { case VerifyDBResult::SUCCESS: case VerifyDBResult::SKIPPED_MISSING_BLOCKS: break; case VerifyDBResult::INTERRUPTED: return {ChainstateLoadStatus::INTERRUPTED, _("Block verification was interrupted")}; case VerifyDBResult::CORRUPTED_BLOCK_DB: return {ChainstateLoadStatus::FAILURE, _("Corrupted block database detected")}; case VerifyDBResult::SKIPPED_L3_CHECKS: if (options.require_full_verification) { return { ChainstateLoadStatus::FAILURE_INSUFFICIENT_DBCACHE, _("Insufficient dbcache for block verification")}; } break; } // no default case, so the compiler can warn about missing cases } } return {ChainstateLoadStatus::SUCCESS, {}}; } } // namespace node diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 5cb858ada..9d323548f 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1,2753 +1,2753 @@ // Copyright (c) 2010 Satoshi Nakamoto // Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include