diff --git a/doc/files.md b/doc/files.md index c22dda98a2..a74cbd28e3 100644 --- a/doc/files.md +++ b/doc/files.md @@ -1,34 +1,33 @@ * banlist.dat: stores the IPs/Subnets of banned nodes * bitcoin.conf: contains configuration settings for bitcoind or bitcoin-qt * bitcoind.pid: stores the process id of bitcoind while running * blocks/blk000??.dat: block data (custom, 128 MiB per file); since 0.8.0 * blocks/rev000??.dat; block undo data (custom); since 0.8.0 (format changed since pre-0.8) * blocks/index/*; block index (LevelDB); since 0.8.0 * chainstate/*; block chain state database (LevelDB); since 0.8.0 * database/*: BDB database environment; only used for wallet since 0.8.0; moved to wallets/ directory on new installs since 0.18.7 * db.log: wallet database log file; moved to wallets/ directory on new installs since 0.18.7 * debug.log: contains debug information and general logging generated by bitcoind or bitcoin-qt -* fee_estimates.dat: stores statistics used to estimate minimum transaction fees and priorities required for confirmation; since 0.10.0 * mempool.dat: dump of the mempool's transactions; since 0.14.0. * peers.dat: peer IP address database (custom format); since 0.7.0 * wallet.dat: personal wallet (BDB) with keys and transactions; moved to wallets/ directory on new installs since 0.18.7 * wallets/database/*: BDB database environment; used for wallets since 0.18.7 * wallets/db.log: wallet database log file; since 0.18.7 * wallets/wallet.dat: personal wallet (BDB) with keys and transactions; since 0.18.7 * .cookie: session RPC authentication cookie (written at start when cookie authentication is used, deleted on shutdown): since 0.12.0 * onion_private_key: cached Tor hidden service private key for `-listenonion`: since 0.12.0 Only used in pre-0.8.0 --------------------- * blktree/*; block chain index (LevelDB); since pre-0.8, replaced by blocks/index/* in 0.8.0 * coins/*; unspent transaction output database (LevelDB); since pre-0.8, replaced by chainstate/* in 0.8.0 Only used before 0.8.0 --------------------- * blkindex.dat: block chain index database (BDB); replaced by {chainstate/*,blocks/index/*,blocks/rev000??.dat} in 0.8.0 * blk000?.dat: block data (custom, 2 GiB per file); replaced by blocks/blk000??.dat in 0.8.0 Only used before 0.7.0 --------------------- * addr.dat: peer IP address database (BDB); replaced by peers.dat in 0.7.0 diff --git a/doc/release-notes.md b/doc/release-notes.md index b5c9183ac5..1da733d9c3 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,9 +1,11 @@ Bitcoin ABC version 0.18.8 is now available from: This release includes the following features and fixes: - `dumpwallet` now includes hex-encoded scripts from the wallet in the dumpfile - `importwallet` now imports these scripts, but corresponding addresses may not be added correctly or a manual rescan may be required to find relevant - transactions \ No newline at end of file + transactions + - Remove miner policy estimator in favor of minimum fees, also remove `fee_estimates.dat`. + Old copies will be left in place. diff --git a/src/init.cpp b/src/init.cpp index f68446d1cd..6d680d3eee 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1,2417 +1,2392 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) #include "config/bitcoin-config.h" #endif #include "init.h" #include "addrman.h" #include "amount.h" #include "chain.h" #include "chainparams.h" #include "checkpoints.h" #include "compat/sanity.h" #include "config.h" #include "consensus/validation.h" #include "diskblockpos.h" #include "fs.h" #include "httprpc.h" #include "httpserver.h" #include "key.h" #include "miner.h" #include "net.h" #include "net_processing.h" #include "netbase.h" #include "policy/policy.h" #include "rpc/register.h" #include "rpc/safemode.h" #include "rpc/server.h" #include "scheduler.h" #include "script/scriptcache.h" #include "script/sigcache.h" #include "script/standard.h" #include "timedata.h" #include "torcontrol.h" #include "txdb.h" #include "txmempool.h" #include "ui_interface.h" #include "util.h" #include "utilmoneystr.h" #include "validation.h" #include "validationinterface.h" #ifdef ENABLE_WALLET #include "wallet/rpcdump.h" #endif #include "walletinitinterface.h" #include "warnings.h" #include #include #include #ifndef WIN32 #include #endif #include #include #include #include #include #include #if ENABLE_ZMQ #include "zmq/zmqnotificationinterface.h" #endif -bool fFeeEstimatesInitialized = false; static const bool DEFAULT_PROXYRANDOMIZE = true; static const bool DEFAULT_REST_ENABLE = false; static const bool DEFAULT_STOPAFTERBLOCKIMPORT = false; std::unique_ptr g_connman; std::unique_ptr peerLogic; #if !(ENABLE_WALLET) class DummyWalletInit : public WalletInitInterface { public: std::string GetHelpString(bool showDebug) override { return std::string{}; } bool ParameterInteraction() override { return true; } void RegisterRPC(CRPCTable &) override {} bool Verify(const CChainParams &chainParams) override { return true; } bool Open(const CChainParams &chainParams) override { return true; } void Start(CScheduler &scheduler) override {} void Flush() override {} void Stop() override {} void Close() override {} }; static DummyWalletInit g_dummy_wallet_init; WalletInitInterface *const g_wallet_init_interface = &g_dummy_wallet_init; #endif #if ENABLE_ZMQ static CZMQNotificationInterface *pzmqNotificationInterface = nullptr; #endif #ifdef WIN32 // Win32 LevelDB doesn't use filedescriptors, and the ones used for accessing // block files don't count towards the fd_set size limit anyway. #define MIN_CORE_FILEDESCRIPTORS 0 #else #define MIN_CORE_FILEDESCRIPTORS 150 #endif -static const char *FEE_ESTIMATES_FILENAME = "fee_estimates.dat"; - ////////////////////////////////////////////////////////////////////////////// // // Shutdown // // // Thread management and startup/shutdown: // // The network-processing threads are all part of a thread group created by // AppInit() or the Qt main() function. // // A clean exit happens when StartShutdown() or the SIGTERM signal handler sets // fRequestShutdown, which triggers the DetectShutdownThread(), which interrupts // the main thread group. DetectShutdownThread() then exits, which causes // AppInit() to continue (it .joins the shutdown thread). Shutdown() is then // called to clean up database connections, and stop other threads that should // only be stopped after the main network-processing threads have exited. // // Note that if running -daemon the parent process returns from AppInit2 before // adding any threads to the threadGroup, so .join_all() returns immediately and // the parent exits from main(). // // Shutdown for Qt is very similar, only it uses a QTimer to detect // fRequestShutdown getting set, and then does the normal Qt shutdown thing. // std::atomic fRequestShutdown(false); std::atomic fDumpMempoolLater(false); void StartShutdown() { fRequestShutdown = true; } bool ShutdownRequested() { return fRequestShutdown; } /** * This is a minimally invasive approach to shutdown on LevelDB read errors from * the chainstate, while keeping user interface out of the common library, which * is shared between bitcoind, and bitcoin-qt and non-server tools. */ class CCoinsViewErrorCatcher final : public CCoinsViewBacked { public: explicit CCoinsViewErrorCatcher(CCoinsView *view) : CCoinsViewBacked(view) {} bool GetCoin(const COutPoint &outpoint, Coin &coin) const override { try { return CCoinsViewBacked::GetCoin(outpoint, coin); } catch (const std::runtime_error &e) { uiInterface.ThreadSafeMessageBox( _("Error reading from database, shutting down."), "", CClientUIInterface::MSG_ERROR); LogPrintf("Error reading from database: %s\n", e.what()); // Starting the shutdown sequence and returning false to the caller // would be interpreted as 'entry not found' (as opposed to unable // to read data), and could lead to invalid interpretation. Just // exit immediately, as we can't continue anyway, and all writes // should be atomic. abort(); } } // Writes do not need similar protection, as failure to write is handled by // the caller. }; static std::unique_ptr pcoinscatcher; static std::unique_ptr globalVerifyHandle; static boost::thread_group threadGroup; static CScheduler scheduler; void Interrupt() { InterruptHTTPServer(); InterruptHTTPRPC(); InterruptRPC(); InterruptREST(); InterruptTorControl(); InterruptMapPort(); if (g_connman) { g_connman->Interrupt(); } } void Shutdown() { LogPrintf("%s: In progress...\n", __func__); static CCriticalSection cs_Shutdown; TRY_LOCK(cs_Shutdown, lockShutdown); if (!lockShutdown) { return; } /// Note: Shutdown() must be able to handle cases in which AppInit2() failed /// part of the way, for example if the data directory was found to be /// locked. Be sure that anything that writes files or flushes caches only /// does this if the respective module was initialized. RenameThread("bitcoin-shutoff"); g_mempool.AddTransactionsUpdated(1); StopHTTPRPC(); StopREST(); StopRPC(); StopHTTPServer(); g_wallet_init_interface->Flush(); StopMapPort(); // Because these depend on each-other, we make sure that neither can be // using the other before destroying them. if (peerLogic) { UnregisterValidationInterface(peerLogic.get()); } if (g_connman) { g_connman->Stop(); } peerLogic.reset(); g_connman.reset(); StopTorControl(); // After everything has been shut down, but before things get flushed, stop // the CScheduler/checkqueue threadGroup threadGroup.interrupt_all(); threadGroup.join_all(); if (fDumpMempoolLater && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(); } - if (fFeeEstimatesInitialized) { - fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; - CAutoFile est_fileout(fsbridge::fopen(est_path, "wb"), SER_DISK, - CLIENT_VERSION); - if (!est_fileout.IsNull()) { - g_mempool.WriteFeeEstimates(est_fileout); - } else { - LogPrintf("%s: Failed to write fee estimates to %s\n", __func__, - est_path.string()); - } - fFeeEstimatesInitialized = false; - } - // FlushStateToDisk generates a SetBestChain callback, which we should avoid // missing if (pcoinsTip != nullptr) { FlushStateToDisk(); } // After there are no more peers/RPC left to give us new data which may // generate CValidationInterface callbacks, flush them... GetMainSignals().FlushBackgroundCallbacks(); // Any future callbacks will be dropped. This should absolutely be safe - if // missing a callback results in an unrecoverable situation, unclean // shutdown would too. The only reason to do the above flushes is to let the // wallet catch up with our current chain to avoid any strange pruning edge // cases and make next startup faster by avoiding rescan. { LOCK(cs_main); if (pcoinsTip != nullptr) { FlushStateToDisk(); } pcoinsTip.reset(); pcoinscatcher.reset(); pcoinsdbview.reset(); pblocktree.reset(); } g_wallet_init_interface->Stop(); #if ENABLE_ZMQ if (pzmqNotificationInterface) { UnregisterValidationInterface(pzmqNotificationInterface); delete pzmqNotificationInterface; pzmqNotificationInterface = nullptr; } #endif #ifndef WIN32 try { fs::remove(GetPidFile()); } catch (const fs::filesystem_error &e) { LogPrintf("%s: Unable to remove pidfile: %s\n", __func__, e.what()); } #endif UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); GetMainSignals().UnregisterWithMempoolSignals(g_mempool); g_wallet_init_interface->Close(); globalVerifyHandle.reset(); ECC_Stop(); LogPrintf("%s: done\n", __func__); } /** * Signal handlers are very limited in what they are allowed to do, so: */ void HandleSIGTERM(int) { fRequestShutdown = true; } void HandleSIGHUP(int) { GetLogger().m_reopen_file = true; } void OnRPCStarted() { uiInterface.NotifyBlockTip.connect(&RPCNotifyBlockChange); } void OnRPCStopped() { uiInterface.NotifyBlockTip.disconnect(&RPCNotifyBlockChange); RPCNotifyBlockChange(false, nullptr); g_best_block_cv.notify_all(); LogPrint(BCLog::RPC, "RPC stopped.\n"); } std::string HelpMessage(HelpMessageMode mode) { const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); const auto defaultChainParams = CreateChainParams(CBaseChainParams::MAIN); const auto testnetChainParams = CreateChainParams(CBaseChainParams::TESTNET); const bool showDebug = gArgs.GetBoolArg("-help-debug", false); // When adding new options to the categories, please keep and ensure // alphabetical ordering. Do not translate _(...) -help-debug options, Many // technical terms, and only a very small audience, so is unnecessary stress // to translators. std::string strUsage = HelpMessageGroup(_("Options:")); strUsage += HelpMessageOpt("-?", _("Print this help message and exit")); strUsage += HelpMessageOpt("-version", _("Print version and exit")); strUsage += HelpMessageOpt( "-alertnotify=", _("Execute command when a relevant alert is received or we see a " "really long fork (%s in cmd is replaced by message)")); strUsage += HelpMessageOpt("-blocknotify=", _("Execute command when the best block changes " "(%s in cmd is replaced by block hash)")); if (showDebug) { strUsage += HelpMessageOpt( "-blocksonly", strprintf( _("Whether to operate in a blocks only mode (default: %d)"), DEFAULT_BLOCKSONLY)); } strUsage += HelpMessageOpt( "-assumevalid=", strprintf( _("If this block is in the chain assume that it and its ancestors " "are valid and potentially skip their script verification (0 to " "verify all, default: %s, testnet: %s)"), defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex())); strUsage += HelpMessageOpt( "-conf=", strprintf(_("Specify configuration file (default: %s)"), BITCOIN_CONF_FILENAME)); if (mode == HMM_BITCOIND) { #if HAVE_DECL_DAEMON strUsage += HelpMessageOpt( "-daemon", _("Run in the background as a daemon and accept commands")); #endif } strUsage += HelpMessageOpt("-datadir=", _("Specify data directory")); if (showDebug) { strUsage += HelpMessageOpt( "-dbbatchsize", strprintf( "Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize)); } strUsage += HelpMessageOpt( "-dbcache=", strprintf( _("Set database cache size in megabytes (%d to %d, default: %d)"), nMinDbCache, nMaxDbCache, nDefaultDbCache)); if (showDebug) { strUsage += HelpMessageOpt( "-feefilter", strprintf("Tell other nodes to filter invs to us by " "our mempool min fee (default: %d)", DEFAULT_FEEFILTER)); } strUsage += HelpMessageOpt( "-finalizationdelay=", strprintf("Set the minimum amount of time to wait between a " "block header reception and the block finalization. " "Unit is seconds (default: %d)", DEFAULT_MIN_FINALIZATION_DELAY)); strUsage += HelpMessageOpt( "-maxreorgdepth=", strprintf("Configure at what depth blocks are considered final " "(default: %d). Use -1 to disable.", DEFAULT_MAX_REORG_DEPTH)); strUsage += HelpMessageOpt( "-loadblock=", _("Imports blocks from external blk000??.dat file on startup")); strUsage += HelpMessageOpt( "-debuglogfile=", strprintf( _("Specify location of debug log file: this can be an absolute " "path or a path relative to the data directory (default: %s)"), DEFAULT_DEBUGLOGFILE)); strUsage += HelpMessageOpt( "-maxorphantx=", strprintf(_("Keep at most unconnectable " "transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-maxmempool=", strprintf(_("Keep the transaction memory pool " "below megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); strUsage += HelpMessageOpt("-mempoolexpiry=", strprintf(_("Do not keep transactions in the mempool " "longer than hours (default: %u)"), DEFAULT_MEMPOOL_EXPIRY)); if (showDebug) { strUsage += HelpMessageOpt( "-minimumchainwork=", strprintf( "Minimum work assumed to exist on a valid chain in hex " "(default: %s, testnet: %s)", defaultChainParams->GetConsensus().nMinimumChainWork.GetHex(), testnetChainParams->GetConsensus().nMinimumChainWork.GetHex())); } strUsage += HelpMessageOpt("-persistmempool", strprintf(_("Whether to save the mempool on shutdown " "and load on restart (default: %u)"), DEFAULT_PERSIST_MEMPOOL)); strUsage += HelpMessageOpt( "-blockreconstructionextratxn=", strprintf(_("Extra transactions to keep in memory for compact block " "reconstructions (default: %u)"), DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN)); strUsage += HelpMessageOpt( "-par=", strprintf(_("Set the number of script verification threads (%u to %d, " "0 = auto, <0 = leave that many cores free, default: %d)"), -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS)); #ifndef WIN32 strUsage += HelpMessageOpt( "-pid=", strprintf(_("Specify pid file (default: %s)"), BITCOIN_PID_FILENAME)); #endif strUsage += HelpMessageOpt( "-prune=", strprintf( _("Reduce storage requirements by enabling pruning (deleting) of " "old blocks. This allows the pruneblockchain RPC to be called to " "delete specific blocks, and enables automatic pruning of old " "blocks if a target size in MiB is provided. This mode is " "incompatible with -txindex and -rescan. " "Warning: Reverting this setting requires re-downloading the " "entire blockchain. " "(default: 0 = disable pruning blocks, 1 = allow manual pruning " "via RPC, >%u = automatically prune block files to stay under " "the specified target size in MiB)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)); strUsage += HelpMessageOpt( "-reindex-chainstate", _("Rebuild chain state from the currently indexed blocks")); strUsage += HelpMessageOpt("-reindex", _("Rebuild chain state and block index from " "the blk*.dat files on disk")); #ifndef WIN32 strUsage += HelpMessageOpt( "-sysperms", _("Create new files with system default permissions, instead of umask " "077 (only effective with disabled wallet functionality)")); #endif strUsage += HelpMessageOpt( "-txindex", strprintf(_("Maintain a full transaction index, used by " "the getrawtransaction rpc call (default: %d)"), DEFAULT_TXINDEX)); strUsage += HelpMessageOpt( "-usecashaddr", _("Use Cash Address for destination encoding instead " "of base58 (activate by default on Jan, 14)")); strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt( "-addnode=", _("Add a node to connect to and attempt to keep the connection open " "(see the `addnode` RPC command help for more info)")); strUsage += HelpMessageOpt( "-banscore=", strprintf( _("Threshold for disconnecting misbehaving peers (default: %u)"), DEFAULT_BANSCORE_THRESHOLD)); strUsage += HelpMessageOpt( "-bantime=", strprintf(_("Number of seconds to keep misbehaving " "peers from reconnecting (default: %u)"), DEFAULT_MISBEHAVING_BANTIME)); strUsage += HelpMessageOpt("-bind=", _("Bind to given address and always listen on " "it. Use [host]:port notation for IPv6")); strUsage += HelpMessageOpt( "-connect=", _("Connect only to the specified node(s); -connect=0 " "disables automatic connections (the rules for this " "peer are the same as for -addnode)")); strUsage += HelpMessageOpt("-discover", _("Discover own IP addresses (default: 1 when " "listening and no -externalip or -proxy)")); strUsage += HelpMessageOpt( "-dns", _("Allow DNS lookups for -addnode, -seednode and -connect") + " " + strprintf(_("(default: %d)"), DEFAULT_NAME_LOOKUP)); strUsage += HelpMessageOpt( "-dnsseed", _("Query for peer addresses via DNS lookup, if low on " "addresses (default: 1 unless -connect/-noconnect)")); strUsage += HelpMessageOpt("-externalip=", _("Specify your own public address")); strUsage += HelpMessageOpt( "-forcednsseed", strprintf( _("Always query for peer addresses via DNS lookup (default: %d)"), DEFAULT_FORCEDNSSEED)); strUsage += HelpMessageOpt("-listen", _("Accept connections from outside (default: " "1 if no -proxy or -connect/-noconnect)")); strUsage += HelpMessageOpt( "-listenonion", strprintf(_("Automatically create Tor hidden service (default: %d)"), DEFAULT_LISTEN_ONION)); strUsage += HelpMessageOpt( "-maxconnections=", strprintf(_("Maintain at most connections to peers (default: %u)"), DEFAULT_MAX_PEER_CONNECTIONS)); strUsage += HelpMessageOpt("-maxreceivebuffer=", strprintf(_("Maximum per-connection receive buffer, " "*1000 bytes (default: %u)"), DEFAULT_MAXRECEIVEBUFFER)); strUsage += HelpMessageOpt( "-maxsendbuffer=", strprintf(_("Maximum per-connection send buffer, " "*1000 bytes (default: %u)"), DEFAULT_MAXSENDBUFFER)); strUsage += HelpMessageOpt( "-maxtimeadjustment", strprintf(_("Maximum allowed median peer time offset adjustment. Local " "perspective of time may be influenced by peers forward or " "backward by this amount. (default: %u seconds)"), DEFAULT_MAX_TIME_ADJUSTMENT)); strUsage += HelpMessageOpt("-onion=", strprintf(_("Use separate SOCKS5 proxy to reach peers " "via Tor hidden services (default: %s)"), "-proxy")); strUsage += HelpMessageOpt( "-onlynet=", _("Only connect to nodes in network (ipv4, ipv6 or onion)")); strUsage += HelpMessageOpt("-permitbaremultisig", strprintf(_("Relay non-P2SH multisig (default: %d)"), DEFAULT_PERMIT_BAREMULTISIG)); strUsage += HelpMessageOpt( "-peerbloomfilters", strprintf(_("Support filtering of blocks and transaction with bloom " "filters (default: %d)"), DEFAULT_PEERBLOOMFILTERS)); strUsage += HelpMessageOpt( "-port=", strprintf( _("Listen for connections on (default: %u or testnet: %u)"), defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort())); strUsage += HelpMessageOpt("-proxy=", _("Connect through SOCKS5 proxy")); strUsage += HelpMessageOpt( "-proxyrandomize", strprintf(_("Randomize credentials for every proxy connection. This " "enables Tor stream isolation (default: %d)"), DEFAULT_PROXYRANDOMIZE)); strUsage += HelpMessageOpt( "-seednode=", _("Connect to a node to retrieve peer addresses, and disconnect")); strUsage += HelpMessageOpt( "-timeout=", strprintf(_("Specify connection timeout in " "milliseconds (minimum: 1, default: %d)"), DEFAULT_CONNECT_TIMEOUT)); strUsage += HelpMessageOpt("-torcontrol=:", strprintf(_("Tor control port to use if onion " "listening enabled (default: %s)"), DEFAULT_TOR_CONTROL)); strUsage += HelpMessageOpt("-torpassword=", _("Tor control port password (default: empty)")); #ifdef USE_UPNP #if USE_UPNP strUsage += HelpMessageOpt("-upnp", _("Use UPnP to map the listening port " "(default: 1 when listening and no -proxy)")); #else strUsage += HelpMessageOpt( "-upnp", strprintf(_("Use UPnP to map the listening port (default: %u)"), 0)); #endif #endif strUsage += HelpMessageOpt("-whitebind=", _("Bind to given address and whitelist peers connecting " "to it. Use [host]:port notation for IPv6")); strUsage += HelpMessageOpt( "-whitelist=", _("Whitelist peers connecting from the given IP address (e.g. 1.2.3.4) " "or CIDR notated network (e.g. 1.2.3.0/24). Can be specified " "multiple times.") + " " + _("Whitelisted peers cannot be DoS banned and their transactions " "are always relayed, even if they are already in the mempool, " "useful e.g. for a gateway")); strUsage += HelpMessageOpt( "-whitelistrelay", strprintf(_("Accept relayed transactions received from whitelisted " "peers even when not relaying transactions (default: %d)"), DEFAULT_WHITELISTRELAY)); strUsage += HelpMessageOpt( "-whitelistforcerelay", strprintf(_("Force relay of transactions from whitelisted peers even " "if they violate local relay policy (default: %d)"), DEFAULT_WHITELISTFORCERELAY)); strUsage += HelpMessageOpt( "-maxuploadtarget=", strprintf(_("Tries to keep outbound traffic under the given target (in " "MiB per 24h), 0 = no limit (default: %d)"), DEFAULT_MAX_UPLOAD_TARGET)); strUsage += g_wallet_init_interface->GetHelpString(showDebug); #if ENABLE_ZMQ strUsage += HelpMessageGroup(_("ZeroMQ notification options:")); strUsage += HelpMessageOpt("-zmqpubhashblock=
", _("Enable publish hash block in
")); strUsage += HelpMessageOpt("-zmqpubhashtx=
", _("Enable publish hash transaction in
")); strUsage += HelpMessageOpt("-zmqpubrawblock=
", _("Enable publish raw block in
")); strUsage += HelpMessageOpt("-zmqpubrawtx=
", _("Enable publish raw transaction in
")); #endif strUsage += HelpMessageGroup(_("Debugging/Testing options:")); strUsage += HelpMessageOpt("-uacomment=", _("Append comment to the user agent string")); if (showDebug) { strUsage += HelpMessageOpt( "-checkblocks=", strprintf( _("How many blocks to check at startup (default: %u, 0 = all)"), DEFAULT_CHECKBLOCKS)); strUsage += HelpMessageOpt("-checklevel=", strprintf(_("How thorough the block verification of " "-checkblocks is (0-4, default: %u)"), DEFAULT_CHECKLEVEL)); strUsage += HelpMessageOpt( "-checkblockindex", strprintf("Do a full consistency check for mapBlockIndex, " "setBlockIndexCandidates, chainActive and " "mapBlocksUnlinked occasionally. Also sets -checkmempool " "(default: %u)", defaultChainParams->DefaultConsistencyChecks())); strUsage += HelpMessageOpt( "-checkmempool=", strprintf("Run checks every transactions (default: %u)", defaultChainParams->DefaultConsistencyChecks())); strUsage += HelpMessageOpt( "-checkpoints", strprintf("Only accept block chain matching " "built-in checkpoints (default: %d)", DEFAULT_CHECKPOINTS_ENABLED)); strUsage += HelpMessageOpt( "-disablesafemode", strprintf("Disable safemode, override a real " "safe mode event (default: %d)", DEFAULT_DISABLE_SAFEMODE)); strUsage += HelpMessageOpt("-deprecatedrpc=", "Allows deprecated RPC method(s) to be used"); strUsage += HelpMessageOpt( "-testsafemode", strprintf("Force safe mode (default: %d)", DEFAULT_TESTSAFEMODE)); strUsage += HelpMessageOpt("-dropmessagestest=", "Randomly drop 1 of every network messages"); strUsage += HelpMessageOpt( "-stopafterblockimport", strprintf( "Stop running after importing blocks from disk (default: %d)", DEFAULT_STOPAFTERBLOCKIMPORT)); strUsage += HelpMessageOpt( "-stopatheight", strprintf("Stop running after reaching the given " "height in the main chain (default: %u)", DEFAULT_STOPATHEIGHT)); strUsage += HelpMessageOpt( "-limitancestorcount=", strprintf("Do not accept transactions if number of in-mempool " "ancestors is or more (default: %u)", DEFAULT_ANCESTOR_LIMIT)); strUsage += HelpMessageOpt("-limitancestorsize=", strprintf("Do not accept transactions whose size " "with all in-mempool ancestors exceeds " " kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT)); strUsage += HelpMessageOpt( "-limitdescendantcount=", strprintf("Do not accept transactions if any ancestor would have " " or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT)); strUsage += HelpMessageOpt( "-limitdescendantsize=", strprintf("Do not accept transactions if any ancestor would have " "more than kilobytes of in-mempool descendants " "(default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT)); strUsage += HelpMessageOpt("-addrmantest", "Allows to test address relay on localhost"); } strUsage += HelpMessageOpt( "-debug=", strprintf(_("Output debugging information (default: %u, supplying " " is optional)"), 0) + ". " + _("If is not supplied or if = 1, output all " "debugging information.") + _(" can be:") + " " + ListLogCategories() + "."); strUsage += HelpMessageOpt( "-debugexclude=", strprintf(_("Exclude debugging information for a category. Can be used " "in conjunction with -debug=1 to output debug logs for all " "categories except one or more specified categories."))); if (showDebug) { strUsage += HelpMessageOpt( "-nodebug", "Turn off debugging messages, same as -debug=0"); } strUsage += HelpMessageOpt( "-help-debug", _("Show all debugging options (usage: --help -help-debug)")); strUsage += HelpMessageOpt( "-logips", strprintf(_("Include IP addresses in debug output (default: %d)"), DEFAULT_LOGIPS)); strUsage += HelpMessageOpt( "-logtimestamps", strprintf(_("Prepend debug output with timestamp (default: %d)"), DEFAULT_LOGTIMESTAMPS)); if (showDebug) { strUsage += HelpMessageOpt( "-logtimemicros", strprintf( "Add microsecond precision to debug timestamps (default: %d)", DEFAULT_LOGTIMEMICROS)); strUsage += HelpMessageOpt( "-mocktime=", "Replace actual time with seconds since epoch (default: 0)"); strUsage += HelpMessageOpt( "-limitfreerelay=", strprintf("Continuously rate-limit free transactions to *1000 " "bytes per minute (default: %u)", DEFAULT_LIMITFREERELAY)); strUsage += HelpMessageOpt("-relaypriority", strprintf("Require high priority for relaying free " "or low-fee transactions (default: %d)", DEFAULT_RELAYPRIORITY)); strUsage += HelpMessageOpt( "-maxsigcachesize=", strprintf("Limit size of signature cache to MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE)); strUsage += HelpMessageOpt( "-maxscriptcachesize=", strprintf("Limit size of script cache to MiB (default: %u)", DEFAULT_MAX_SCRIPT_CACHE_SIZE)); strUsage += HelpMessageOpt( "-maxtipage=", strprintf("Maximum tip age in seconds to consider node in initial " "block download (default: %u)", DEFAULT_MAX_TIP_AGE)); } strUsage += HelpMessageOpt( "-excessutxocharge=", strprintf(_("Fees (in %s/kB) to charge per utxo created for" "relaying, and mining (default: %s)"), CURRENCY_UNIT, FormatMoney(DEFAULT_UTXO_FEE))); strUsage += HelpMessageOpt( "-minrelaytxfee=", strprintf( _("Fees (in %s/kB) smaller than this are considered zero fee for " "relaying, mining and transaction creation (default: %s)"), CURRENCY_UNIT, FormatMoney(DEFAULT_MIN_RELAY_TX_FEE_PER_KB))); strUsage += HelpMessageOpt( "-maxtxfee=", strprintf(_("Maximum total fees (in %s) to use in a single wallet " "transaction or raw transaction; setting this too low may " "abort large transactions (default: %s)"), CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE))); strUsage += HelpMessageOpt( "-printtoconsole", _("Send trace/debug info to console instead of debug.log file")); if (showDebug) { strUsage += HelpMessageOpt( "-printpriority", strprintf("Log transaction priority and fee per " "kB when mining blocks (default: %d)", DEFAULT_PRINTPRIORITY)); } strUsage += HelpMessageOpt("-shrinkdebugfile", _("Shrink debug.log file on client startup " "(default: 1 when no -debug)")); AppendParamsHelpMessages(strUsage, showDebug); strUsage += HelpMessageGroup(_("Node relay options:")); if (showDebug) { strUsage += HelpMessageOpt( "-acceptnonstdtxn", strprintf( "Relay and mine \"non-standard\" transactions (%sdefault: %u)", "testnet/regtest only; ", defaultChainParams->RequireStandard())); strUsage += HelpMessageOpt("-excessiveblocksize=", strprintf(_("Do not accept blocks larger than this " "limit, in bytes (default: %d)"), DEFAULT_MAX_BLOCK_SIZE)); strUsage += HelpMessageOpt( "-dustrelayfee=", strprintf("Fee rate (in %s/kB) used to defined dust, the value of " "an output such that it will cost about 1/3 of its value " "in fees at this fee rate to spend it. (default: %s)", CURRENCY_UNIT, FormatMoney(DUST_RELAY_TX_FEE))); } strUsage += HelpMessageOpt("-bytespersigop", strprintf(_("Equivalent bytes per sigop in transactions " "for relay and mining (default: %u)"), DEFAULT_BYTES_PER_SIGOP)); strUsage += HelpMessageOpt( "-datacarrier", strprintf(_("Relay and mine data carrier transactions (default: %d)"), DEFAULT_ACCEPT_DATACARRIER)); strUsage += HelpMessageOpt( "-datacarriersize", strprintf(_("Maximum size of data in data carrier transactions we " "relay and mine (default: %u)"), MAX_OP_RETURN_RELAY)); strUsage += HelpMessageGroup(_("Block creation options:")); strUsage += HelpMessageOpt( "-blockmaxsize=", strprintf(_("Set maximum block size in bytes (default: %d)"), DEFAULT_MAX_GENERATED_BLOCK_SIZE)); strUsage += HelpMessageOpt( "-blockprioritypercentage=", strprintf(_("Set maximum percentage of a block reserved to " "high-priority/low-fee transactions (default: %d)"), DEFAULT_BLOCK_PRIORITY_PERCENTAGE)); strUsage += HelpMessageOpt( "-blockmintxfee=", strprintf(_("Set lowest fee rate (in %s/kB) for transactions to be " "included in block creation. (default: %s)"), CURRENCY_UNIT, FormatMoney(DEFAULT_BLOCK_MIN_TX_FEE_PER_KB))); if (showDebug) { strUsage += HelpMessageOpt("-blockversion=", "Override block version to test forking scenarios"); } strUsage += HelpMessageGroup(_("RPC server options:")); strUsage += HelpMessageOpt("-server", _("Accept command line and JSON-RPC commands")); strUsage += HelpMessageOpt( "-rest", strprintf(_("Accept public REST requests (default: %d)"), DEFAULT_REST_ENABLE)); strUsage += HelpMessageOpt( "-rpcbind=", _("Bind to given address to listen for JSON-RPC connections. Use " "[host]:port notation for IPv6. This option can be specified " "multiple times (default: bind to all interfaces)")); strUsage += HelpMessageOpt("-rpccookiefile=", _("Location of the auth cookie (default: data dir)")); strUsage += HelpMessageOpt("-rpcuser=", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=", _("Password for JSON-RPC connections")); strUsage += HelpMessageOpt( "-rpcauth=", _("Username and hashed password for JSON-RPC connections. The field " " comes in the format: :$. A canonical " "python script is included in share/rpcuser. The client then " "connects normally using the " "rpcuser=/rpcpassword= pair of arguments. This " "option can be specified multiple times")); strUsage += HelpMessageOpt( "-rpcport=", strprintf(_("Listen for JSON-RPC connections on (default: %u or " "testnet: %u)"), defaultBaseParams->RPCPort(), testnetBaseParams->RPCPort())); strUsage += HelpMessageOpt( "-rpcallowip=", _("Allow JSON-RPC connections from specified source. Valid for " "are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. " "1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This " "option can be specified multiple times")); strUsage += HelpMessageOpt( "-rpcthreads=", strprintf( _("Set the number of threads to service RPC calls (default: %d)"), DEFAULT_HTTP_THREADS)); strUsage += HelpMessageOpt( "-rpccorsdomain=value", "Domain from which to accept cross origin requests (browser enforced)"); if (showDebug) { strUsage += HelpMessageOpt( "-rpcworkqueue=", strprintf("Set the depth of the work queue to " "service RPC calls (default: %d)", DEFAULT_HTTP_WORKQUEUE)); strUsage += HelpMessageOpt( "-rpcservertimeout=", strprintf("Timeout during HTTP requests (default: %d)", DEFAULT_HTTP_SERVER_TIMEOUT)); } return strUsage; } std::string LicenseInfo() { const std::string URL_SOURCE_CODE = ""; const std::string URL_WEBSITE = ""; return CopyrightHolders( strprintf(_("Copyright (C) %i-%i"), 2009, COPYRIGHT_YEAR) + " ") + "\n" + "\n" + strprintf(_("Please contribute if you find %s useful. " "Visit %s for further information about the software."), PACKAGE_NAME, URL_WEBSITE) + "\n" + strprintf(_("The source code is available from %s."), URL_SOURCE_CODE) + "\n" + "\n" + _("This is experimental software.") + "\n" + strprintf(_("Distributed under the MIT software license, see the " "accompanying file %s or %s"), "COPYING", "") + "\n" + "\n" + strprintf(_("This product includes software developed by the " "OpenSSL Project for use in the OpenSSL Toolkit %s and " "cryptographic software written by Eric Young and UPnP " "software written by Thomas Bernard."), "") + "\n"; } static void BlockNotifyCallback(bool initialSync, const CBlockIndex *pBlockIndex) { if (initialSync || !pBlockIndex) { return; } std::string strCmd = gArgs.GetArg("-blocknotify", ""); if (!strCmd.empty()) { boost::replace_all(strCmd, "%s", pBlockIndex->GetBlockHash().GetHex()); std::thread t(runCommand, strCmd); // thread runs free t.detach(); } } static bool fHaveGenesis = false; static CWaitableCriticalSection cs_GenesisWait; static CConditionVariable condvar_GenesisWait; static void BlockNotifyGenesisWait(bool, const CBlockIndex *pBlockIndex) { if (pBlockIndex != nullptr) { { LOCK(cs_GenesisWait); fHaveGenesis = true; } condvar_GenesisWait.notify_all(); } } struct CImportingNow { CImportingNow() { assert(fImporting == false); fImporting = true; } ~CImportingNow() { assert(fImporting == true); fImporting = false; } }; // If we're using -prune with -reindex, then delete block files that will be // ignored by the reindex. Since reindexing works by starting at block file 0 // and looping until a blockfile is missing, do the same here to delete any // later block files after a gap. Also delete all rev files since they'll be // rewritten by the reindex anyway. This ensures that vinfoBlockFile is in sync // with what's actually on disk by the time we start downloading, so that // pruning works correctly. void CleanupBlockRevFiles() { std::map mapBlockFiles; // Glob all blk?????.dat and rev?????.dat files from the blocks directory. // Remove the rev files immediately and insert the blk file paths into an // ordered map keyed by block file index. LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for " "-reindex with -prune\n"); fs::path blocksdir = GetDataDir() / "blocks"; for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { if (is_regular_file(*it) && it->path().filename().string().length() == 12 && it->path().filename().string().substr(8, 4) == ".dat") { if (it->path().filename().string().substr(0, 3) == "blk") { mapBlockFiles[it->path().filename().string().substr(3, 5)] = it->path(); } else if (it->path().filename().string().substr(0, 3) == "rev") { remove(it->path()); } } } // Remove all block files that aren't part of a contiguous set starting at // zero by walking the ordered map (keys are block file indices) by keeping // a separate counter. Once we hit a gap (or if 0 doesn't exist) start // removing block files. int nContigCounter = 0; for (const std::pair &item : mapBlockFiles) { if (atoi(item.first) == nContigCounter) { nContigCounter++; continue; } remove(item.second); } } void ThreadImport(const Config &config, std::vector vImportFiles) { RenameThread("bitcoin-loadblk"); { CImportingNow imp; // -reindex if (fReindex) { int nFile = 0; while (true) { CDiskBlockPos pos(nFile, 0); if (!fs::exists(GetBlockPosFilename(pos, "blk"))) { // No block files left to reindex break; } FILE *file = OpenBlockFile(pos, true); if (!file) { // This error is logged in OpenBlockFile break; } LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile); LoadExternalBlockFile(config, file, &pos); nFile++; } pblocktree->WriteReindexing(false); fReindex = false; LogPrintf("Reindexing finished\n"); // To avoid ending up in a situation without genesis block, re-try // initializing (no-op if reindexing worked): LoadGenesisBlock(config.GetChainParams()); } // hardcoded $DATADIR/bootstrap.dat fs::path pathBootstrap = GetDataDir() / "bootstrap.dat"; if (fs::exists(pathBootstrap)) { FILE *file = fsbridge::fopen(pathBootstrap, "rb"); if (file) { fs::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old"; LogPrintf("Importing bootstrap.dat...\n"); LoadExternalBlockFile(config, file); RenameOver(pathBootstrap, pathBootstrapOld); } else { LogPrintf("Warning: Could not open bootstrap file %s\n", pathBootstrap.string()); } } // -loadblock= for (const fs::path &path : vImportFiles) { FILE *file = fsbridge::fopen(path, "rb"); if (file) { LogPrintf("Importing blocks file %s...\n", path.string()); LoadExternalBlockFile(config, file); } else { LogPrintf("Warning: Could not open blocks file %s\n", path.string()); } } // scan for better chains in the block chain database, that are not yet // connected in the active best chain CValidationState state; if (!ActivateBestChain(config, state)) { LogPrintf("Failed to connect best block"); StartShutdown(); } if (gArgs.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { LogPrintf("Stopping after block import\n"); StartShutdown(); } } // End scope of CImportingNow if (gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { LoadMempool(config); fDumpMempoolLater = !fRequestShutdown; } } /** Sanity checks * Ensure that Bitcoin is running in a usable environment with all * necessary library support. */ bool InitSanityCheck(void) { if (!ECC_InitSanityCheck()) { InitError( "Elliptic curve cryptography sanity check failure. Aborting."); return false; } if (!glibc_sanity_test() || !glibcxx_sanity_test()) { return false; } if (!Random_SanityCheck()) { InitError("OS cryptographic RNG sanity check failure. Aborting."); return false; } return true; } static bool AppInitServers(Config &config, HTTPRPCRequestProcessor &httpRPCRequestProcessor) { RPCServerSignals::OnStarted(&OnRPCStarted); RPCServerSignals::OnStopped(&OnRPCStopped); if (!InitHTTPServer(config)) { return false; } if (!StartRPC()) { return false; } if (!StartHTTPRPC(config, httpRPCRequestProcessor)) { return false; } if (gArgs.GetBoolArg("-rest", DEFAULT_REST_ENABLE) && !StartREST()) { return false; } if (!StartHTTPServer()) { return false; } return true; } // Parameter interaction based on rules void InitParameterInteraction() { // when specifying an explicit binding address, you want to listen on it // even when -connect or -proxy is specified. if (gArgs.IsArgSet("-bind")) { if (gArgs.SoftSetBoolArg("-listen", true)) { LogPrintf( "%s: parameter interaction: -bind set -> setting -listen=1\n", __func__); } } if (gArgs.IsArgSet("-whitebind")) { if (gArgs.SoftSetBoolArg("-listen", true)) { LogPrintf("%s: parameter interaction: -whitebind set -> setting " "-listen=1\n", __func__); } } if (gArgs.IsArgSet("-connect")) { // when only connecting to trusted nodes, do not seed via DNS, or listen // by default. if (gArgs.SoftSetBoolArg("-dnsseed", false)) { LogPrintf("%s: parameter interaction: -connect set -> setting " "-dnsseed=0\n", __func__); } if (gArgs.SoftSetBoolArg("-listen", false)) { LogPrintf("%s: parameter interaction: -connect set -> setting " "-listen=0\n", __func__); } } if (gArgs.IsArgSet("-proxy")) { // to protect privacy, do not listen by default if a default proxy // server is specified. if (gArgs.SoftSetBoolArg("-listen", false)) { LogPrintf( "%s: parameter interaction: -proxy set -> setting -listen=0\n", __func__); } // to protect privacy, do not use UPNP when a proxy is set. The user may // still specify -listen=1 to listen locally, so don't rely on this // happening through -listen below. if (gArgs.SoftSetBoolArg("-upnp", false)) { LogPrintf( "%s: parameter interaction: -proxy set -> setting -upnp=0\n", __func__); } // to protect privacy, do not discover addresses by default if (gArgs.SoftSetBoolArg("-discover", false)) { LogPrintf("%s: parameter interaction: -proxy set -> setting " "-discover=0\n", __func__); } } if (!gArgs.GetBoolArg("-listen", DEFAULT_LISTEN)) { // do not map ports or try to retrieve public IP when not listening // (pointless) if (gArgs.SoftSetBoolArg("-upnp", false)) { LogPrintf( "%s: parameter interaction: -listen=0 -> setting -upnp=0\n", __func__); } if (gArgs.SoftSetBoolArg("-discover", false)) { LogPrintf( "%s: parameter interaction: -listen=0 -> setting -discover=0\n", __func__); } if (gArgs.SoftSetBoolArg("-listenonion", false)) { LogPrintf("%s: parameter interaction: -listen=0 -> setting " "-listenonion=0\n", __func__); } } if (gArgs.IsArgSet("-externalip")) { // if an explicit public IP is specified, do not try to find others if (gArgs.SoftSetBoolArg("-discover", false)) { LogPrintf("%s: parameter interaction: -externalip set -> setting " "-discover=0\n", __func__); } } // disable whitelistrelay in blocksonly mode if (gArgs.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY)) { if (gArgs.SoftSetBoolArg("-whitelistrelay", false)) { LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting " "-whitelistrelay=0\n", __func__); } } // Forcing relay from whitelisted hosts implies we will accept relays from // them in the first place. if (gArgs.GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY)) { if (gArgs.SoftSetBoolArg("-whitelistrelay", true)) { LogPrintf("%s: parameter interaction: -whitelistforcerelay=1 -> " "setting -whitelistrelay=1\n", __func__); } } } static std::string ResolveErrMsg(const char *const optname, const std::string &strBind) { return strprintf(_("Cannot resolve -%s address: '%s'"), optname, strBind); } void InitLogging() { BCLog::Logger &logger = GetLogger(); logger.m_print_to_console = gArgs.GetBoolArg("-printtoconsole", false); logger.m_log_timestamps = gArgs.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); logger.m_log_time_micros = gArgs.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); fLogIPs = gArgs.GetBoolArg("-logips", DEFAULT_LOGIPS); LogPrintf("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"); LogPrintf("%s version %s\n", CLIENT_NAME, FormatFullVersion()); } namespace { // Variables internal to initialization process only int nMaxConnections; int nUserMaxConnections; int nFD; ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_NETWORK_LIMITED); } // namespace [[noreturn]] static void new_handler_terminate() { // Rather than throwing std::bad-alloc if allocation fails, terminate // immediately to (try to) avoid chain corruption. Since LogPrintf may // itself allocate memory, set the handler directly to terminate first. std::set_new_handler(std::terminate); LogPrintf("Error: Out of memory. Terminating.\n"); // The log was successful, terminate now. std::terminate(); }; bool AppInitBasicSetup() { // Step 1: setup #ifdef _MSC_VER // Turn off Microsoft heap dump noise _CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_FILE); _CrtSetReportFile(_CRT_WARN, CreateFileA("NUL", GENERIC_WRITE, 0, nullptr, OPEN_EXISTING, 0, 0)); #endif #if _MSC_VER >= 1400 // Disable confusing "helpful" text message on abort, Ctrl-C _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); #endif #ifdef WIN32 // Enable Data Execution Prevention (DEP) // Minimum supported OS versions: WinXP SP3, WinVista >= SP1, Win Server 2008 // A failure is non-critical and needs no further attention! #ifndef PROCESS_DEP_ENABLE // We define this here, because GCCs winbase.h limits this to _WIN32_WINNT >= // 0x0601 (Windows 7), which is not correct. Can be removed, when GCCs winbase.h // is fixed! #define PROCESS_DEP_ENABLE 0x00000001 #endif typedef BOOL(WINAPI * PSETPROCDEPPOL)(DWORD); PSETPROCDEPPOL setProcDEPPol = (PSETPROCDEPPOL)GetProcAddress( GetModuleHandleA("Kernel32.dll"), "SetProcessDEPPolicy"); if (setProcDEPPol != nullptr) { setProcDEPPol(PROCESS_DEP_ENABLE); } #endif if (!SetupNetworking()) { return InitError("Initializing networking failed"); } #ifndef WIN32 if (!gArgs.GetBoolArg("-sysperms", false)) { umask(077); } // Clean shutdown on SIGTERM struct sigaction sa; sa.sa_handler = HandleSIGTERM; sigemptyset(&sa.sa_mask); sa.sa_flags = 0; sigaction(SIGTERM, &sa, nullptr); sigaction(SIGINT, &sa, nullptr); // Reopen debug.log on SIGHUP struct sigaction sa_hup; sa_hup.sa_handler = HandleSIGHUP; sigemptyset(&sa_hup.sa_mask); sa_hup.sa_flags = 0; sigaction(SIGHUP, &sa_hup, nullptr); // Ignore SIGPIPE, otherwise it will bring the daemon down if the client // closes unexpectedly signal(SIGPIPE, SIG_IGN); #endif std::set_new_handler(new_handler_terminate); return true; } bool AppInitParameterInteraction(Config &config, RPCServer &rpcServer) { const CChainParams &chainparams = config.GetChainParams(); // Step 2: parameter interactions // also see: InitParameterInteraction() // if using block pruning, then disallow txindex if (gArgs.GetArg("-prune", 0)) { if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { return InitError(_("Prune mode is incompatible with -txindex.")); } } // if space reserved for high priority transactions is misconfigured // stop program execution and warn the user with a proper error message const int64_t blkprio = gArgs.GetArg("-blockprioritypercentage", DEFAULT_BLOCK_PRIORITY_PERCENTAGE); if (!config.SetBlockPriorityPercentage(blkprio)) { return InitError(_("Block priority percentage has to belong to the " "[0..100] interval.")); } // -bind and -whitebind can't be set when not listening size_t nUserBind = gArgs.GetArgs("-bind").size() + gArgs.GetArgs("-whitebind").size(); if (nUserBind != 0 && !gArgs.GetBoolArg("-listen", DEFAULT_LISTEN)) { return InitError( "Cannot set -bind or -whitebind together with -listen=0"); } // Make sure enough file descriptors are available int nBind = std::max(nUserBind, size_t(1)); nUserMaxConnections = gArgs.GetArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); nMaxConnections = std::max(nUserMaxConnections, 0); // Trim requested connection counts, to fit into system limitations nMaxConnections = std::max(std::min(nMaxConnections, (int)(FD_SETSIZE - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS)), 0); nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS); if (nFD < MIN_CORE_FILEDESCRIPTORS) { return InitError(_("Not enough file descriptors available.")); } nMaxConnections = std::min(nFD - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS, nMaxConnections); if (nMaxConnections < nUserMaxConnections) { InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, " "because of system limitations."), nUserMaxConnections, nMaxConnections)); } // Step 3: parameter-to-internal-flags if (gArgs.IsArgSet("-debug")) { // Special-case: if -debug=0/-nodebug is set, turn off debugging // messages const std::vector &categories = gArgs.GetArgs("-debug"); if (find(categories.begin(), categories.end(), std::string("0")) == categories.end()) { for (const auto &cat : categories) { BCLog::LogFlags flag; if (!GetLogCategory(flag, cat)) { InitWarning( strprintf(_("Unsupported logging category %s=%s."), "-debug", cat)); } GetLogger().EnableCategory(flag); } } } // Now remove the logging categories which were explicitly excluded for (const std::string &cat : gArgs.GetArgs("-debugexclude")) { BCLog::LogFlags flag; if (!GetLogCategory(flag, cat)) { InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); } GetLogger().DisableCategory(flag); } // Check for -debugnet if (gArgs.GetBoolArg("-debugnet", false)) { InitWarning( _("Unsupported argument -debugnet ignored, use -debug=net.")); } // Check for -socks - as this is a privacy risk to continue, exit here if (gArgs.IsArgSet("-socks")) { return InitError( _("Unsupported argument -socks found. Setting SOCKS version isn't " "possible anymore, only SOCKS5 proxies are supported.")); } // Check for -tor - as this is a privacy risk to continue, exit here if (gArgs.GetBoolArg("-tor", false)) { return InitError(_("Unsupported argument -tor found, use -onion.")); } if (gArgs.GetBoolArg("-benchmark", false)) { InitWarning( _("Unsupported argument -benchmark ignored, use -debug=bench.")); } if (gArgs.GetBoolArg("-whitelistalwaysrelay", false)) { InitWarning(_("Unsupported argument -whitelistalwaysrelay ignored, use " "-whitelistrelay and/or -whitelistforcerelay.")); } if (gArgs.IsArgSet("-blockminsize")) { InitWarning("Unsupported argument -blockminsize ignored."); } // Checkmempool and checkblockindex default to true in regtest mode int ratio = std::min( std::max( gArgs.GetArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0), 1000000); if (ratio != 0) { g_mempool.setSanityCheck(1.0 / ratio); } fCheckBlockIndex = gArgs.GetBoolArg("-checkblockindex", chainparams.DefaultConsistencyChecks()); fCheckpointsEnabled = gArgs.GetBoolArg("-checkpoints", DEFAULT_CHECKPOINTS_ENABLED); hashAssumeValid = uint256S( gArgs.GetArg("-assumevalid", chainparams.GetConsensus().defaultAssumeValid.GetHex())); if (!hashAssumeValid.IsNull()) { LogPrintf("Assuming ancestors of block %s have valid signatures.\n", hashAssumeValid.GetHex()); } else { LogPrintf("Validating signatures for all blocks.\n"); } if (gArgs.IsArgSet("-minimumchainwork")) { const std::string minChainWorkStr = gArgs.GetArg("-minimumchainwork", ""); if (!IsHexNumber(minChainWorkStr)) { return InitError(strprintf( "Invalid non-hex (%s) minimum chain work value specified", minChainWorkStr)); } nMinimumChainWork = UintToArith256(uint256S(minChainWorkStr)); } else { nMinimumChainWork = UintToArith256(chainparams.GetConsensus().nMinimumChainWork); } LogPrintf("Setting nMinimumChainWork=%s\n", nMinimumChainWork.GetHex()); if (nMinimumChainWork < UintToArith256(chainparams.GetConsensus().nMinimumChainWork)) { LogPrintf("Warning: nMinimumChainWork set below default value of %s\n", chainparams.GetConsensus().nMinimumChainWork.GetHex()); } // mempool limits int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t nMempoolSizeMin = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000 * 40; if (nMempoolSizeMax < 0 || nMempoolSizeMax < nMempoolSizeMin) { return InitError(strprintf(_("-maxmempool must be at least %d MB"), std::ceil(nMempoolSizeMin / 1000000.0))); } // -par=0 means autodetect, but nScriptCheckThreads==0 means no concurrency nScriptCheckThreads = gArgs.GetArg("-par", DEFAULT_SCRIPTCHECK_THREADS); if (nScriptCheckThreads <= 0) { nScriptCheckThreads += GetNumCores(); } if (nScriptCheckThreads <= 1) { nScriptCheckThreads = 0; } else if (nScriptCheckThreads > MAX_SCRIPTCHECK_THREADS) { nScriptCheckThreads = MAX_SCRIPTCHECK_THREADS; } // Configure excessive block size. const uint64_t nProposedExcessiveBlockSize = gArgs.GetArg("-excessiveblocksize", DEFAULT_MAX_BLOCK_SIZE); if (!config.SetMaxBlockSize(nProposedExcessiveBlockSize)) { return InitError( _("Excessive block size must be > 1,000,000 bytes (1MB)")); } // Check blockmaxsize does not exceed maximum accepted block size. const uint64_t nProposedMaxGeneratedBlockSize = gArgs.GetArg("-blockmaxsize", DEFAULT_MAX_GENERATED_BLOCK_SIZE); if (nProposedMaxGeneratedBlockSize > config.GetMaxBlockSize()) { auto msg = _("Max generated block size (blockmaxsize) cannot exceed " "the excessive block size (excessiveblocksize)"); return InitError(msg); } // block pruning; get the amount of disk space (in MiB) to allot for block & // undo files int64_t nPruneArg = gArgs.GetArg("-prune", 0); if (nPruneArg < 0) { return InitError( _("Prune cannot be configured with a negative value.")); } nPruneTarget = (uint64_t)nPruneArg * 1024 * 1024; if (nPruneArg == 1) { // manual pruning: -prune=1 LogPrintf("Block pruning enabled. Use RPC call " "pruneblockchain(height) to manually prune block and undo " "files.\n"); nPruneTarget = std::numeric_limits::max(); fPruneMode = true; } else if (nPruneTarget) { if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) { return InitError( strprintf(_("Prune configured below the minimum of %d MiB. " "Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)); } LogPrintf("Prune configured to target %uMiB on disk for block and undo " "files.\n", nPruneTarget / 1024 / 1024); fPruneMode = true; } RegisterAllRPCCommands(config, rpcServer, tableRPC); g_wallet_init_interface->RegisterRPC(tableRPC); #ifdef ENABLE_WALLET RegisterDumpRPCCommands(tableRPC); #endif nConnectTimeout = gArgs.GetArg("-timeout", DEFAULT_CONNECT_TIMEOUT); if (nConnectTimeout <= 0) { nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; } // Obtain the amount to charge excess UTXO if (gArgs.IsArgSet("-excessutxocharge")) { Amount n = Amount::zero(); auto parsed = ParseMoney(gArgs.GetArg("-excessutxocharge", ""), n); if (!parsed || Amount::zero() > n) { return InitError(AmountErrMsg( "excessutxocharge", gArgs.GetArg("-excessutxocharge", ""))); } config.SetExcessUTXOCharge(n); } else { config.SetExcessUTXOCharge(DEFAULT_UTXO_FEE); } // Fee-per-kilobyte amount considered the same as "free". If you are mining, // be careful setting this: if you set it to zero then a transaction spammer // can cheaply fill blocks using 1-satoshi-fee transactions. It should be // set above the real cost to you of processing a transaction. if (gArgs.IsArgSet("-minrelaytxfee")) { Amount n = Amount::zero(); auto parsed = ParseMoney(gArgs.GetArg("-minrelaytxfee", ""), n); if (!parsed || Amount::zero() == n) { return InitError(AmountErrMsg("minrelaytxfee", gArgs.GetArg("-minrelaytxfee", ""))); } // High fee check is done afterward in WalletParameterInteraction() config.SetMinFeePerKB(CFeeRate(n)); } else { config.SetMinFeePerKB(CFeeRate(DEFAULT_MIN_RELAY_TX_FEE_PER_KB)); } // Sanity check argument for min fee for including tx in block // TODO: Harmonize which arguments need sanity checking and where that // happens. if (gArgs.IsArgSet("-blockmintxfee")) { Amount n = Amount::zero(); if (!ParseMoney(gArgs.GetArg("-blockmintxfee", ""), n)) { return InitError(AmountErrMsg("blockmintxfee", gArgs.GetArg("-blockmintxfee", ""))); } } // Feerate used to define dust. Shouldn't be changed lightly as old // implementations may inadvertently create non-standard transactions. if (gArgs.IsArgSet("-dustrelayfee")) { Amount n = Amount::zero(); auto parsed = ParseMoney(gArgs.GetArg("-dustrelayfee", ""), n); if (!parsed || Amount::zero() == n) { return InitError(AmountErrMsg("dustrelayfee", gArgs.GetArg("-dustrelayfee", ""))); } dustRelayFee = CFeeRate(n); } fRequireStandard = !gArgs.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); if (chainparams.RequireStandard() && !fRequireStandard) { return InitError( strprintf("acceptnonstdtxn is not currently supported for %s chain", chainparams.NetworkIDString())); } nBytesPerSigOp = gArgs.GetArg("-bytespersigop", nBytesPerSigOp); if (!g_wallet_init_interface->ParameterInteraction()) { return false; } fIsBareMultisigStd = gArgs.GetBoolArg("-permitbaremultisig", DEFAULT_PERMIT_BAREMULTISIG); fAcceptDatacarrier = gArgs.GetBoolArg("-datacarrier", DEFAULT_ACCEPT_DATACARRIER); // Option to startup with mocktime set (used for regression testing): SetMockTime(gArgs.GetArg("-mocktime", 0)); // SetMockTime(0) is a no-op if (gArgs.GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) { nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); } // Signal Bitcoin Cash support. // TODO: remove some time after the hardfork when no longer needed // to differentiate the network nodes. nLocalServices = ServiceFlags(nLocalServices | NODE_BITCOIN_CASH); nMaxTipAge = gArgs.GetArg("-maxtipage", DEFAULT_MAX_TIP_AGE); return true; } static bool LockDataDirectory(bool probeOnly) { std::string strDataDir = GetDataDir().string(); // Make sure only a single Bitcoin process is using the data directory. fs::path pathLockFile = GetDataDir() / ".lock"; // empty lock file; created if it doesn't exist. FILE *file = fsbridge::fopen(pathLockFile, "a"); if (file) { fclose(file); } try { static boost::interprocess::file_lock lock( pathLockFile.string().c_str()); if (!lock.try_lock()) { return InitError( strprintf(_("Cannot obtain a lock on data directory %s. %s is " "probably already running."), strDataDir, _(PACKAGE_NAME))); } if (probeOnly) { lock.unlock(); } } catch (const boost::interprocess::interprocess_exception &e) { return InitError(strprintf(_("Cannot obtain a lock on data directory " "%s. %s is probably already running.") + " %s.", strDataDir, _(PACKAGE_NAME), e.what())); } return true; } bool AppInitSanityChecks() { // Step 4: sanity checks // Initialize elliptic curve code std::string sha256_algo = SHA256AutoDetect(); LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo); RandomInit(); ECC_Start(); globalVerifyHandle.reset(new ECCVerifyHandle()); // Sanity check if (!InitSanityCheck()) { return InitError(strprintf( _("Initialization sanity check failed. %s is shutting down."), _(PACKAGE_NAME))); } // Probe the data directory lock to give an early error message, if possible // We cannot hold the data directory lock here, as the forking for daemon() // hasn't yet happened, and a fork will cause weird behavior to it. return LockDataDirectory(true); } bool AppInitLockDataDirectory() { // After daemonization get the data directory lock again and hold on to it // until exit. This creates a slight window for a race condition to happen, // however this condition is harmless: it will at most make us exit without // printing a message to console. if (!LockDataDirectory(false)) { // Detailed error printed inside LockDataDirectory return false; } return true; } bool AppInitMain(Config &config, HTTPRPCRequestProcessor &httpRPCRequestProcessor) { // Step 4a: application initialization const CChainParams &chainparams = config.GetChainParams(); #ifndef WIN32 CreatePidFile(GetPidFile(), getpid()); #endif BCLog::Logger &logger = GetLogger(); bool default_shrinkdebugfile = logger.DefaultShrinkDebugFile(); if (gArgs.GetBoolArg("-shrinkdebugfile", default_shrinkdebugfile)) { // Do this first since it both loads a bunch of debug.log into memory, // and because this needs to happen before any other debug.log printing. logger.ShrinkDebugFile(); } if (logger.m_print_to_file) { if (!logger.OpenDebugLog()) { return InitError(strprintf("Could not open debug log file %s", logger.GetDebugLogPath().string())); } } if (!logger.m_log_timestamps) { LogPrintf("Startup time: %s\n", DateTimeStrFormat("%Y-%m-%d %H:%M:%S", GetTime())); } LogPrintf("Default data directory %s\n", GetDefaultDataDir().string()); LogPrintf("Using data directory %s\n", GetDataDir().string()); LogPrintf( "Using config file %s\n", GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME)).string()); LogPrintf("Using at most %i automatic connections (%i file descriptors " "available)\n", nMaxConnections, nFD); // Warn about relative -datadir path. if (gArgs.IsArgSet("-datadir") && !fs::path(gArgs.GetArg("-datadir", "")).is_absolute()) { LogPrintf("Warning: relative datadir option '%s' specified, which will " "be interpreted relative to the current working directory " "'%s'. This is fragile, because if bitcoin is started in the " "future from a different location, it will be unable to " "locate the current data files. There could also be data " "loss if bitcoin is started while in a temporary " "directory.\n", gArgs.GetArg("-datadir", ""), fs::current_path().string()); } InitSignatureCache(); InitScriptExecutionCache(); LogPrintf("Using %u threads for script verification\n", nScriptCheckThreads); if (nScriptCheckThreads) { for (int i = 0; i < nScriptCheckThreads - 1; i++) { threadGroup.create_thread(&ThreadScriptCheck); } } // Start the lightweight task scheduler thread CScheduler::Function serviceLoop = boost::bind(&CScheduler::serviceQueue, &scheduler); threadGroup.create_thread(boost::bind(&TraceThread, "scheduler", serviceLoop)); GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); GetMainSignals().RegisterWithMempoolSignals(g_mempool); /** * Start the RPC server. It will be started in "warmup" mode and not * process calls yet (but it will verify that the server is there and will * be ready later). Warmup mode will be completed when initialisation is * finished. */ if (gArgs.GetBoolArg("-server", false)) { uiInterface.InitMessage.connect(SetRPCWarmupStatus); if (!AppInitServers(config, httpRPCRequestProcessor)) { return InitError( _("Unable to start HTTP server. See debug log for details.")); } } // Step 5: verify wallet database integrity if (!g_wallet_init_interface->Verify(chainparams)) { return false; } // Step 6: network initialization // Note that we absolutely cannot open any actual connections // until the very end ("start node") as the UTXO/block state // is not yet setup and may end up being set up twice if we // need to reindex later. assert(!g_connman); g_connman = std::unique_ptr( new CConnman(config, GetRand(std::numeric_limits::max()), GetRand(std::numeric_limits::max()))); CConnman &connman = *g_connman; peerLogic.reset(new PeerLogicValidation(&connman, scheduler)); RegisterValidationInterface(peerLogic.get()); // sanitize comments per BIP-0014, format user agent and check total size std::vector uacomments; for (const std::string &cmt : gArgs.GetArgs("-uacomment")) { if (cmt != SanitizeString(cmt, SAFE_CHARS_UA_COMMENT)) return InitError(strprintf( _("User Agent comment (%s) contains unsafe characters."), cmt)); uacomments.push_back(cmt); } const std::string strSubVersion = FormatSubVersion(CLIENT_NAME, CLIENT_VERSION, uacomments); if (strSubVersion.size() > MAX_SUBVERSION_LENGTH) { return InitError(strprintf( _("Total length of network version string (%i) exceeds maximum " "length (%i). Reduce the number or size of uacomments."), strSubVersion.size(), MAX_SUBVERSION_LENGTH)); } if (gArgs.IsArgSet("-onlynet")) { std::set nets; for (const std::string &snet : gArgs.GetArgs("-onlynet")) { enum Network net = ParseNetwork(snet); if (net == NET_UNROUTABLE) { return InitError(strprintf( _("Unknown network specified in -onlynet: '%s'"), snet)); } nets.insert(net); } for (int n = 0; n < NET_MAX; n++) { enum Network net = (enum Network)n; if (!nets.count(net)) SetLimited(net); } } // Check for host lookup allowed before parsing any network related // parameters fNameLookup = gArgs.GetBoolArg("-dns", DEFAULT_NAME_LOOKUP); bool proxyRandomize = gArgs.GetBoolArg("-proxyrandomize", DEFAULT_PROXYRANDOMIZE); // -proxy sets a proxy for all outgoing network traffic // -noproxy (or -proxy=0) as well as the empty string can be used to not set // a proxy, this is the default std::string proxyArg = gArgs.GetArg("-proxy", ""); SetLimited(NET_TOR); if (proxyArg != "" && proxyArg != "0") { CService proxyAddr; if (!Lookup(proxyArg.c_str(), proxyAddr, 9050, fNameLookup)) { return InitError(strprintf( _("Invalid -proxy address or hostname: '%s'"), proxyArg)); } proxyType addrProxy = proxyType(proxyAddr, proxyRandomize); if (!addrProxy.IsValid()) { return InitError(strprintf( _("Invalid -proxy address or hostname: '%s'"), proxyArg)); } SetProxy(NET_IPV4, addrProxy); SetProxy(NET_IPV6, addrProxy); SetProxy(NET_TOR, addrProxy); SetNameProxy(addrProxy); // by default, -proxy sets onion as reachable, unless -noonion later SetLimited(NET_TOR, false); } // -onion can be used to set only a proxy for .onion, or override normal // proxy for .onion addresses. // -noonion (or -onion=0) disables connecting to .onion entirely. An empty // string is used to not override the onion proxy (in which case it defaults // to -proxy set above, or none) std::string onionArg = gArgs.GetArg("-onion", ""); if (onionArg != "") { if (onionArg == "0") { // Handle -noonion/-onion=0 SetLimited(NET_TOR); // set onions as unreachable } else { CService onionProxy; if (!Lookup(onionArg.c_str(), onionProxy, 9050, fNameLookup)) { return InitError(strprintf( _("Invalid -onion address or hostname: '%s'"), onionArg)); } proxyType addrOnion = proxyType(onionProxy, proxyRandomize); if (!addrOnion.IsValid()) { return InitError(strprintf( _("Invalid -onion address or hostname: '%s'"), onionArg)); } SetProxy(NET_TOR, addrOnion); SetLimited(NET_TOR, false); } } // see Step 2: parameter interactions for more information about these fListen = gArgs.GetBoolArg("-listen", DEFAULT_LISTEN); fDiscover = gArgs.GetBoolArg("-discover", true); fRelayTxes = !gArgs.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY); for (const std::string &strAddr : gArgs.GetArgs("-externalip")) { CService addrLocal; if (Lookup(strAddr.c_str(), addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) { AddLocal(addrLocal, LOCAL_MANUAL); } else { return InitError(ResolveErrMsg("externalip", strAddr)); } } #if ENABLE_ZMQ pzmqNotificationInterface = CZMQNotificationInterface::Create(); if (pzmqNotificationInterface) { RegisterValidationInterface(pzmqNotificationInterface); } #endif // unlimited unless -maxuploadtarget is set uint64_t nMaxOutboundLimit = 0; uint64_t nMaxOutboundTimeframe = MAX_UPLOAD_TIMEFRAME; if (gArgs.IsArgSet("-maxuploadtarget")) { nMaxOutboundLimit = gArgs.GetArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET) * 1024 * 1024; } // Step 7: load block chain fReindex = gArgs.GetBoolArg("-reindex", false); bool fReindexChainState = gArgs.GetBoolArg("-reindex-chainstate", false); // cache size calculations int64_t nTotalCache = (gArgs.GetArg("-dbcache", nDefaultDbCache) << 20); // total cache cannot be less than nMinDbCache nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be greater than nMaxDbcache nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); int64_t nBlockTreeDBCache = nTotalCache / 8; nBlockTreeDBCache = std::min(nBlockTreeDBCache, (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxBlockDBAndTxIndexCache : nMaxBlockDBCache) << 20); nTotalCache -= nBlockTreeDBCache; // use 25%-50% of the remainder for disk cache int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // cap total coins db cache nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); nTotalCache -= nCoinDBCache; // the rest goes to in-memory cache nCoinCacheUsage = nTotalCache; int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; LogPrintf("Cache configuration:\n"); LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for in-memory UTXO set (plus up to %.1fMiB of " "unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024)); int64_t nStart = 0; bool fLoaded = false; while (!fLoaded && !fRequestShutdown) { bool fReset = fReindex; std::string strLoadError; uiInterface.InitMessage(_("Loading block index...")); nStart = GetTimeMillis(); do { try { UnloadBlockIndex(); pcoinsTip.reset(); pcoinsdbview.reset(); pcoinscatcher.reset(); pblocktree.reset( new CBlockTreeDB(nBlockTreeDBCache, false, fReset)); if (fReindex) { pblocktree->WriteReindexing(true); // If we're reindexing in prune mode, wipe away unusable // block files and all undo data files if (fPruneMode) { CleanupBlockRevFiles(); } } if (fRequestShutdown) { break; } // LoadBlockIndex will load fTxIndex from the db, or set it if // we're reindexing. It will also load fHavePruned if we've // ever removed a block file from disk. if (!LoadBlockIndex(config)) { strLoadError = _("Error loading block database"); break; } // If the loaded chain has a wrong genesis, bail out immediately // (we're likely using a testnet datadir, or the other way // around). if (!mapBlockIndex.empty() && mapBlockIndex.count( chainparams.GetConsensus().hashGenesisBlock) == 0) { return InitError(_("Incorrect or no genesis block found. " "Wrong datadir for network?")); } // Check for changed -txindex state if (fTxIndex != gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { strLoadError = _("You need to rebuild the database using " "-reindex-chainstate to change -txindex"); break; } // Check for changed -prune state. What we are concerned about // is a user who has pruned blocks in the past, but is now // trying to run unpruned. if (fHavePruned && !fPruneMode) { strLoadError = _("You need to rebuild the database using -reindex to " "go back to unpruned mode. This will redownload the " "entire blockchain"); break; } // At this point blocktree args are consistent with what's on // disk. If we're not mid-reindex (based on disk + args), add a // genesis block on disk. This is called again in ThreadImport // if the reindex completes. if (!fReindex && !LoadGenesisBlock(chainparams)) { strLoadError = _("Error initializing block database"); break; } // At this point we're either in reindex or we've loaded a // useful block tree into mapBlockIndex! pcoinsdbview.reset(new CCoinsViewDB( nCoinDBCache, false, fReset || fReindexChainState)); pcoinscatcher.reset( new CCoinsViewErrorCatcher(pcoinsdbview.get())); // If necessary, upgrade from older database format. // This is a no-op if we cleared the coinsviewdb with -reindex // or -reindex-chainstate if (!pcoinsdbview->Upgrade()) { strLoadError = _("Error upgrading chainstate database"); break; } // ReplayBlocks is a no-op if we cleared the coinsviewdb with // -reindex or -reindex-chainstate if (!ReplayBlocks(config, pcoinsdbview.get())) { strLoadError = _("Unable to replay blocks. You will need to rebuild " "the database using -reindex-chainstate."); break; } // The on-disk coinsdb is now in a good state, create the cache pcoinsTip.reset(new CCoinsViewCache(pcoinscatcher.get())); if (!fReindex && !fReindexChainState) { // LoadChainTip sets chainActive based on pcoinsTip's best // block if (!LoadChainTip(config)) { strLoadError = _("Error initializing block database"); break; } assert(chainActive.Tip() != nullptr); } if (!fReindex) { // Note that RewindBlockIndex MUST run even if we're about // to -reindex-chainstate. It both disconnects blocks based // on chainActive, and drops block data in mapBlockIndex // based on lack of available witness data. uiInterface.InitMessage(_("Rewinding blocks...")); if (!RewindBlockIndex(config)) { strLoadError = _("Unable to rewind the database to a " "pre-fork state. You will need to " "redownload the blockchain"); break; } } if (!fReindex && !fReindexChainState) { uiInterface.InitMessage(_("Verifying blocks...")); if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { LogPrintf("Prune: pruned datadir may not have more " "than %d blocks; only checking available " "blocks", MIN_BLOCKS_TO_KEEP); } { LOCK(cs_main); CBlockIndex *tip = chainActive.Tip(); RPCNotifyBlockChange(true, tip); if (tip && tip->nTime > GetAdjustedTime() + MAX_FUTURE_BLOCK_TIME) { strLoadError = _("The block database contains a block which " "appears to be from the future. This may be " "due to your computer's date and time being " "set incorrectly. Only rebuild the block " "database if you are sure that your " "computer's date and time are correct"); break; } } if (!CVerifyDB().VerifyDB( config, pcoinsdbview.get(), gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { strLoadError = _("Corrupted block database detected"); break; } } } catch (const std::exception &e) { LogPrintf("%s\n", e.what()); strLoadError = _("Error opening block database"); break; } fLoaded = true; } while (false); if (!fLoaded && !fRequestShutdown) { // first suggest a reindex if (!fReset) { bool fRet = uiInterface.ThreadSafeQuestion( strLoadError + ".\n\n" + _("Do you want to rebuild the block database now?"), strLoadError + ".\nPlease restart with -reindex or " "-reindex-chainstate to recover.", "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); if (fRet) { fReindex = true; fRequestShutdown = false; } else { LogPrintf("Aborted block database rebuild. Exiting.\n"); return false; } } else { return InitError(strLoadError); } } } // As LoadBlockIndex can take several minutes, it's possible the user // requested to kill the GUI during the last operation. If so, exit. // As the program has not fully started yet, Shutdown() is possibly // overkill. if (fRequestShutdown) { LogPrintf("Shutdown requested. Exiting.\n"); return false; } LogPrintf(" block index %15dms\n", GetTimeMillis() - nStart); - fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; - CAutoFile est_filein(fsbridge::fopen(est_path, "rb"), SER_DISK, - CLIENT_VERSION); - // Allowed to fail as this file IS missing on first startup. - if (!est_filein.IsNull()) { - g_mempool.ReadFeeEstimates(est_filein); - } - fFeeEstimatesInitialized = true; - // Encoded addresses using cashaddr instead of base58 // Activates by default on Jan, 14 config.SetCashAddrEncoding( gArgs.GetBoolArg("-usecashaddr", GetAdjustedTime() > 1515900000)); // Step 8: load wallet if (!g_wallet_init_interface->Open(chainparams)) { return false; } // Step 9: data directory maintenance // if pruning, unset the service bit and perform the initial blockstore // prune after any wallet rescanning has taken place. if (fPruneMode) { LogPrintf("Unsetting NODE_NETWORK on prune mode\n"); nLocalServices = ServiceFlags(nLocalServices & ~NODE_NETWORK); if (!fReindex) { uiInterface.InitMessage(_("Pruning blockstore...")); PruneAndFlush(); } } // Step 10: import blocks if (!CheckDiskSpace()) { return false; } // Either install a handler to notify us when genesis activates, or set // fHaveGenesis directly. // No locking, as this happens before any background thread is started. if (chainActive.Tip() == nullptr) { uiInterface.NotifyBlockTip.connect(BlockNotifyGenesisWait); } else { fHaveGenesis = true; } if (gArgs.IsArgSet("-blocknotify")) { uiInterface.NotifyBlockTip.connect(BlockNotifyCallback); } std::vector vImportFiles; for (const std::string &strFile : gArgs.GetArgs("-loadblock")) { vImportFiles.push_back(strFile); } threadGroup.create_thread( boost::bind(&ThreadImport, std::ref(config), vImportFiles)); // Wait for genesis block to be processed { WAIT_LOCK(cs_GenesisWait, lock); // We previously could hang here if StartShutdown() is called prior to // ThreadImport getting started, so instead we just wait on a timer to // check ShutdownRequested() regularly. while (!fHaveGenesis && !ShutdownRequested()) { condvar_GenesisWait.wait_for(lock, std::chrono::milliseconds(500)); } uiInterface.NotifyBlockTip.disconnect(BlockNotifyGenesisWait); } // Step 11: start node int chain_active_height; //// debug print { LOCK(cs_main); LogPrintf("mapBlockIndex.size() = %u\n", mapBlockIndex.size()); chain_active_height = chainActive.Height(); } LogPrintf("nBestHeight = %d\n", chain_active_height); if (gArgs.GetBoolArg("-listenonion", DEFAULT_LISTEN_ONION)) { StartTorControl(); } Discover(); // Map ports with UPnP if (gArgs.GetBoolArg("-upnp", DEFAULT_UPNP)) { StartMapPort(); } CConnman::Options connOptions; connOptions.nLocalServices = nLocalServices; connOptions.nMaxConnections = nMaxConnections; connOptions.nMaxOutbound = std::min(MAX_OUTBOUND_CONNECTIONS, connOptions.nMaxConnections); connOptions.nMaxAddnode = MAX_ADDNODE_CONNECTIONS; connOptions.nMaxFeeler = 1; connOptions.nBestHeight = chain_active_height; connOptions.uiInterface = &uiInterface; connOptions.m_msgproc = peerLogic.get(); connOptions.nSendBufferMaxSize = 1000 * gArgs.GetArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); connOptions.nReceiveFloodSize = 1000 * gArgs.GetArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); connOptions.m_added_nodes = gArgs.GetArgs("-addnode"); connOptions.nMaxOutboundTimeframe = nMaxOutboundTimeframe; connOptions.nMaxOutboundLimit = nMaxOutboundLimit; for (const std::string &strBind : gArgs.GetArgs("-bind")) { CService addrBind; if (!Lookup(strBind.c_str(), addrBind, GetListenPort(), false)) { return InitError(ResolveErrMsg("bind", strBind)); } connOptions.vBinds.push_back(addrBind); } for (const std::string &strBind : gArgs.GetArgs("-whitebind")) { CService addrBind; if (!Lookup(strBind.c_str(), addrBind, 0, false)) { return InitError(ResolveErrMsg("whitebind", strBind)); } if (addrBind.GetPort() == 0) { return InitError(strprintf( _("Need to specify a port with -whitebind: '%s'"), strBind)); } connOptions.vWhiteBinds.push_back(addrBind); } for (const auto &net : gArgs.GetArgs("-whitelist")) { CSubNet subnet; LookupSubNet(net.c_str(), subnet); if (!subnet.IsValid()) { return InitError(strprintf( _("Invalid netmask specified in -whitelist: '%s'"), net)); } connOptions.vWhitelistedRange.push_back(subnet); } connOptions.vSeedNodes = gArgs.GetArgs("-seednode"); // Initiate outbound connections unless connect=0 connOptions.m_use_addrman_outgoing = !gArgs.IsArgSet("-connect"); if (!connOptions.m_use_addrman_outgoing) { const auto connect = gArgs.GetArgs("-connect"); if (connect.size() != 1 || connect[0] != "0") { connOptions.m_specified_outgoing = connect; } } if (!connman.Start(scheduler, connOptions)) { return false; } // Step 12: finished SetRPCWarmupFinished(); uiInterface.InitMessage(_("Done loading")); g_wallet_init_interface->Start(scheduler); return !fRequestShutdown; } diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 699b0170b0..85135747ff 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -1,558 +1,29 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Copyright (c) 2018 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "policy/fees.h" -#include "policy/policy.h" - #include "amount.h" -#include "primitives/transaction.h" -#include "random.h" -#include "streams.h" -#include "txmempool.h" -#include "util.h" - -void TxConfirmStats::Initialize(std::vector &defaultBuckets, - unsigned int maxConfirms, double _decay) { - decay = _decay; - for (size_t i = 0; i < defaultBuckets.size(); i++) { - buckets.push_back(defaultBuckets[i]); - bucketMap[defaultBuckets[i]] = i; - } - confAvg.resize(maxConfirms); - curBlockConf.resize(maxConfirms); - unconfTxs.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - confAvg[i].resize(buckets.size()); - curBlockConf[i].resize(buckets.size()); - unconfTxs[i].resize(buckets.size()); - } - - oldUnconfTxs.resize(buckets.size()); - curBlockTxCt.resize(buckets.size()); - txCtAvg.resize(buckets.size()); - curBlockVal.resize(buckets.size()); - avg.resize(buckets.size()); -} - -// Zero out the data for the current block -void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight) { - for (size_t j = 0; j < buckets.size(); j++) { - oldUnconfTxs[j] += unconfTxs[nBlockHeight % unconfTxs.size()][j]; - unconfTxs[nBlockHeight % unconfTxs.size()][j] = 0; - for (size_t i = 0; i < curBlockConf.size(); i++) { - curBlockConf[i][j] = 0; - } - curBlockTxCt[j] = 0; - curBlockVal[j] = 0; - } -} - -void TxConfirmStats::Record(int blocksToConfirm, double val) { - // blocksToConfirm is 1-based - if (blocksToConfirm < 1) { - return; - } - unsigned int bucketindex = bucketMap.lower_bound(val)->second; - for (size_t i = blocksToConfirm; i <= curBlockConf.size(); i++) { - curBlockConf[i - 1][bucketindex]++; - } - curBlockTxCt[bucketindex]++; - curBlockVal[bucketindex] += val; -} - -void TxConfirmStats::UpdateMovingAverages() { - for (unsigned int j = 0; j < buckets.size(); j++) { - for (unsigned int i = 0; i < confAvg.size(); i++) { - confAvg[i][j] = confAvg[i][j] * decay + curBlockConf[i][j]; - } - avg[j] = avg[j] * decay + curBlockVal[j]; - txCtAvg[j] = txCtAvg[j] * decay + curBlockTxCt[j]; - } -} - -// returns -1 on error conditions -CFeeRate TxConfirmStats::EstimateMedianFeeRate(int confTarget, - double sufficientTxVal, - double successBreakPoint, - bool requireGreater, - unsigned int nBlockHeight) { - // Counters for a bucket (or range of buckets) - // Number of tx's confirmed within the confTarget - double nConf = 0; - // Total number of tx's that were ever confirmed - double totalNum = 0; - // Number of tx's still in mempool for confTarget or longer - int extraNum = 0; - - int maxbucketindex = buckets.size() - 1; - - // requireGreater means we are looking for the lowest feerate such that all - // higher values pass, so we start at maxbucketindex (highest feerate) and - // look at successively smaller buckets until we reach failure. Otherwise, - // we are looking for the highest feerate such that all lower values fail, - // and we go in the opposite direction. - unsigned int startbucket = requireGreater ? maxbucketindex : 0; - int step = requireGreater ? -1 : 1; - - // We'll combine buckets until we have enough samples. - // The near and far variables will define the range we've combined - // The best variables are the last range we saw which still had a high - // enough confirmation rate to count as success. - // The cur variables are the current range we're counting. - unsigned int curNearBucket = startbucket; - unsigned int bestNearBucket = startbucket; - unsigned int curFarBucket = startbucket; - unsigned int bestFarBucket = startbucket; - - bool foundAnswer = false; - unsigned int bins = unconfTxs.size(); - - // Start counting from highest(default) or lowest feerate transactions - for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; - bucket += step) { - curFarBucket = bucket; - nConf += confAvg[confTarget - 1][bucket]; - totalNum += txCtAvg[bucket]; - for (unsigned int confct = confTarget; confct < GetMaxConfirms(); - confct++) { - extraNum += unconfTxs[(nBlockHeight - confct) % bins][bucket]; - } - - extraNum += oldUnconfTxs[bucket]; - // If we have enough transaction data points in this range of buckets, - // we can test for success (Only count the confirmed data points, so - // that each confirmation count will be looking at the same amount of - // data and same bucket breaks) - if (totalNum >= sufficientTxVal / (1 - decay)) { - double curPct = nConf / (totalNum + extraNum); - - // Check to see if we are no longer getting confirmed at the success - // rate - if (requireGreater && curPct < successBreakPoint) { - break; - } - if (!requireGreater && curPct > successBreakPoint) { - break; - } - - // Otherwise update the cumulative stats, and the bucket variables - // and reset the counters - foundAnswer = true; - nConf = 0; - totalNum = 0; - extraNum = 0; - bestNearBucket = curNearBucket; - bestFarBucket = curFarBucket; - curNearBucket = bucket + step; - } - } - - double median = -1; - double txSum = 0; - - // Calculate the "average" feerate of the best bucket range that met success - // conditions. Find the bucket with the median transaction and then report - // the average feerate from that bucket. This is a compromise between - // finding the median which we can't since we don't save all tx's and - // reporting the average which is less accurate - unsigned int minBucket = - bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; - unsigned int maxBucket = - bestNearBucket > bestFarBucket ? bestNearBucket : bestFarBucket; - for (unsigned int j = minBucket; j <= maxBucket; j++) { - txSum += txCtAvg[j]; - } - if (foundAnswer && txSum != 0) { - txSum = txSum / 2; - for (unsigned int j = minBucket; j <= maxBucket; j++) { - if (txCtAvg[j] < txSum) { - txSum -= txCtAvg[j]; - } else { - // we're in the right bucket - median = avg[j] / txCtAvg[j]; - break; - } - } - } - - LogPrint( - BCLog::ESTIMATEFEE, - "%3d: For conf success %s %4.2f need feerate %s: %12.5g from buckets " - "%8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", - confTarget, requireGreater ? ">" : "<", successBreakPoint, - requireGreater ? ">" : "<", median, buckets[minBucket], - buckets[maxBucket], 100 * nConf / (totalNum + extraNum), nConf, - totalNum, extraNum); - - return CFeeRate(int64_t(ceill(median)) * SATOSHI); -} - -void TxConfirmStats::Write(CAutoFile &fileout) { - fileout << decay; - fileout << buckets; - fileout << avg; - fileout << txCtAvg; - fileout << confAvg; -} - -void TxConfirmStats::Read(CAutoFile &filein) { - // Read data file into temporary variables and do some very basic sanity - // checking - std::vector fileBuckets; - std::vector fileAvg; - std::vector> fileConfAvg; - std::vector fileTxCtAvg; - double fileDecay; - size_t maxConfirms; - size_t numBuckets; - - filein >> fileDecay; - if (fileDecay <= 0 || fileDecay >= 1) { - throw std::runtime_error("Corrupt estimates file. Decay must be " - "between 0 and 1 (non-inclusive)"); - } - filein >> fileBuckets; - numBuckets = fileBuckets.size(); - if (numBuckets <= 1 || numBuckets > 1000) { - throw std::runtime_error("Corrupt estimates file. Must have between 2 " - "and 1000 feerate buckets"); - } - filein >> fileAvg; - if (fileAvg.size() != numBuckets) { - throw std::runtime_error( - "Corrupt estimates file. Mismatch in feerate average bucket count"); - } - filein >> fileTxCtAvg; - if (fileTxCtAvg.size() != numBuckets) { - throw std::runtime_error( - "Corrupt estimates file. Mismatch in tx count bucket count"); - } - filein >> fileConfAvg; - maxConfirms = fileConfAvg.size(); - if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) { - // one week - throw std::runtime_error("Corrupt estimates file. Must maintain " - "estimates for between 1 and 1008 (one week) " - "confirms"); - } - for (unsigned int i = 0; i < maxConfirms; i++) { - if (fileConfAvg[i].size() != numBuckets) { - throw std::runtime_error("Corrupt estimates file. Mismatch in " - "feerate conf average bucket count"); - } - } - // Now that we've processed the entire feerate estimate data file and not - // thrown any errors, we can copy it to our data structures - decay = fileDecay; - buckets = fileBuckets; - avg = fileAvg; - confAvg = fileConfAvg; - txCtAvg = fileTxCtAvg; - bucketMap.clear(); - - // Resize the current block variables which aren't stored in the data file - // to match the number of confirms and buckets - curBlockConf.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - curBlockConf[i].resize(buckets.size()); - } - curBlockTxCt.resize(buckets.size()); - curBlockVal.resize(buckets.size()); - - unconfTxs.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - unconfTxs[i].resize(buckets.size()); - } - oldUnconfTxs.resize(buckets.size()); - - for (size_t i = 0; i < buckets.size(); i++) { - bucketMap[buckets[i]] = i; - } - - LogPrint( - BCLog::ESTIMATEFEE, - "Reading estimates: %u buckets counting confirms up to %u blocks\n", - numBuckets, maxConfirms); -} - -unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val) { - unsigned int bucketindex = bucketMap.lower_bound(val)->second; - unsigned int blockIndex = nBlockHeight % unconfTxs.size(); - unconfTxs[blockIndex][bucketindex]++; - return bucketindex; -} - -void TxConfirmStats::removeTx(unsigned int entryHeight, - unsigned int nBestSeenHeight, - unsigned int bucketindex) { - // nBestSeenHeight is not updated yet for the new block - int blocksAgo = nBestSeenHeight - entryHeight; - if (nBestSeenHeight == 0) { - // the BlockPolicyEstimator hasn't seen any blocks yet - blocksAgo = 0; - } - if (blocksAgo < 0) { - // This can't happen because we call this with our best seen height, no - // entries can have higher - LogPrint(BCLog::ESTIMATEFEE, - "Blockpolicy error, blocks ago is negative for mempool tx\n"); - return; - } - - if (blocksAgo >= (int)unconfTxs.size()) { - if (oldUnconfTxs[bucketindex] > 0) { - oldUnconfTxs[bucketindex]--; - } else { - LogPrint(BCLog::ESTIMATEFEE, - "Blockpolicy error, mempool tx removed from >25 blocks, " - "bucketIndex=%u already\n", - bucketindex); - } - } else { - unsigned int blockIndex = entryHeight % unconfTxs.size(); - if (unconfTxs[blockIndex][bucketindex] > 0) { - unconfTxs[blockIndex][bucketindex]--; - } else { - LogPrint(BCLog::ESTIMATEFEE, - "Blockpolicy error, mempool tx removed from " - "blockIndex=%u,bucketIndex=%u already\n", - blockIndex, bucketindex); - } - } -} - -// This function is called from CTxMemPool::removeUnchecked to ensure txs -// removed from the mempool for any reason are no longer tracked. Txs that were -// part of a block have already been removed in processBlockTx to ensure they -// are never double tracked, but it is of no harm to try to remove them again. -bool CBlockPolicyEstimator::removeTx(uint256 hash) { - std::map::iterator pos = mapMemPoolTxs.find(hash); - if (pos == mapMemPoolTxs.end()) { - return false; - } - - feeStats.removeTx(pos->second.blockHeight, nBestSeenHeight, - pos->second.bucketIndex); - mapMemPoolTxs.erase(hash); - return true; -} - -CBlockPolicyEstimator::CBlockPolicyEstimator() - : nBestSeenHeight(0), trackedTxs(0), untrackedTxs(0) { - static_assert(MIN_FEERATE > Amount::zero(), "Min feerate must be nonzero"); - CFeeRate minFeeRate(MIN_FEERATE); - std::vector vfeelist; - for (double bucketBoundary = minFeeRate.GetFeePerK() / SATOSHI; - bucketBoundary <= double(MAX_FEERATE / SATOSHI); - bucketBoundary *= FEE_SPACING) { - vfeelist.push_back(bucketBoundary); - } - - vfeelist.push_back(double(INF_FEERATE / SATOSHI)); - feeStats.Initialize(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY); -} - -void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry &entry, - bool validFeeEstimate) { - uint32_t txHeight = entry.GetHeight(); - uint256 txid = entry.GetTx().GetId(); - if (mapMemPoolTxs.count(txid)) { - LogPrint(BCLog::ESTIMATEFEE, - "Blockpolicy error mempool tx %s already being tracked\n", - txid.ToString().c_str()); - return; - } - - if (txHeight != nBestSeenHeight) { - // Ignore side chains and re-orgs; assuming they are random they don't - // affect the estimate. We'll potentially double count transactions in - // 1-block reorgs. Ignore txs if BlockPolicyEstimator is not in sync - // with chainActive.Tip(). It will be synced next time a block is - // processed. - return; - } - - // Only want to be updating estimates when our blockchain is synced, - // otherwise we'll miscalculate how many blocks its taking to get included. - if (!validFeeEstimate) { - untrackedTxs++; - return; - } - trackedTxs++; - - // Feerates are stored and reported as BCH-per-kb: - CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); - - mapMemPoolTxs[txid].blockHeight = txHeight; - mapMemPoolTxs[txid].bucketIndex = - feeStats.NewTx(txHeight, double(feeRate.GetFeePerK() / SATOSHI)); -} - -bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, - const CTxMemPoolEntry *entry) { - if (!removeTx(entry->GetTx().GetId())) { - // This transaction wasn't being tracked for fee estimation - return false; - } - - // How many blocks did it take for miners to include this transaction? - // blocksToConfirm is 1-based, so a transaction included in the earliest - // possible block has confirmation count of 1 - int blocksToConfirm = nBlockHeight - entry->GetHeight(); - if (blocksToConfirm <= 0) { - // This can't happen because we don't process transactions from a block - // with a height lower than our greatest seen height - LogPrint( - BCLog::ESTIMATEFEE, - "Blockpolicy error Transaction had negative blocksToConfirm\n"); - return false; - } - - // Feerates are stored and reported as BCH-per-kb: - CFeeRate feeRate(entry->GetFee(), entry->GetTxSize()); - - feeStats.Record(blocksToConfirm, double(feeRate.GetFeePerK() / SATOSHI)); - return true; -} - -void CBlockPolicyEstimator::processBlock( - unsigned int nBlockHeight, std::vector &entries) { - if (nBlockHeight <= nBestSeenHeight) { - // Ignore side chains and re-orgs; assuming they are random they don't - // affect the estimate. And if an attacker can re-org the chain at will, - // then you've got much bigger problems than "attacker can influence - // transaction fees." - return; - } - - // Must update nBestSeenHeight in sync with ClearCurrent so that calls to - // removeTx (via processBlockTx) correctly calculate age of unconfirmed txs - // to remove from tracking. - nBestSeenHeight = nBlockHeight; - - // Clear the current block state and update unconfirmed circular buffer - feeStats.ClearCurrent(nBlockHeight); - - unsigned int countedTxs = 0; - // Repopulate the current block states - for (size_t i = 0; i < entries.size(); i++) { - if (processBlockTx(nBlockHeight, entries[i])) { - countedTxs++; - } - } - - // Update all exponential averages with the current block state - feeStats.UpdateMovingAverages(); - - LogPrint(BCLog::ESTIMATEFEE, - "Blockpolicy after updating estimates for %u of %u txs in block, " - "since last block %u of %u tracked, new mempool map size %u\n", - countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, - mapMemPoolTxs.size()); - - trackedTxs = 0; - untrackedTxs = 0; -} - -CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) { - // Return failure if trying to analyze a target we're not tracking - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget <= 1 || - (unsigned int)confTarget > feeStats.GetMaxConfirms()) { - return CFeeRate(Amount::zero()); - } - - CFeeRate median = feeStats.EstimateMedianFeeRate( - confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); - - if (median < CFeeRate(Amount::zero())) { - return CFeeRate(Amount::zero()); - } - - return median; -} - -CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, - int *answerFoundAtTarget, - const CTxMemPool &pool) { - if (answerFoundAtTarget) { - *answerFoundAtTarget = confTarget; - } - // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || - (unsigned int)confTarget > feeStats.GetMaxConfirms()) { - return CFeeRate(Amount::zero()); - } - - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget == 1) { - confTarget = 2; - } - - CFeeRate median = CFeeRate(-1 * SATOSHI); - while (median < CFeeRate(Amount::zero()) && - uint32_t(confTarget) <= feeStats.GetMaxConfirms()) { - median = feeStats.EstimateMedianFeeRate(confTarget++, SUFFICIENT_FEETXS, - MIN_SUCCESS_PCT, true, - nBestSeenHeight); - } - - if (answerFoundAtTarget) { - *answerFoundAtTarget = confTarget - 1; - } - - // If mempool is limiting txs , return at least the min feerate from the - // mempool - Amount minPoolFee = - pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * - 1000000) - .GetFeePerK(); - if (minPoolFee > Amount::zero() && minPoolFee > median.GetFeePerK()) { - return CFeeRate(minPoolFee); - } - - if (median < CFeeRate(Amount::zero())) { - return CFeeRate(Amount::zero()); - } - - return median; -} - -void CBlockPolicyEstimator::Write(CAutoFile &fileout) { - fileout << nBestSeenHeight; - feeStats.Write(fileout); -} - -void CBlockPolicyEstimator::Read(CAutoFile &filein, int nFileVersion) { - int nFileBestSeenHeight; - filein >> nFileBestSeenHeight; - feeStats.Read(filein); - nBestSeenHeight = nFileBestSeenHeight; - if (nFileVersion < 139900) { - TxConfirmStats priStats; - priStats.Read(filein); - } -} +#include "feerate.h" FeeFilterRounder::FeeFilterRounder(const CFeeRate &minIncrementalFee) { Amount minFeeLimit = std::max(SATOSHI, minIncrementalFee.GetFeePerK() / 2); feeset.insert(Amount::zero()); for (double bucketBoundary = minFeeLimit / SATOSHI; bucketBoundary <= double(MAX_FEERATE / SATOSHI); bucketBoundary *= FEE_SPACING) { feeset.insert(int64_t(bucketBoundary) * SATOSHI); } } Amount FeeFilterRounder::round(const Amount currentMinFee) { auto it = feeset.lower_bound(currentMinFee); if ((it != feeset.begin() && insecure_rand.rand32() % 3 != 0) || it == feeset.end()) { it--; } return *it; } diff --git a/src/policy/fees.h b/src/policy/fees.h index 5cb09011e0..6b65ab4b6f 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -1,287 +1,40 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_POLICYESTIMATOR_H #define BITCOIN_POLICYESTIMATOR_H #include "amount.h" #include "random.h" #include "uint256.h" #include #include #include -class CAutoFile; class CFeeRate; -class CTxMemPoolEntry; -class CTxMemPool; - -/** \class CBlockPolicyEstimator - * The BlockPolicyEstimator is used for estimating the feerate needed for a - * transaction to be included in a block within a certain number of blocks. - * - * At a high level the algorithm works by grouping transactions into buckets - * based on having similar feerates and then tracking how long it takes - * transactions in the various buckets to be mined. It operates under the - * assumption that in general transactions of higher feerate will be included in - * blocks before transactions of lower feerate. So for example if you wanted to - * know what feerate you should put on a transaction to be included in a block - * within the next 5 blocks, you would start by looking at the bucket with the - * highest feerate transactions and verifying that a sufficiently high - * percentage of them were confirmed within 5 blocks and then you would look at - * the next highest feerate bucket, and so on, stopping at the last bucket to - * pass the test. The average feerate of transactions in this bucket will give - * you an indication of the lowest feerate you can put on a transaction and - * still have a sufficiently high chance of being confirmed within your desired - * 5 blocks. - * - * Here is a brief description of the implementation: - * When a transaction enters the mempool, we track the height of the block chain - * at entry. Whenever a block comes in, we count the number of transactions in - * each bucket and the total amount of feerate paid in each bucket. Then we - * calculate how many blocks Y it took each transaction to be mined and we track - * an array of counters in each bucket for how long it to took transactions to - * get confirmed from 1 to a max of 25 and we increment all the counters from Y - * up to 25. This is because for any number Z>=Y the transaction was - * successfully mined within Z blocks. We want to save a history of this - * information, so at any time we have a counter of the total number of - * transactions that happened in a given feerate bucket and the total number - * that were confirmed in each number 1-25 blocks or less for any bucket. We - * save this history by keeping an exponentially decaying moving average of each - * one of these stats. Furthermore we also keep track of the number unmined (in - * mempool) transactions in each bucket and for how many blocks they have been - * outstanding and use that to increase the number of transactions we've seen in - * that feerate bucket when calculating an estimate for any number of - * confirmations below the number of blocks they've been outstanding. - */ - -/** - * We will instantiate an instance of this class to track transactions that were - * included in a block. We will lump transactions into a bucket according to - * their approximate feerate and then track how long it took for those txs to be - * included in a block. - * - * The tracking of unconfirmed (mempool) transactions is completely independent - * of the historical tracking of transactions that have been confirmed in a - * block. - */ -class TxConfirmStats { -private: - // Define the buckets we will group transactions into - // The upper-bound of the range for the bucket (inclusive) - std::vector buckets; - // Map of bucket upper-bound to index into all vectors by bucket - std::map bucketMap; - - // For each bucket X: - // Count the total # of txs in each bucket - // Track the historical moving average of this total over blocks - std::vector txCtAvg; - // and calculate the total for the current block to update the moving - // average - std::vector curBlockTxCt; - - // Count the total # of txs confirmed within Y blocks in each bucket - // Track the historical moving average of theses totals over blocks - // confAvg[Y][X] - std::vector> confAvg; - // and calculate the totals for the current block to update the moving - // averages - // curBlockConf[Y][X] - std::vector> curBlockConf; - - // Sum the total feerate of all tx's in each bucket - // Track the historical moving average of this total over blocks - std::vector avg; - // and calculate the total for the current block to update the moving - // average - std::vector curBlockVal; - - // Combine the conf counts with tx counts to calculate the confirmation % - // for each Y,X. Combine the total value with the tx counts to calculate the - // avg feerate per bucket - double decay; - - // Mempool counts of outstanding transactions - // For each bucket X, track the number of transactions in the mempool that - // are unconfirmed for each possible confirmation value Y - // unconfTxs[Y][X] - std::vector> unconfTxs; - // transactions still unconfirmed after MAX_CONFIRMS for each bucket - std::vector oldUnconfTxs; - -public: - /** - * Initialize the data structures. This is called by BlockPolicyEstimator's - * constructor with default values. - * @param defaultBuckets contains the upper limits for the bucket boundaries - * @param maxConfirms max number of confirms to track - * @param decay how much to decay the historical moving average per block - */ - void Initialize(std::vector &defaultBuckets, - unsigned int maxConfirms, double decay); - - /** - * Clear the state of the curBlock variables to start counting for the new - * block. - */ - void ClearCurrent(unsigned int nBlockHeight); - - /** - * Record a new transaction data point in the current block stats - * @param blocksToConfirm the number of blocks it took this transaction to - * confirm - * @param val the feerate of the transaction - * @warning blocksToConfirm is 1-based and has to be >= 1 - */ - void Record(int blocksToConfirm, double val); - - /** Record a new transaction entering the mempool*/ - unsigned int NewTx(unsigned int nBlockHeight, double val); - - /** Remove a transaction from mempool tracking stats*/ - void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, - unsigned int bucketIndex); - - /** - * Update our estimates by decaying our historical moving average and - * updating with the data gathered from the current block. - */ - void UpdateMovingAverages(); - - /** - * Calculate a feerate estimate. Find the lowest value bucket (or range of - * buckets to make sure we have enough data points) whose transactions still - * have sufficient likelihood of being confirmed within the target number of - * confirmations - * @param confTarget target number of confirmations - * @param sufficientTxVal required average number of transactions per block - * in a bucket range - * @param minSuccess the success probability we require - * @param requireGreater return the lowest feerate such that all higher - * values pass minSuccess OR - * return the highest feerate such that all lower values fail - * minSuccess - * @param nBlockHeight the current block height - */ - CFeeRate EstimateMedianFeeRate(int confTarget, double sufficientTxVal, - double minSuccess, bool requireGreater, - unsigned int nBlockHeight); - - /** Return the max number of confirms we're tracking */ - unsigned int GetMaxConfirms() { return confAvg.size(); } - - /** Write state of estimation data to a file*/ - void Write(CAutoFile &fileout); - - /** - * Read saved state of estimation data from a file and replace all internal - * data structures and variables with this state. - */ - void Read(CAutoFile &filein); -}; - -/** Track confirm delays up to 25 blocks, can't estimate beyond that */ -static const unsigned int MAX_BLOCK_CONFIRMS = 25; - -/** Decay of .998 is a half-life of 346 blocks or about 2.4 days */ -static const double DEFAULT_DECAY = .998; - -/** Require greater than 95% of X feerate transactions to be confirmed within Y - * blocks for X to be big enough */ -static const double MIN_SUCCESS_PCT = .95; - -/** Require an avg of 1 tx in the combined feerate bucket per block to have stat - * significance */ -static const double SUFFICIENT_FEETXS = 1; // Minimum and Maximum values for tracking feerates static constexpr Amount MIN_FEERATE(10 * SATOSHI); static const Amount MAX_FEERATE(int64_t(1e7) * SATOSHI); -static const Amount INF_FEERATE(MAX_MONEY); // We have to lump transactions into buckets based on feerate, but we want to be // able to give accurate estimates over a large range of potential feerates. // Therefore it makes sense to exponentially space the buckets /** Spacing of FeeRate buckets */ static const double FEE_SPACING = 1.1; -/** - * We want to be able to estimate feerates that are needed on tx's to be - * included in a certain number of blocks. Every time a block is added to the - * best chain, this class records stats on the transactions included in that - * block - */ -class CBlockPolicyEstimator { -public: - /** - * Create new BlockPolicyEstimator and initialize stats tracking classes - * with default values. - */ - CBlockPolicyEstimator(); - - /** Process all the transactions that have been included in a block */ - void processBlock(unsigned int nBlockHeight, - std::vector &entries); - - /** Process a transaction confirmed in a block*/ - bool processBlockTx(unsigned int nBlockHeight, - const CTxMemPoolEntry *entry); - - /** Process a transaction accepted to the mempool*/ - void processTransaction(const CTxMemPoolEntry &entry, - bool validFeeEstimate); - - /** Remove a transaction from the mempool tracking stats*/ - bool removeTx(uint256 hash); - - /** Return a feerate estimate */ - CFeeRate estimateFee(int confTarget); - - /** Estimate feerate needed to get be included in a block within - * confTarget blocks. If no answer can be given at confTarget, return an - * estimate at the lowest target where one can be given. - */ - CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, - const CTxMemPool &pool); - - /** Write estimation data to a file */ - void Write(CAutoFile &fileout); - - /** Read estimation data from a file */ - void Read(CAutoFile &filein, int nFileVersion); - -private: - //!< Passed to constructor to avoid dependency on main - unsigned int nBestSeenHeight; - struct TxStatsInfo { - unsigned int blockHeight; - unsigned int bucketIndex; - TxStatsInfo() : blockHeight(0), bucketIndex(0) {} - }; - - // map of txids to information about that transaction - std::map mapMemPoolTxs; - - /** Classes to track historical data on transaction confirmations */ - TxConfirmStats feeStats; - - unsigned int trackedTxs; - unsigned int untrackedTxs; -}; - class FeeFilterRounder { public: /** Create new FeeFilterRounder */ explicit FeeFilterRounder(const CFeeRate &minIncrementalFee); /** Quantize a minimum fee for privacy purpose before broadcast **/ Amount round(const Amount currentMinFee); private: std::set feeset; FastRandomContext insecure_rand; }; #endif /*BITCOIN_POLICYESTIMATOR_H */ diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index b142546784..149659bde1 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -1,340 +1,107 @@ // Copyright (c) 2011-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "policy/fees.h" #include "policy/policy.h" #include "txmempool.h" #include "uint256.h" #include "util.h" #include "test/test_bitcoin.h" #include BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, BasicTestingSetup) -BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) { - CTxMemPool mpool; - TestMemPoolEntryHelper entry; - Amount basefee = 2000 * SATOSHI; - Amount deltaFee = 100 * SATOSHI; - std::vector feeV; - - // Populate vectors of increasing fees - for (int j = 0; j < 10; j++) { - feeV.push_back((j + 1) * basefee); - } - - // Store the hashes of transactions that have been added to the mempool by - // their associate fee txIds[j] is populated with transactions either of - // fee = basefee * (j+1) - std::array, 10> txIds; - - // Create a transaction template - CScript garbage; - for (unsigned int i = 0; i < 128; i++) { - garbage.push_back('X'); - } - - CMutableTransaction tx; - tx.vin.resize(1); - tx.vin[0].scriptSig = garbage; - tx.vout.resize(1); - tx.vout[0].nValue = Amount::zero(); - CFeeRate baseRate(basefee, CTransaction(tx).GetTotalSize()); - - // Create a fake block - std::vector block; - int blocknum = 0; - - // Loop through 200 blocks - // At a decay .998 and 4 fee transactions per block - // This makes the tx count about 1.33 per bucket, above the 1 threshold - while (blocknum < 200) { - // For each fee - for (size_t j = 0; j < txIds.size(); j++) { - // add 4 fee txs - for (int k = 0; k < 4; k++) { - // make transaction unique - tx.vin[0].nSequence = 10000 * blocknum + 100 * j + k; - TxId txid = tx.GetId(); - mpool.addUnchecked(txid, entry.Fee(feeV[j]) - .Time(GetTime()) - .Priority(0) - .Height(blocknum) - .FromTx(tx, &mpool)); - txIds[j].push_back(txid); - } - } - // Create blocks where higher fee txs are included more often - for (size_t h = 0; h <= blocknum % txIds.size(); h++) { - // 10/10 blocks add highest fee transactions - // 9/10 blocks add 2nd highest and so on until ... - // 1/10 blocks add lowest fee transactions - size_t i = txIds.size() - h - 1; - while (txIds[i].size()) { - CTransactionRef ptx = mpool.get(txIds[i].back()); - if (ptx) { - block.push_back(ptx); - } - txIds[i].pop_back(); - } - } - mpool.removeForBlock(block, ++blocknum); - block.clear(); - if (blocknum == 30) { - // At this point we should need to combine 5 buckets to get enough - // data points. So estimateFee(1,2,3) should fail and estimateFee(4) - // should return somewhere around 8*baserate. estimateFee(4) %'s - // are 100,100,100,100,90 = average 98% - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(Amount::zero())); - BOOST_CHECK(mpool.estimateFee(2) == CFeeRate(Amount::zero())); - BOOST_CHECK(mpool.estimateFee(3) == CFeeRate(Amount::zero())); - BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() < - 8 * baseRate.GetFeePerK() + deltaFee); - BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() > - 8 * baseRate.GetFeePerK() - deltaFee); - int answerFound; - BOOST_CHECK(mpool.estimateSmartFee(1, &answerFound) == - mpool.estimateFee(4) && - answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(3, &answerFound) == - mpool.estimateFee(4) && - answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(4, &answerFound) == - mpool.estimateFee(4) && - answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(8, &answerFound) == - mpool.estimateFee(8) && - answerFound == 8); - } - } - - std::vector origFeeEst; - // Highest feerate is 10*baseRate and gets in all blocks, second highest - // feerate is 9*baseRate and gets in 9/10 blocks = 90%, third highest - // feerate is 8*base rate, and gets in 8/10 blocks = 80%, so estimateFee(1) - // would return 10*baseRate but is hardcoded to return failure. Second - // highest feerate has 100% chance of being included by 2 blocks, so - // estimateFee(2) should return 9*baseRate etc... - for (int i = 1; i < 10; i++) { - origFeeEst.push_back(mpool.estimateFee(i).GetFeePerK()); - // Fee estimates should be monotonically decreasing - if (i > 2) { - BOOST_CHECK(origFeeEst[i - 1] <= origFeeEst[i - 2]); - } - int mult = 11 - i; - if (i > 1) { - BOOST_CHECK(origFeeEst[i - 1] < - mult * baseRate.GetFeePerK() + deltaFee); - BOOST_CHECK(origFeeEst[i - 1] > - mult * baseRate.GetFeePerK() - deltaFee); - } else { - BOOST_CHECK(origFeeEst[i - 1] == - CFeeRate(Amount::zero()).GetFeePerK()); - } - } - - // Mine 50 more blocks with no transactions happening, estimates shouldn't - // change. We haven't decayed the moving average enough so we still have - // enough data points in every bucket - while (blocknum < 250) { - mpool.removeForBlock(block, ++blocknum); - } - - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(Amount::zero())); - for (int i = 2; i < 10; i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < - origFeeEst[i - 1] + deltaFee); - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > - origFeeEst[i - 1] - deltaFee); - } - - // Mine 15 more blocks with lots of transactions happening and not getting - // mined. Estimates should go up - while (blocknum < 265) { - // For each fee multiple - for (size_t j = 0; j < txIds.size(); j++) { - // add 4 fee txs - for (int k = 0; k < 4; k++) { - tx.vin[0].nSequence = 10000 * blocknum + 100 * j + k; - TxId txid = tx.GetId(); - mpool.addUnchecked(txid, entry.Fee(feeV[j]) - .Time(GetTime()) - .Priority(0) - .Height(blocknum) - .FromTx(tx, &mpool)); - txIds[j].push_back(txid); - } - } - mpool.removeForBlock(block, ++blocknum); - } - - int answerFound; - for (int i = 1; i < 10; i++) { - BOOST_CHECK(mpool.estimateFee(i) == CFeeRate(Amount::zero()) || - mpool.estimateFee(i).GetFeePerK() > - origFeeEst[i - 1] - deltaFee); - Amount a1 = mpool.estimateSmartFee(i, &answerFound).GetFeePerK(); - Amount a2 = origFeeEst[answerFound - 1] - deltaFee; - BOOST_CHECK(a1 > a2); - } - - // Mine all those transactions - // Estimates should still not be below original - for (size_t j = 0; j < txIds.size(); j++) { - while (txIds[j].size()) { - CTransactionRef ptx = mpool.get(txIds[j].back()); - if (ptx) { - block.push_back(ptx); - } - txIds[j].pop_back(); - } - } - mpool.removeForBlock(block, 265); - block.clear(); - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(Amount::zero())); - for (int i = 2; i < 10; i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > - origFeeEst[i - 1] - deltaFee); - } - - // Mine 200 more blocks where everything is mined every block - // Estimates should be below original estimates - while (blocknum < 465) { - // For each fee multiple - for (size_t j = 0; j < txIds.size(); j++) { - // add 4 fee txs - for (int k = 0; k < 4; k++) { - tx.vin[0].nSequence = 10000 * blocknum + 100 * j + k; - TxId txid = tx.GetId(); - mpool.addUnchecked(txid, entry.Fee(feeV[j]) - .Time(GetTime()) - .Priority(0) - .Height(blocknum) - .FromTx(tx, &mpool)); - CTransactionRef ptx = mpool.get(txid); - if (ptx) { - block.push_back(ptx); - } - } - } - mpool.removeForBlock(block, ++blocknum); - block.clear(); - } - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(Amount::zero())); - for (int i = 2; i < 10; i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < - origFeeEst[i - 1] - deltaFee); - } - - // Test that if the mempool is limited, estimateSmartFee won't return a - // value below the mempool min - mpool.addUnchecked( - tx.GetId(), - entry.Fee(feeV[5]).Time(GetTime()).Priority(0).Height(blocknum).FromTx( - tx, &mpool)); - // evict that transaction which should set a mempool min fee of - // minRelayTxFee + feeV[5] - mpool.TrimToSize(1); - BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[5]); - for (int i = 1; i < 10; i++) { - BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= - mpool.estimateFee(i).GetFeePerK()); - BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= - mpool.GetMinFee(1).GetFeePerK()); - } -} - BOOST_AUTO_TEST_CASE(MempoolMinimumFeeEstimate) { CTxMemPool mpool; TestMemPoolEntryHelper entry; // Create a transaction template CScript garbage; for (unsigned int i = 0; i < 128; i++) { garbage.push_back('X'); } CMutableTransaction tx; tx.vin.resize(1); tx.vin[0].scriptSig = garbage; tx.vout.resize(1); tx.vout[0].nValue = Amount::zero(); // Create a fake block std::vector block; int blocknum = 0; // Loop through 200 blocks adding transactions so we have a estimateFee // that is calculable. while (blocknum < 200) { for (int64_t j = 0; j < 100; j++) { // make transaction unique tx.vin[0].nSequence = 10000 * blocknum + j; TxId txid = tx.GetId(); mpool.addUnchecked( txid, entry.Fee((j + 1) * DEFAULT_BLOCK_MIN_TX_FEE_PER_KB) .Time(GetTime()) .Priority(0) .Height(blocknum) .FromTx(tx, &mpool)); CTransactionRef ptx = mpool.get(txid); block.push_back(ptx); } mpool.removeForBlock(block, ++blocknum); block.clear(); } // Check that the estimate is above the rolling minimum fee. This should // be true since we have not trimmed the mempool. BOOST_CHECK(CFeeRate(Amount::zero()) == mpool.estimateFee(1)); BOOST_CHECK(mpool.GetMinFee(1) <= mpool.estimateFee(2)); BOOST_CHECK(mpool.GetMinFee(1) <= mpool.estimateFee(3)); BOOST_CHECK(mpool.GetMinFee(1) <= mpool.estimateFee(4)); BOOST_CHECK(mpool.GetMinFee(1) <= mpool.estimateFee(5)); // Check that estimateFee returns the minimum rolling fee even when the // mempool grows very quickly and no blocks have been mined. // Add a bunch of low fee transactions which are not in the mempool // And have zero fees. CMutableTransaction mtx; tx.vin.resize(1); tx.vin[0].scriptSig = garbage; tx.vout.resize(1); block.clear(); // Add tons of transactions to the mempool, // but don't mine them. for (int64_t i = 0; i < 10000; i++) { // Mutate the hash tx.vin[0].nSequence = 10000 * blocknum + i; // Add new transaction to the mempool with a increasing fee // The average should end up as 1/2 * 100 * // DEFAULT_BLOCK_MIN_TX_FEE_PER_KB mpool.addUnchecked(tx.GetId(), entry.Fee((i + 1) * DEFAULT_BLOCK_MIN_TX_FEE_PER_KB) .Time(GetTime()) .Priority(0) .Height(blocknum) .FromTx(tx, &mpool)); } // Trim to size. GetMinFee should be more than 10000 * // DEFAULT_BLOCK_MIN_TX_FEE_PER_KB But the estimateFee should be // unchanged. mpool.TrimToSize(1); BOOST_CHECK(mpool.GetMinFee(1) >= CFeeRate(10000 * DEFAULT_BLOCK_MIN_TX_FEE_PER_KB, CTransaction(tx).GetTotalSize())); for (int i = 1; i < 10; i++) { BOOST_CHECK_MESSAGE(mpool.estimateFee(i) == mpool.GetMinFee(1), "Confirm blocks has failed on iteration " << i); } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 2538fc5993..ee8d1c2f9e 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1,1412 +1,1368 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "txmempool.h" #include "chainparams.h" // for GetConsensus. #include "clientversion.h" +#include "config.h" #include "consensus/consensus.h" #include "consensus/tx_verify.h" #include "consensus/validation.h" #include "policy/fees.h" #include "policy/policy.h" #include "reverse_iterator.h" #include "streams.h" #include "timedata.h" #include "util.h" #include "utilmoneystr.h" #include "utiltime.h" #include "validation.h" #include "version.h" #include CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef &_tx, const Amount _nFee, int64_t _nTime, double _entryPriority, unsigned int _entryHeight, Amount _inChainInputValue, bool _spendsCoinbase, int64_t _sigOpsCount, LockPoints lp) : tx(_tx), nFee(_nFee), nTime(_nTime), entryPriority(_entryPriority), entryHeight(_entryHeight), inChainInputValue(_inChainInputValue), spendsCoinbase(_spendsCoinbase), sigOpCount(_sigOpsCount), lockPoints(lp) { nTxSize = tx->GetTotalSize(); nTxBillableSize = tx->GetBillableSize(); nModSize = tx->CalculateModifiedSize(GetTxSize()); nUsageSize = RecursiveDynamicUsage(tx); nCountWithDescendants = 1; nSizeWithDescendants = GetTxSize(); nBillableSizeWithDescendants = GetTxBillableSize(); nModFeesWithDescendants = nFee; Amount nValueIn = tx->GetValueOut() + nFee; assert(inChainInputValue <= nValueIn); feeDelta = Amount::zero(); nCountWithAncestors = 1; nSizeWithAncestors = GetTxSize(); nBillableSizeWithAncestors = GetTxBillableSize(); nModFeesWithAncestors = nFee; nSigOpCountWithAncestors = sigOpCount; } double CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const { double deltaPriority = double((currentHeight - entryHeight) * (inChainInputValue / SATOSHI)) / nModSize; double dResult = entryPriority + deltaPriority; // This should only happen if it was called with a height below entry height if (dResult < 0) { dResult = 0; } return dResult; } void CTxMemPoolEntry::UpdateFeeDelta(Amount newFeeDelta) { nModFeesWithDescendants += newFeeDelta - feeDelta; nModFeesWithAncestors += newFeeDelta - feeDelta; feeDelta = newFeeDelta; } void CTxMemPoolEntry::UpdateLockPoints(const LockPoints &lp) { lockPoints = lp; } // Update the given tx for any in-mempool descendants. // Assumes that setMemPoolChildren is correct for the given tx and all // descendants. void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap &cachedDescendants, const std::set &setExclude) { setEntries stageEntries, setAllDescendants; stageEntries = GetMemPoolChildren(updateIt); while (!stageEntries.empty()) { const txiter cit = *stageEntries.begin(); setAllDescendants.insert(cit); stageEntries.erase(cit); const setEntries &setChildren = GetMemPoolChildren(cit); for (const txiter childEntry : setChildren) { cacheMap::iterator cacheIt = cachedDescendants.find(childEntry); if (cacheIt != cachedDescendants.end()) { // We've already calculated this one, just add the entries for // this set but don't traverse again. for (const txiter cacheEntry : cacheIt->second) { setAllDescendants.insert(cacheEntry); } } else if (!setAllDescendants.count(childEntry)) { // Schedule for later processing stageEntries.insert(childEntry); } } } // setAllDescendants now contains all in-mempool descendants of updateIt. // Update and add to cached descendant map int64_t modifySize = 0; int64_t modifyBillableSize = 0; int64_t modifyCount = 0; Amount modifyFee = Amount::zero(); for (txiter cit : setAllDescendants) { if (!setExclude.count(cit->GetTx().GetId())) { modifySize += cit->GetTxSize(); modifyBillableSize += cit->GetTxBillableSize(); modifyFee += cit->GetModifiedFee(); modifyCount++; cachedDescendants[updateIt].insert(cit); // Update ancestor state for each descendant mapTx.modify(cit, update_ancestor_state(updateIt->GetTxSize(), updateIt->GetTxBillableSize(), updateIt->GetModifiedFee(), 1, updateIt->GetSigOpCount())); } } mapTx.modify(updateIt, update_descendant_state(modifySize, modifyBillableSize, modifyFee, modifyCount)); } // txidsToUpdate is the set of transaction hashes from a disconnected block // which has been re-added to the mempool. For each entry, look for descendants // that are outside txidsToUpdate, and add fee/size information for such // descendants to the parent. For each such descendant, also update the ancestor // state to include the parent. void CTxMemPool::UpdateTransactionsFromBlock( const std::vector &txidsToUpdate) { LOCK(cs); // For each entry in txidsToUpdate, store the set of in-mempool, but not // in-txidsToUpdate transactions, so that we don't have to recalculate // descendants when we come across a previously seen entry. cacheMap mapMemPoolDescendantsToUpdate; // Use a set for lookups into txidsToUpdate (these entries are already // accounted for in the state of their ancestors) std::set setAlreadyIncluded(txidsToUpdate.begin(), txidsToUpdate.end()); // Iterate in reverse, so that whenever we are looking at at a transaction // we are sure that all in-mempool descendants have already been processed. // This maximizes the benefit of the descendant cache and guarantees that // setMemPoolChildren will be updated, an assumption made in // UpdateForDescendants. for (const TxId &txid : reverse_iterate(txidsToUpdate)) { // we cache the in-mempool children to avoid duplicate updates setEntries setChildren; // calculate children from mapNextTx txiter it = mapTx.find(txid); if (it == mapTx.end()) { continue; } auto iter = mapNextTx.lower_bound(COutPoint(txid, 0)); // First calculate the children, and update setMemPoolChildren to // include them, and update their setMemPoolParents to include this tx. for (; iter != mapNextTx.end() && iter->first->GetTxId() == txid; ++iter) { const TxId &childTxId = iter->second->GetId(); txiter childIter = mapTx.find(childTxId); assert(childIter != mapTx.end()); // We can skip updating entries we've encountered before or that are // in the block (which are already accounted for). if (setChildren.insert(childIter).second && !setAlreadyIncluded.count(childTxId)) { UpdateChild(it, childIter, true); UpdateParent(childIter, it, true); } } UpdateForDescendants(it, mapMemPoolDescendantsToUpdate, setAlreadyIncluded); } } bool CTxMemPool::CalculateMemPoolAncestors( const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents /* = true */) const { LOCK(cs); setEntries parentHashes; const CTransaction &tx = entry.GetTx(); if (fSearchForParents) { // Get parents of this transaction that are in the mempool // GetMemPoolParents() is only valid for entries in the mempool, so we // iterate mapTx to find parents. for (const CTxIn &in : tx.vin) { txiter piter = mapTx.find(in.prevout.GetTxId()); if (piter == mapTx.end()) { continue; } parentHashes.insert(piter); if (parentHashes.size() + 1 > limitAncestorCount) { errString = strprintf("too many unconfirmed parents [limit: %u]", limitAncestorCount); return false; } } } else { // If we're not searching for parents, we require this to be an entry in // the mempool already. txiter it = mapTx.iterator_to(entry); parentHashes = GetMemPoolParents(it); } size_t totalSizeWithAncestors = entry.GetTxSize(); while (!parentHashes.empty()) { txiter stageit = *parentHashes.begin(); setAncestors.insert(stageit); parentHashes.erase(stageit); totalSizeWithAncestors += stageit->GetTxSize(); if (stageit->GetSizeWithDescendants() + entry.GetTxSize() > limitDescendantSize) { errString = strprintf( "exceeds descendant size limit for tx %s [limit: %u]", stageit->GetTx().GetId().ToString(), limitDescendantSize); return false; } if (stageit->GetCountWithDescendants() + 1 > limitDescendantCount) { errString = strprintf("too many descendants for tx %s [limit: %u]", stageit->GetTx().GetId().ToString(), limitDescendantCount); return false; } if (totalSizeWithAncestors > limitAncestorSize) { errString = strprintf("exceeds ancestor size limit [limit: %u]", limitAncestorSize); return false; } const setEntries &setMemPoolParents = GetMemPoolParents(stageit); for (const txiter &phash : setMemPoolParents) { // If this is a new ancestor, add it. if (setAncestors.count(phash) == 0) { parentHashes.insert(phash); } if (parentHashes.size() + setAncestors.size() + 1 > limitAncestorCount) { errString = strprintf("too many unconfirmed ancestors [limit: %u]", limitAncestorCount); return false; } } } return true; } void CTxMemPool::UpdateAncestorsOf(bool add, txiter it, setEntries &setAncestors) { setEntries parentIters = GetMemPoolParents(it); // add or remove this tx as a child of each parent for (txiter piter : parentIters) { UpdateChild(piter, it, add); } const int64_t updateCount = (add ? 1 : -1); const int64_t updateSize = updateCount * it->GetTxSize(); const int64_t updateBillableSize = updateCount * it->GetTxBillableSize(); const Amount updateFee = updateCount * it->GetModifiedFee(); for (txiter ancestorIt : setAncestors) { mapTx.modify(ancestorIt, update_descendant_state(updateSize, updateBillableSize, updateFee, updateCount)); } } void CTxMemPool::UpdateEntryForAncestors(txiter it, const setEntries &setAncestors) { int64_t updateCount = setAncestors.size(); int64_t updateSize = 0; int64_t updateBillableSize = 0; int64_t updateSigOpsCount = 0; Amount updateFee = Amount::zero(); for (txiter ancestorIt : setAncestors) { updateSize += ancestorIt->GetTxSize(); updateBillableSize += ancestorIt->GetTxBillableSize(); updateFee += ancestorIt->GetModifiedFee(); updateSigOpsCount += ancestorIt->GetSigOpCount(); } mapTx.modify(it, update_ancestor_state(updateSize, updateBillableSize, updateFee, updateCount, updateSigOpsCount)); } void CTxMemPool::UpdateChildrenForRemoval(txiter it) { const setEntries &setMemPoolChildren = GetMemPoolChildren(it); for (txiter updateIt : setMemPoolChildren) { UpdateParent(updateIt, it, false); } } void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove, bool updateDescendants) { // For each entry, walk back all ancestors and decrement size associated // with this transaction. const uint64_t nNoLimit = std::numeric_limits::max(); if (updateDescendants) { // updateDescendants should be true whenever we're not recursively // removing a tx and all its descendants, eg when a transaction is // confirmed in a block. Here we only update statistics and not data in // mapLinks (which we need to preserve until we're finished with all // operations that need to traverse the mempool). for (txiter removeIt : entriesToRemove) { setEntries setDescendants; CalculateDescendants(removeIt, setDescendants); setDescendants.erase(removeIt); // don't update state for self int64_t modifySize = -int64_t(removeIt->GetTxSize()); int64_t modifyBillableSize = -int64_t(removeIt->GetTxBillableSize()); Amount modifyFee = -1 * removeIt->GetModifiedFee(); int modifySigOps = -removeIt->GetSigOpCount(); for (txiter dit : setDescendants) { mapTx.modify( dit, update_ancestor_state(modifySize, modifyBillableSize, modifyFee, -1, modifySigOps)); } } } for (txiter removeIt : entriesToRemove) { setEntries setAncestors; const CTxMemPoolEntry &entry = *removeIt; std::string dummy; // Since this is a tx that is already in the mempool, we can call CMPA // with fSearchForParents = false. If the mempool is in a consistent // state, then using true or false should both be correct, though false // should be a bit faster. // However, if we happen to be in the middle of processing a reorg, then // the mempool can be in an inconsistent state. In this case, the set of // ancestors reachable via mapLinks will be the same as the set of // ancestors whose packages include this transaction, because when we // add a new transaction to the mempool in addUnchecked(), we assume it // has no children, and in the case of a reorg where that assumption is // false, the in-mempool children aren't linked to the in-block tx's // until UpdateTransactionsFromBlock() is called. So if we're being // called during a reorg, ie before UpdateTransactionsFromBlock() has // been called, then mapLinks[] will differ from the set of mempool // parents we'd calculate by searching, and it's important that we use // the mapLinks[] notion of ancestor transactions as the set of things // to update for removal. CalculateMemPoolAncestors(entry, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false); // Note that UpdateAncestorsOf severs the child links that point to // removeIt in the entries for the parents of removeIt. UpdateAncestorsOf(false, removeIt, setAncestors); } // After updating all the ancestor sizes, we can now sever the link between // each transaction being removed and any mempool children (ie, update // setMemPoolParents for each direct child of a transaction being removed). for (txiter removeIt : entriesToRemove) { UpdateChildrenForRemoval(removeIt); } } void CTxMemPoolEntry::UpdateDescendantState(int64_t modifySize, int64_t modifyBillableSize, Amount modifyFee, int64_t modifyCount) { nSizeWithDescendants += modifySize; assert(int64_t(nSizeWithDescendants) > 0); nBillableSizeWithDescendants += modifyBillableSize; assert(int64_t(nBillableSizeWithDescendants) >= 0); nModFeesWithDescendants += modifyFee; nCountWithDescendants += modifyCount; assert(int64_t(nCountWithDescendants) > 0); } void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, int64_t modifyBillableSize, Amount modifyFee, int64_t modifyCount, int modifySigOps) { nSizeWithAncestors += modifySize; assert(int64_t(nSizeWithAncestors) > 0); nBillableSizeWithAncestors += modifyBillableSize; assert(int64_t(nBillableSizeWithAncestors) >= 0); nModFeesWithAncestors += modifyFee; nCountWithAncestors += modifyCount; assert(int64_t(nCountWithAncestors) > 0); nSigOpCountWithAncestors += modifySigOps; assert(int(nSigOpCountWithAncestors) >= 0); } CTxMemPool::CTxMemPool() : nTransactionsUpdated(0) { // lock free clear _clear(); // Sanity checks off by default for performance, because otherwise accepting // transactions becomes O(N^2) where N is the number of transactions in the // pool nCheckFrequency = 0; - - minerPolicyEstimator = new CBlockPolicyEstimator(); } -CTxMemPool::~CTxMemPool() { - delete minerPolicyEstimator; -} +CTxMemPool::~CTxMemPool() {} bool CTxMemPool::isSpent(const COutPoint &outpoint) { LOCK(cs); return mapNextTx.count(outpoint); } unsigned int CTxMemPool::GetTransactionsUpdated() const { LOCK(cs); return nTransactionsUpdated; } void CTxMemPool::AddTransactionsUpdated(unsigned int n) { LOCK(cs); nTransactionsUpdated += n; } bool CTxMemPool::addUnchecked(const uint256 &hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool validFeeEstimate) { NotifyEntryAdded(entry.GetSharedTx()); // Add to memory pool without checking anything. // Used by AcceptToMemoryPool(), which DOES do all the appropriate checks. LOCK(cs); indexed_transaction_set::iterator newit = mapTx.insert(entry).first; mapLinks.insert(make_pair(newit, TxLinks())); // Update transaction for any feeDelta created by PrioritiseTransaction // TODO: refactor so that the fee delta is calculated before inserting into // mapTx. std::map::const_iterator pos = mapDeltas.find(hash); if (pos != mapDeltas.end()) { const TXModifier &deltas = pos->second; if (deltas.second != Amount::zero()) { mapTx.modify(newit, update_fee_delta(deltas.second)); } } // Update cachedInnerUsage to include contained transaction's usage. // (When we update the entry for in-mempool parents, memory usage will be // further updated.) cachedInnerUsage += entry.DynamicMemoryUsage(); const CTransaction &tx = newit->GetTx(); std::set setParentTransactions; for (const CTxIn &in : tx.vin) { mapNextTx.insert(std::make_pair(&in.prevout, &tx)); setParentTransactions.insert(in.prevout.GetTxId()); } // Don't bother worrying about child transactions of this one. Normal case // of a new transaction arriving is that there can't be any children, // because such children would be orphans. An exception to that is if a // transaction enters that used to be in a block. In that case, our // disconnect block logic will call UpdateTransactionsFromBlock to clean up // the mess we're leaving here. // Update ancestors with information about this tx for (const uint256 &phash : setParentTransactions) { txiter pit = mapTx.find(phash); if (pit != mapTx.end()) { UpdateParent(newit, pit, true); } } UpdateAncestorsOf(true, newit, setAncestors); UpdateEntryForAncestors(newit, setAncestors); nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); - minerPolicyEstimator->processTransaction(entry, validFeeEstimate); vTxHashes.emplace_back(tx.GetHash(), newit); newit->vTxHashesIdx = vTxHashes.size() - 1; return true; } void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) { NotifyEntryRemoved(it->GetSharedTx(), reason); - const uint256 txid = it->GetTx().GetId(); for (const CTxIn &txin : it->GetTx().vin) { mapNextTx.erase(txin.prevout); } if (vTxHashes.size() > 1) { vTxHashes[it->vTxHashesIdx] = std::move(vTxHashes.back()); vTxHashes[it->vTxHashesIdx].second->vTxHashesIdx = it->vTxHashesIdx; vTxHashes.pop_back(); if (vTxHashes.size() * 2 < vTxHashes.capacity()) { vTxHashes.shrink_to_fit(); } } else { vTxHashes.clear(); } totalTxSize -= it->GetTxSize(); cachedInnerUsage -= it->DynamicMemoryUsage(); cachedInnerUsage -= memusage::DynamicUsage(mapLinks[it].parents) + memusage::DynamicUsage(mapLinks[it].children); mapLinks.erase(it); mapTx.erase(it); nTransactionsUpdated++; - minerPolicyEstimator->removeTx(txid); } // Calculates descendants of entry that are not already in setDescendants, and // adds to setDescendants. Assumes entryit is already a tx in the mempool and // setMemPoolChildren is correct for tx and all descendants. Also assumes that // if an entry is in setDescendants already, then all in-mempool descendants of // it are already in setDescendants as well, so that we can save time by not // iterating over those entries. void CTxMemPool::CalculateDescendants(txiter entryit, setEntries &setDescendants) const { setEntries stage; if (setDescendants.count(entryit) == 0) { stage.insert(entryit); } // Traverse down the children of entry, only adding children that are not // accounted for in setDescendants already (because those children have // either already been walked, or will be walked in this iteration). while (!stage.empty()) { txiter it = *stage.begin(); setDescendants.insert(it); stage.erase(it); const setEntries &setChildren = GetMemPoolChildren(it); for (const txiter &childiter : setChildren) { if (!setDescendants.count(childiter)) { stage.insert(childiter); } } } } void CTxMemPool::removeRecursive(const CTransaction &origTx, MemPoolRemovalReason reason) { // Remove transaction from memory pool. LOCK(cs); setEntries txToRemove; txiter origit = mapTx.find(origTx.GetId()); if (origit != mapTx.end()) { txToRemove.insert(origit); } else { // When recursively removing but origTx isn't in the mempool be sure to // remove any children that are in the pool. This can happen during // chain re-orgs if origTx isn't re-accepted into the mempool for any // reason. for (size_t i = 0; i < origTx.vout.size(); i++) { auto it = mapNextTx.find(COutPoint(origTx.GetId(), i)); if (it == mapNextTx.end()) { continue; } txiter nextit = mapTx.find(it->second->GetId()); assert(nextit != mapTx.end()); txToRemove.insert(nextit); } } setEntries setAllRemoves; for (txiter it : txToRemove) { CalculateDescendants(it, setAllRemoves); } RemoveStaged(setAllRemoves, false, reason); } void CTxMemPool::removeForReorg(const Config &config, const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags) { // Remove transactions spending a coinbase which are now immature and // no-longer-final transactions. LOCK(cs); setEntries txToRemove; for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { const CTransaction &tx = it->GetTx(); LockPoints lp = it->GetLockPoints(); bool validLP = TestLockPointValidity(&lp); CValidationState state; if (!ContextualCheckTransactionForCurrentBlock(config, tx, state, flags) || !CheckSequenceLocks(tx, flags, &lp, validLP)) { // Note if CheckSequenceLocks fails the LockPoints may still be // invalid. So it's critical that we remove the tx and not depend on // the LockPoints. txToRemove.insert(it); } else if (it->GetSpendsCoinbase()) { for (const CTxIn &txin : tx.vin) { indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.GetTxId()); if (it2 != mapTx.end()) { continue; } const Coin &coin = pcoins->AccessCoin(txin.prevout); if (nCheckFrequency != 0) { assert(!coin.IsSpent()); } if (coin.IsSpent() || (coin.IsCoinBase() && int64_t(nMemPoolHeight) - coin.GetHeight() < COINBASE_MATURITY)) { txToRemove.insert(it); break; } } } if (!validLP) { mapTx.modify(it, update_lock_points(lp)); } } setEntries setAllRemoves; for (txiter it : txToRemove) { CalculateDescendants(it, setAllRemoves); } RemoveStaged(setAllRemoves, false, MemPoolRemovalReason::REORG); } void CTxMemPool::removeConflicts(const CTransaction &tx) { // Remove transactions which depend on inputs of tx, recursively LOCK(cs); for (const CTxIn &txin : tx.vin) { auto it = mapNextTx.find(txin.prevout); if (it != mapNextTx.end()) { const CTransaction &txConflict = *it->second; if (txConflict != tx) { ClearPrioritisation(txConflict.GetId()); removeRecursive(txConflict, MemPoolRemovalReason::CONFLICT); } } } } /** * Called when a block is connected. Removes from mempool and updates the miner * fee estimator. */ void CTxMemPool::removeForBlock(const std::vector &vtx, unsigned int nBlockHeight) { LOCK(cs); DisconnectedBlockTransactions disconnectpool; disconnectpool.addForBlock(vtx); std::vector entries; for (const CTransactionRef &tx : reverse_iterate(disconnectpool.GetQueuedTx().get())) { uint256 txid = tx->GetId(); indexed_transaction_set::iterator i = mapTx.find(txid); if (i != mapTx.end()) { entries.push_back(&*i); } } - // Before the txs in the new block have been removed from the mempool, - // update policy estimates - minerPolicyEstimator->processBlock(nBlockHeight, entries); for (const CTransactionRef &tx : reverse_iterate(disconnectpool.GetQueuedTx().get())) { txiter it = mapTx.find(tx->GetId()); if (it != mapTx.end()) { setEntries stage; stage.insert(it); RemoveStaged(stage, true, MemPoolRemovalReason::BLOCK); } removeConflicts(*tx); ClearPrioritisation(tx->GetId()); } disconnectpool.clear(); lastRollingFeeUpdate = GetTime(); blockSinceLastRollingFeeBump = true; } void CTxMemPool::_clear() { mapLinks.clear(); mapTx.clear(); mapNextTx.clear(); vTxHashes.clear(); totalTxSize = 0; cachedInnerUsage = 0; lastRollingFeeUpdate = GetTime(); blockSinceLastRollingFeeBump = false; rollingMinimumFeeRate = 0; ++nTransactionsUpdated; } void CTxMemPool::clear() { LOCK(cs); _clear(); } void CTxMemPool::check(const CCoinsViewCache *pcoins) const { if (nCheckFrequency == 0) { return; } if (GetRand(std::numeric_limits::max()) >= nCheckFrequency) { return; } LogPrint(BCLog::MEMPOOL, "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); uint64_t checkTotal = 0; uint64_t innerUsage = 0; CCoinsViewCache mempoolDuplicate(const_cast(pcoins)); const int64_t nSpendHeight = GetSpendHeight(mempoolDuplicate); LOCK(cs); std::list waitingOnDependants; for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { unsigned int i = 0; checkTotal += it->GetTxSize(); innerUsage += it->DynamicMemoryUsage(); const CTransaction &tx = it->GetTx(); txlinksMap::const_iterator linksiter = mapLinks.find(it); assert(linksiter != mapLinks.end()); const TxLinks &links = linksiter->second; innerUsage += memusage::DynamicUsage(links.parents) + memusage::DynamicUsage(links.children); bool fDependsWait = false; setEntries setParentCheck; int64_t parentSizes = 0; int64_t parentSigOpCount = 0; for (const CTxIn &txin : tx.vin) { // Check that every mempool transaction's inputs refer to available // coins, or other mempool tx's. indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.GetTxId()); if (it2 != mapTx.end()) { const CTransaction &tx2 = it2->GetTx(); assert(tx2.vout.size() > txin.prevout.GetN() && !tx2.vout[txin.prevout.GetN()].IsNull()); fDependsWait = true; if (setParentCheck.insert(it2).second) { parentSizes += it2->GetTxSize(); parentSigOpCount += it2->GetSigOpCount(); } } else { assert(pcoins->HaveCoin(txin.prevout)); } // Check whether its inputs are marked in mapNextTx. auto it3 = mapNextTx.find(txin.prevout); assert(it3 != mapNextTx.end()); assert(it3->first == &txin.prevout); assert(it3->second == &tx); i++; } assert(setParentCheck == GetMemPoolParents(it)); // Verify ancestor state is correct. setEntries setAncestors; uint64_t nNoLimit = std::numeric_limits::max(); std::string dummy; CalculateMemPoolAncestors(*it, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy); uint64_t nCountCheck = setAncestors.size() + 1; uint64_t nSizeCheck = it->GetTxSize(); Amount nFeesCheck = it->GetModifiedFee(); int64_t nSigOpCheck = it->GetSigOpCount(); for (txiter ancestorIt : setAncestors) { nSizeCheck += ancestorIt->GetTxSize(); nFeesCheck += ancestorIt->GetModifiedFee(); nSigOpCheck += ancestorIt->GetSigOpCount(); } assert(it->GetCountWithAncestors() == nCountCheck); assert(it->GetSizeWithAncestors() == nSizeCheck); assert(it->GetSigOpCountWithAncestors() == nSigOpCheck); assert(it->GetModFeesWithAncestors() == nFeesCheck); // Check children against mapNextTx CTxMemPool::setEntries setChildrenCheck; auto iter = mapNextTx.lower_bound(COutPoint(it->GetTx().GetId(), 0)); int64_t childSizes = 0; for (; iter != mapNextTx.end() && iter->first->GetTxId() == it->GetTx().GetId(); ++iter) { txiter childit = mapTx.find(iter->second->GetId()); // mapNextTx points to in-mempool transactions assert(childit != mapTx.end()); if (setChildrenCheck.insert(childit).second) { childSizes += childit->GetTxSize(); } } assert(setChildrenCheck == GetMemPoolChildren(it)); // Also check to make sure size is greater than sum with immediate // children. Just a sanity check, not definitive that this calc is // correct... assert(it->GetSizeWithDescendants() >= childSizes + it->GetTxSize()); if (fDependsWait) { waitingOnDependants.push_back(&(*it)); } else { CValidationState state; bool fCheckResult = tx.IsCoinBase() || Consensus::CheckTxInputs( tx, state, mempoolDuplicate, nSpendHeight); assert(fCheckResult); UpdateCoins(mempoolDuplicate, tx, 1000000); } } unsigned int stepsSinceLastRemove = 0; while (!waitingOnDependants.empty()) { const CTxMemPoolEntry *entry = waitingOnDependants.front(); waitingOnDependants.pop_front(); CValidationState state; if (!mempoolDuplicate.HaveInputs(entry->GetTx())) { waitingOnDependants.push_back(entry); stepsSinceLastRemove++; assert(stepsSinceLastRemove < waitingOnDependants.size()); } else { bool fCheckResult = entry->GetTx().IsCoinBase() || Consensus::CheckTxInputs(entry->GetTx(), state, mempoolDuplicate, nSpendHeight); assert(fCheckResult); UpdateCoins(mempoolDuplicate, entry->GetTx(), 1000000); stepsSinceLastRemove = 0; } } for (auto it = mapNextTx.cbegin(); it != mapNextTx.cend(); it++) { uint256 txid = it->second->GetId(); indexed_transaction_set::const_iterator it2 = mapTx.find(txid); const CTransaction &tx = it2->GetTx(); assert(it2 != mapTx.end()); assert(&tx == it->second); } assert(totalTxSize == checkTotal); assert(innerUsage == cachedInnerUsage); } bool CTxMemPool::CompareDepthAndScore(const uint256 &hasha, const uint256 &hashb) { LOCK(cs); indexed_transaction_set::const_iterator i = mapTx.find(hasha); if (i == mapTx.end()) { return false; } indexed_transaction_set::const_iterator j = mapTx.find(hashb); if (j == mapTx.end()) { return true; } uint64_t counta = i->GetCountWithAncestors(); uint64_t countb = j->GetCountWithAncestors(); if (counta == countb) { return CompareTxMemPoolEntryByScore()(*i, *j); } return counta < countb; } namespace { class DepthAndScoreComparator { public: bool operator()(const CTxMemPool::indexed_transaction_set::const_iterator &a, const CTxMemPool::indexed_transaction_set::const_iterator &b) { uint64_t counta = a->GetCountWithAncestors(); uint64_t countb = b->GetCountWithAncestors(); if (counta == countb) { return CompareTxMemPoolEntryByScore()(*a, *b); } return counta < countb; } }; } // namespace std::vector CTxMemPool::GetSortedDepthAndScore() const { std::vector iters; AssertLockHeld(cs); iters.reserve(mapTx.size()); for (indexed_transaction_set::iterator mi = mapTx.begin(); mi != mapTx.end(); ++mi) { iters.push_back(mi); } std::sort(iters.begin(), iters.end(), DepthAndScoreComparator()); return iters; } void CTxMemPool::queryHashes(std::vector &vtxid) { LOCK(cs); auto iters = GetSortedDepthAndScore(); vtxid.clear(); vtxid.reserve(mapTx.size()); for (auto it : iters) { vtxid.push_back(it->GetTx().GetId()); } } static TxMempoolInfo GetInfo(CTxMemPool::indexed_transaction_set::const_iterator it) { return TxMempoolInfo{it->GetSharedTx(), it->GetTime(), CFeeRate(it->GetFee(), it->GetTxBillableSize()), it->GetModifiedFee() - it->GetFee()}; } std::vector CTxMemPool::infoAll() const { LOCK(cs); auto iters = GetSortedDepthAndScore(); std::vector ret; ret.reserve(mapTx.size()); for (auto it : iters) { ret.push_back(GetInfo(it)); } return ret; } CTransactionRef CTxMemPool::get(const uint256 &txid) const { LOCK(cs); indexed_transaction_set::const_iterator i = mapTx.find(txid); if (i == mapTx.end()) { return nullptr; } return i->GetSharedTx(); } TxMempoolInfo CTxMemPool::info(const uint256 &txid) const { LOCK(cs); indexed_transaction_set::const_iterator i = mapTx.find(txid); if (i == mapTx.end()) { return TxMempoolInfo(); } return GetInfo(i); } CFeeRate CTxMemPool::estimateFee(int nBlocks) const { LOCK(cs); uint64_t maxMempoolSize = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; // minerPolicy uses recent blocks to figure out a reasonable fee. This // may disagree with the rollingMinimumFeerate under certain scenarios // where the mempool increases rapidly, or blocks are being mined which // do not contain propagated transactions. - return std::max(minerPolicyEstimator->estimateFee(nBlocks), - GetMinFee(maxMempoolSize)); + return std::max(GetConfig().GetMinFeePerKB(), GetMinFee(maxMempoolSize)); } + CFeeRate CTxMemPool::estimateSmartFee(int nBlocks, int *answerFoundAtBlocks) const { - LOCK(cs); + if (answerFoundAtBlocks != nullptr) { + *answerFoundAtBlocks = 1; + } // estimateSmartFee already includes the GetMinFee check, this is the // reason it takes `*this`. It does not need std::max as above. - return minerPolicyEstimator->estimateSmartFee(nBlocks, answerFoundAtBlocks, - *this); -} - -bool CTxMemPool::WriteFeeEstimates(CAutoFile &fileout) const { - try { - LOCK(cs); - // version required to read: 0.13.99 or later - fileout << 139900; - // version that wrote the file - fileout << CLIENT_VERSION; - minerPolicyEstimator->Write(fileout); - } catch (const std::exception &) { - LogPrintf("CTxMemPool::WriteFeeEstimates(): unable to write policy " - "estimator data (non-fatal)\n"); - return false; - } - return true; -} - -bool CTxMemPool::ReadFeeEstimates(CAutoFile &filein) { - try { - int nVersionRequired, nVersionThatWrote; - filein >> nVersionRequired >> nVersionThatWrote; - if (nVersionRequired > CLIENT_VERSION) { - return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee " - "estimate file", - nVersionRequired); - } - - LOCK(cs); - minerPolicyEstimator->Read(filein, nVersionThatWrote); - } catch (const std::exception &) { - LogPrintf("CTxMemPool::ReadFeeEstimates(): unable to read policy " - "estimator data (non-fatal)\n"); - return false; - } - return true; + return estimateFee(nBlocks); } void CTxMemPool::PrioritiseTransaction(const uint256 hash, const std::string strHash, double dPriorityDelta, const Amount nFeeDelta) { { LOCK(cs); TXModifier &deltas = mapDeltas[hash]; deltas.first += dPriorityDelta; deltas.second += nFeeDelta; txiter it = mapTx.find(hash); if (it != mapTx.end()) { mapTx.modify(it, update_fee_delta(deltas.second)); // Now update all ancestors' modified fees with descendants setEntries setAncestors; uint64_t nNoLimit = std::numeric_limits::max(); std::string dummy; CalculateMemPoolAncestors(*it, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false); for (txiter ancestorIt : setAncestors) { mapTx.modify(ancestorIt, update_descendant_state(0, 0, nFeeDelta, 0)); } // Now update all descendants' modified fees with ancestors setEntries setDescendants; CalculateDescendants(it, setDescendants); setDescendants.erase(it); for (txiter descendantIt : setDescendants) { mapTx.modify(descendantIt, update_ancestor_state(0, 0, nFeeDelta, 0, 0)); } } } LogPrintf("PrioritiseTransaction: %s priority += %f, fee += %d\n", strHash, dPriorityDelta, FormatMoney(nFeeDelta)); } void CTxMemPool::ApplyDeltas(const uint256 hash, double &dPriorityDelta, Amount &nFeeDelta) const { LOCK(cs); std::map::const_iterator pos = mapDeltas.find(hash); if (pos == mapDeltas.end()) { return; } const TXModifier &deltas = pos->second; dPriorityDelta += deltas.first; nFeeDelta += deltas.second; } void CTxMemPool::ClearPrioritisation(const uint256 hash) { LOCK(cs); mapDeltas.erase(hash); } bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const { for (const CTxIn &in : tx.vin) { if (exists(in.prevout.GetTxId())) { return false; } } return true; } CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView *baseIn, const CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) {} bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { // If an entry in the mempool exists, always return that one, as it's // guaranteed to never conflict with the underlying cache, and it cannot // have pruned entries (as it contains full) transactions. First checking // the underlying cache risks returning a pruned entry instead. CTransactionRef ptx = mempool.get(outpoint.GetTxId()); if (ptx) { if (outpoint.GetN() < ptx->vout.size()) { coin = Coin(ptx->vout[outpoint.GetN()], MEMPOOL_HEIGHT, false); return true; } return false; } return base->GetCoin(outpoint, coin) && !coin.IsSpent(); } bool CCoinsViewMemPool::HaveCoin(const COutPoint &outpoint) const { return mempool.exists(outpoint) || base->HaveCoin(outpoint); } size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); // Estimate the overhead of mapTx to be 15 pointers + an allocation, as no // exact formula for boost::multi_index_contained is implemented. return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void *)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; } void CTxMemPool::RemoveStaged(setEntries &stage, bool updateDescendants, MemPoolRemovalReason reason) { AssertLockHeld(cs); UpdateForRemoveFromMempool(stage, updateDescendants); for (const txiter &it : stage) { removeUnchecked(it, reason); } } int CTxMemPool::Expire(int64_t time) { LOCK(cs); indexed_transaction_set::index::type::iterator it = mapTx.get().begin(); setEntries toremove; while (it != mapTx.get().end() && it->GetTime() < time) { toremove.insert(mapTx.project<0>(it)); it++; } setEntries stage; for (txiter removeit : toremove) { CalculateDescendants(removeit, stage); } RemoveStaged(stage, false, MemPoolRemovalReason::EXPIRY); return stage.size(); } void CTxMemPool::LimitSize(size_t limit, unsigned long age) { int expired = Expire(GetTime() - age); if (expired != 0) { LogPrint(BCLog::MEMPOOL, "Expired %i transactions from the memory pool\n", expired); } std::vector vNoSpendsRemaining; TrimToSize(limit, &vNoSpendsRemaining); for (const COutPoint &removed : vNoSpendsRemaining) { pcoinsTip->Uncache(removed); } } bool CTxMemPool::addUnchecked(const uint256 &hash, const CTxMemPoolEntry &entry, bool validFeeEstimate) { LOCK(cs); setEntries setAncestors; uint64_t nNoLimit = std::numeric_limits::max(); std::string dummy; CalculateMemPoolAncestors(entry, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy); return addUnchecked(hash, entry, setAncestors, validFeeEstimate); } void CTxMemPool::UpdateChild(txiter entry, txiter child, bool add) { setEntries s; if (add && mapLinks[entry].children.insert(child).second) { cachedInnerUsage += memusage::IncrementalDynamicUsage(s); } else if (!add && mapLinks[entry].children.erase(child)) { cachedInnerUsage -= memusage::IncrementalDynamicUsage(s); } } void CTxMemPool::UpdateParent(txiter entry, txiter parent, bool add) { setEntries s; if (add && mapLinks[entry].parents.insert(parent).second) { cachedInnerUsage += memusage::IncrementalDynamicUsage(s); } else if (!add && mapLinks[entry].parents.erase(parent)) { cachedInnerUsage -= memusage::IncrementalDynamicUsage(s); } } const CTxMemPool::setEntries & CTxMemPool::GetMemPoolParents(txiter entry) const { assert(entry != mapTx.end()); txlinksMap::const_iterator it = mapLinks.find(entry); assert(it != mapLinks.end()); return it->second.parents; } const CTxMemPool::setEntries & CTxMemPool::GetMemPoolChildren(txiter entry) const { assert(entry != mapTx.end()); txlinksMap::const_iterator it = mapLinks.find(entry); assert(it != mapLinks.end()); return it->second.children; } CFeeRate CTxMemPool::GetMinFee(size_t sizelimit) const { LOCK(cs); if (!blockSinceLastRollingFeeBump || rollingMinimumFeeRate == 0) { return CFeeRate(int64_t(ceill(rollingMinimumFeeRate)) * SATOSHI); } int64_t time = GetTime(); if (time > lastRollingFeeUpdate + 10) { double halflife = ROLLING_FEE_HALFLIFE; if (DynamicMemoryUsage() < sizelimit / 4) { halflife /= 4; } else if (DynamicMemoryUsage() < sizelimit / 2) { halflife /= 2; } rollingMinimumFeeRate = rollingMinimumFeeRate / pow(2.0, (time - lastRollingFeeUpdate) / halflife); lastRollingFeeUpdate = time; } return CFeeRate(int64_t(ceill(rollingMinimumFeeRate)) * SATOSHI); } void CTxMemPool::trackPackageRemoved(const CFeeRate &rate) { AssertLockHeld(cs); if ((rate.GetFeePerK() / SATOSHI) > rollingMinimumFeeRate) { rollingMinimumFeeRate = rate.GetFeePerK() / SATOSHI; blockSinceLastRollingFeeBump = false; } } void CTxMemPool::TrimToSize(size_t sizelimit, std::vector *pvNoSpendsRemaining) { LOCK(cs); unsigned nTxnRemoved = 0; CFeeRate maxFeeRateRemoved(Amount::zero()); while (!mapTx.empty() && DynamicMemoryUsage() > sizelimit) { indexed_transaction_set::index::type::iterator it = mapTx.get().begin(); // We set the new mempool min fee to the feerate of the removed set, // plus the "minimum reasonable fee rate" (ie some value under which we // consider txn to have 0 fee). This way, we don't allow txn to enter // mempool with feerate equal to txn which were removed with no block in // between. CFeeRate removed(it->GetModFeesWithDescendants(), it->GetSizeWithDescendants()); removed += MEMPOOL_FULL_FEE_INCREMENT; trackPackageRemoved(removed); maxFeeRateRemoved = std::max(maxFeeRateRemoved, removed); setEntries stage; CalculateDescendants(mapTx.project<0>(it), stage); nTxnRemoved += stage.size(); std::vector txn; if (pvNoSpendsRemaining) { txn.reserve(stage.size()); for (txiter iter : stage) { txn.push_back(iter->GetTx()); } } RemoveStaged(stage, false, MemPoolRemovalReason::SIZELIMIT); if (pvNoSpendsRemaining) { for (const CTransaction &tx : txn) { for (const CTxIn &txin : tx.vin) { if (exists(txin.prevout.GetTxId())) { continue; } if (!mapNextTx.count(txin.prevout)) { pvNoSpendsRemaining->push_back(txin.prevout); } } } } } if (maxFeeRateRemoved > CFeeRate(Amount::zero())) { LogPrint(BCLog::MEMPOOL, "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); } } bool CTxMemPool::TransactionWithinChainLimit(const uint256 &txid, size_t chainLimit) const { LOCK(cs); auto it = mapTx.find(txid); return it == mapTx.end() || (it->GetCountWithAncestors() < chainLimit && it->GetCountWithDescendants() < chainLimit); } SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} /** Maximum bytes for transactions to store for processing during reorg */ static const size_t MAX_DISCONNECTED_TX_POOL_SIZE = 20 * DEFAULT_MAX_BLOCK_SIZE; void DisconnectedBlockTransactions::addForBlock( const std::vector &vtx) { for (const auto &tx : reverse_iterate(vtx)) { // If we already added it, just skip. auto it = queuedTx.find(tx->GetId()); if (it != queuedTx.end()) { continue; } // Insert the transaction into the pool. addTransaction(tx); // Fill in the set of parents. std::unordered_set parents; for (const CTxIn &in : tx->vin) { parents.insert(in.prevout.GetTxId()); } // In order to make sure we keep things in topological order, we check // if we already know of the parent of the current transaction. If so, // we remove them from the set and then add them back. while (parents.size() > 0) { std::unordered_set worklist( std::move(parents)); parents.clear(); for (const TxId &txid : worklist) { // If we do not have that txid in the set, nothing needs to be // done. auto pit = queuedTx.find(txid); if (pit == queuedTx.end()) { continue; } // We have parent in our set, we reinsert them at the right // position. const CTransactionRef ptx = *pit; queuedTx.erase(pit); queuedTx.insert(ptx); // And we make sure ancestors are covered. for (const CTxIn &in : ptx->vin) { parents.insert(in.prevout.GetTxId()); } } } } // Keep the size under control. while (DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE) { // Drop the earliest entry, and remove its children from the // mempool. auto it = queuedTx.get().begin(); g_mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); removeEntry(it); } } void DisconnectedBlockTransactions::updateMempoolForReorg(const Config &config, bool fAddToMempool) { AssertLockHeld(cs_main); std::vector txidsUpdate; // disconnectpool's insertion_order index sorts the entries from oldest to // newest, but the oldest entry will be the last tx from the latest mined // block that was disconnected. // Iterate disconnectpool in reverse, so that we add transactions back to // the mempool starting with the earliest transaction that had been // previously seen in a block. for (const CTransactionRef &tx : reverse_iterate(queuedTx.get())) { // ignore validation errors in resurrected transactions CValidationState stateDummy; if (!fAddToMempool || tx->IsCoinBase() || !AcceptToMemoryPool(config, g_mempool, stateDummy, tx, false, nullptr, true)) { // If the transaction doesn't make it in to the mempool, remove any // transactions that depend on it (which would now be orphans). g_mempool.removeRecursive(*tx, MemPoolRemovalReason::REORG); } else if (g_mempool.exists(tx->GetId())) { txidsUpdate.push_back(tx->GetId()); } } queuedTx.clear(); // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have // no in-mempool children, which is generally not true when adding // previously-confirmed transactions back to the mempool. // UpdateTransactionsFromBlock finds descendants of any transactions in the // disconnectpool that were added back and cleans up the mempool state. g_mempool.UpdateTransactionsFromBlock(txidsUpdate); // We also need to remove any now-immature transactions g_mempool.removeForReorg(config, pcoinsTip.get(), chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); // Re-limit mempool size, in case we added any transactions g_mempool.LimitSize( gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); } diff --git a/src/txmempool.h b/src/txmempool.h index fe11568d85..f77b2d7670 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -1,951 +1,944 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_TXMEMPOOL_H #define BITCOIN_TXMEMPOOL_H #include "amount.h" #include "coins.h" #include "indirectmap.h" #include "primitives/transaction.h" #include "random.h" #include "sync.h" #include #include #include #include #include #include #include #include #include #include #include class CAutoFile; class CBlockIndex; class Config; inline double AllowFreeThreshold() { return (144 * COIN) / (250 * SATOSHI); } inline bool AllowFree(double dPriority) { // Large (in bytes) low-priority (new, small-coin) transactions need a fee. return dPriority > AllowFreeThreshold(); } /** * Fake height value used in Coins to signify they are only in the memory * pool(since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; struct LockPoints { // Will be set to the blockchain height and median time past values that // would be necessary to satisfy all relative locktime constraints (BIP68) // of this tx given our view of block chain history int height; int64_t time; // As long as the current chain descends from the highest height block // containing one of the inputs used in the calculation, then the cached // values are still valid even after a reorg. CBlockIndex *maxInputBlock; LockPoints() : height(0), time(0), maxInputBlock(nullptr) {} }; class CTxMemPool; /** \class CTxMemPoolEntry * * CTxMemPoolEntry stores data about the corresponding transaction, as well as * data about all in-mempool transactions that depend on the transaction * ("descendant" transactions). * * When a new entry is added to the mempool, we update the descendant state * (nCountWithDescendants, nSizeWithDescendants, and nModFeesWithDescendants) * for all ancestors of the newly added transaction. * * If updating the descendant state is skipped, we can mark the entry as * "dirty", and set nSizeWithDescendants/nModFeesWithDescendants to equal * nTxSize/nFee+feeDelta. (This can potentially happen during a reorg, where we * limit the amount of work we're willing to do to avoid consuming too much * CPU.) */ class CTxMemPoolEntry { private: CTransactionRef tx; //!< Cached to avoid expensive parent-transaction lookups Amount nFee; //!< ... and avoid recomputing tx size size_t nTxSize; //!< ... and billable size for billing size_t nTxBillableSize; //!< ... and modified size for priority size_t nModSize; //!< ... and total memory usage size_t nUsageSize; //!< Local time when entering the mempool int64_t nTime; //!< Priority when entering the mempool double entryPriority; //!< Chain height when entering the mempool unsigned int entryHeight; //!< Sum of all txin values that are already in blockchain Amount inChainInputValue; //!< keep track of transactions that spend a coinbase bool spendsCoinbase; //!< Total sigop plus P2SH sigops count int64_t sigOpCount; //!< Used for determining the priority of the transaction for mining in a //! block Amount feeDelta; //!< Track the height and time at which tx was final LockPoints lockPoints; // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these // descendants as well. if nCountWithDescendants is 0, treat this entry as // dirty, and nSizeWithDescendants and nModFeesWithDescendants will not be // correct. //!< number of descendant transactions uint64_t nCountWithDescendants; //!< ... and size uint64_t nSizeWithDescendants; uint64_t nBillableSizeWithDescendants; //!< ... and total fees (all including us) Amount nModFeesWithDescendants; // Analogous statistics for ancestor transactions uint64_t nCountWithAncestors; uint64_t nSizeWithAncestors; uint64_t nBillableSizeWithAncestors; Amount nModFeesWithAncestors; int64_t nSigOpCountWithAncestors; public: CTxMemPoolEntry(const CTransactionRef &_tx, const Amount _nFee, int64_t _nTime, double _entryPriority, unsigned int _entryHeight, Amount _inChainInputValue, bool spendsCoinbase, int64_t nSigOpsCost, LockPoints lp); const CTransaction &GetTx() const { return *this->tx; } CTransactionRef GetSharedTx() const { return this->tx; } /** * Fast calculation of lower bound of current priority as update from entry * priority. Only inputs that were originally in-chain will age. */ double GetPriority(unsigned int currentHeight) const; const Amount GetFee() const { return nFee; } size_t GetTxSize() const { return nTxSize; } size_t GetTxBillableSize() const { return nTxBillableSize; } int64_t GetTime() const { return nTime; } unsigned int GetHeight() const { return entryHeight; } int64_t GetSigOpCount() const { return sigOpCount; } Amount GetModifiedFee() const { return nFee + feeDelta; } size_t DynamicMemoryUsage() const { return nUsageSize; } const LockPoints &GetLockPoints() const { return lockPoints; } // Adjusts the descendant state, if this entry is not dirty. void UpdateDescendantState(int64_t modifySize, int64_t modifyBillableSize, Amount modifyFee, int64_t modifyCount); // Adjusts the ancestor state void UpdateAncestorState(int64_t modifySize, int64_t modifyBillableSize, Amount modifyFee, int64_t modifyCount, int modifySigOps); // Updates the fee delta used for mining priority score, and the // modified fees with descendants. void UpdateFeeDelta(Amount feeDelta); // Update the LockPoints after a reorg void UpdateLockPoints(const LockPoints &lp); uint64_t GetCountWithDescendants() const { return nCountWithDescendants; } uint64_t GetSizeWithDescendants() const { return nSizeWithDescendants; } uint64_t GetBillableSizeWithDescendants() const { return nBillableSizeWithDescendants; } Amount GetModFeesWithDescendants() const { return nModFeesWithDescendants; } bool GetSpendsCoinbase() const { return spendsCoinbase; } uint64_t GetCountWithAncestors() const { return nCountWithAncestors; } uint64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } uint64_t GetBillableSizeWithAncestors() const { return nBillableSizeWithAncestors; } Amount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } int64_t GetSigOpCountWithAncestors() const { return nSigOpCountWithAncestors; } //!< Index in mempool's vTxHashes mutable size_t vTxHashesIdx; }; // Helpers for modifying CTxMemPool::mapTx, which is a boost multi_index. struct update_descendant_state { update_descendant_state(int64_t _modifySize, int64_t _modifyBillableSize, Amount _modifyFee, int64_t _modifyCount) : modifySize(_modifySize), modifyBillableSize(_modifyBillableSize), modifyFee(_modifyFee), modifyCount(_modifyCount) {} void operator()(CTxMemPoolEntry &e) { e.UpdateDescendantState(modifySize, modifyBillableSize, modifyFee, modifyCount); } private: int64_t modifySize; int64_t modifyBillableSize; Amount modifyFee; int64_t modifyCount; }; struct update_ancestor_state { update_ancestor_state(int64_t _modifySize, int64_t _modifyBillableSize, Amount _modifyFee, int64_t _modifyCount, int64_t _modifySigOpsCost) : modifySize(_modifySize), modifyBillableSize(_modifyBillableSize), modifyFee(_modifyFee), modifyCount(_modifyCount), modifySigOpsCost(_modifySigOpsCost) {} void operator()(CTxMemPoolEntry &e) { e.UpdateAncestorState(modifySize, modifyBillableSize, modifyFee, modifyCount, modifySigOpsCost); } private: int64_t modifySize; int64_t modifyBillableSize; Amount modifyFee; int64_t modifyCount; int64_t modifySigOpsCost; }; struct update_fee_delta { explicit update_fee_delta(Amount _feeDelta) : feeDelta(_feeDelta) {} void operator()(CTxMemPoolEntry &e) { e.UpdateFeeDelta(feeDelta); } private: Amount feeDelta; }; struct update_lock_points { explicit update_lock_points(const LockPoints &_lp) : lp(_lp) {} void operator()(CTxMemPoolEntry &e) { e.UpdateLockPoints(lp); } private: const LockPoints &lp; }; // extracts a transaction hash from CTxMempoolEntry or CTransactionRef struct mempoolentry_txid { typedef uint256 result_type; result_type operator()(const CTxMemPoolEntry &entry) const { return entry.GetTx().GetId(); } result_type operator()(const CTransactionRef &tx) const { return tx->GetId(); } }; /** \class CompareTxMemPoolEntryByDescendantScore * * Sort an entry by max(score/size of entry's tx, score/size with all * descendants). */ class CompareTxMemPoolEntryByDescendantScore { public: bool operator()(const CTxMemPoolEntry &a, const CTxMemPoolEntry &b) const { bool fUseADescendants = UseDescendantScore(a); bool fUseBDescendants = UseDescendantScore(b); double aModFee = (fUseADescendants ? a.GetModFeesWithDescendants() : a.GetModifiedFee()) / SATOSHI; double aSize = fUseADescendants ? a.GetSizeWithDescendants() : a.GetTxSize(); double bModFee = (fUseBDescendants ? b.GetModFeesWithDescendants() : b.GetModifiedFee()) / SATOSHI; double bSize = fUseBDescendants ? b.GetSizeWithDescendants() : b.GetTxSize(); // Avoid division by rewriting (a/b > c/d) as (a*d > c*b). double f1 = aModFee * bSize; double f2 = aSize * bModFee; if (f1 == f2) { return a.GetTime() >= b.GetTime(); } return f1 < f2; } // Calculate which score to use for an entry (avoiding division). bool UseDescendantScore(const CTxMemPoolEntry &a) const { double f1 = a.GetSizeWithDescendants() * (a.GetModifiedFee() / SATOSHI); double f2 = a.GetTxSize() * (a.GetModFeesWithDescendants() / SATOSHI); return f2 > f1; } }; /** \class CompareTxMemPoolEntryByScore * * Sort by score of entry ((fee+delta)/size) in descending order */ class CompareTxMemPoolEntryByScore { public: bool operator()(const CTxMemPoolEntry &a, const CTxMemPoolEntry &b) const { double f1 = b.GetTxSize() * (a.GetModifiedFee() / SATOSHI); double f2 = a.GetTxSize() * (b.GetModifiedFee() / SATOSHI); if (f1 == f2) { return b.GetTx().GetId() < a.GetTx().GetId(); } return f1 > f2; } }; class CompareTxMemPoolEntryByEntryTime { public: bool operator()(const CTxMemPoolEntry &a, const CTxMemPoolEntry &b) const { return a.GetTime() < b.GetTime(); } }; class CompareTxMemPoolEntryByAncestorFee { public: bool operator()(const CTxMemPoolEntry &a, const CTxMemPoolEntry &b) const { double aFees = a.GetModFeesWithAncestors() / SATOSHI; double aSize = a.GetSizeWithAncestors(); double bFees = b.GetModFeesWithAncestors() / SATOSHI; double bSize = b.GetSizeWithAncestors(); // Avoid division by rewriting (a/b > c/d) as (a*d > c*b). double f1 = aFees * bSize; double f2 = aSize * bFees; if (f1 == f2) { return a.GetTx().GetId() < b.GetTx().GetId(); } return f1 > f2; } }; // Multi_index tag names struct descendant_score {}; struct entry_time {}; struct mining_score {}; struct ancestor_score {}; -class CBlockPolicyEstimator; - /** * Information about a mempool transaction. */ struct TxMempoolInfo { /** The transaction itself */ CTransactionRef tx; /** Time the transaction entered the mempool. */ int64_t nTime; /** Feerate of the transaction. */ CFeeRate feeRate; /** The fee delta. */ Amount nFeeDelta; }; /** * Reason why a transaction was removed from the mempool, this is passed to the * notification signal. */ enum class MemPoolRemovalReason { //! Manually removed or unknown reason UNKNOWN = 0, //! Expired from mempool EXPIRY, //! Removed in size limiting SIZELIMIT, //! Removed for reorganization REORG, //! Removed for block BLOCK, //! Removed for conflict with in-block transaction CONFLICT, //! Removed for replacement REPLACED }; class SaltedTxidHasher { private: /** Salt */ const uint64_t k0, k1; public: SaltedTxidHasher(); size_t operator()(const uint256 &txid) const { return SipHashUint256(k0, k1, txid); } }; typedef std::pair TXModifier; /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions that * may be included in the next block. * * Transactions are added when they are seen on the network (or created by the * local node), but not all transactions seen are added to the pool. For * example, the following new transactions will not be added to the mempool: * - a transaction which doesn't meet the minimum fee requirements. * - a new transaction that double-spends an input of a transaction already in * the pool where the new transaction does not meet the Replace-By-Fee * requirements as defined in BIP 125. * - a non-standard transaction. * * CTxMemPool::mapTx, and CTxMemPoolEntry bookkeeping: * * mapTx is a boost::multi_index that sorts the mempool on 4 criteria: * - transaction hash * - feerate [we use max(feerate of tx, feerate of tx with all descendants)] * - time in mempool * - mining score (feerate modified by any fee deltas from * PrioritiseTransaction) * * Note: the term "descendant" refers to in-mempool transactions that depend on * this one, while "ancestor" refers to in-mempool transactions that a given * transaction depends on. * * In order for the feerate sort to remain correct, we must update transactions * in the mempool when new descendants arrive. To facilitate this, we track the * set of in-mempool direct parents and direct children in mapLinks. Within each * CTxMemPoolEntry, we track the size and fees of all descendants. * * Usually when a new transaction is added to the mempool, it has no in-mempool * children (because any such children would be an orphan). So in * addUnchecked(), we: * - update a new entry's setMemPoolParents to include all in-mempool parents * - update the new entry's direct parents to include the new tx as a child * - update all ancestors of the transaction to include the new tx's size/fee * * When a transaction is removed from the mempool, we must: * - update all in-mempool parents to not track the tx in setMemPoolChildren * - update all ancestors to not include the tx's size/fees in descendant state * - update all in-mempool children to not include it as a parent * * These happen in UpdateForRemoveFromMempool(). (Note that when removing a * transaction along with its descendants, we must calculate that set of * transactions to be removed before doing the removal, or else the mempool can * be in an inconsistent state where it's impossible to walk the ancestors of a * transaction.) * * In the event of a reorg, the assumption that a newly added tx has no * in-mempool children is false. In particular, the mempool is in an * inconsistent state while new transactions are being added, because there may * be descendant transactions of a tx coming from a disconnected block that are * unreachable from just looking at transactions in the mempool (the linking * transactions may also be in the disconnected block, waiting to be added). * Because of this, there's not much benefit in trying to search for in-mempool * children in addUnchecked(). Instead, in the special case of transactions * being added from a disconnected block, we require the caller to clean up the * state, to account for in-mempool, out-of-block descendants for all the * in-block transactions by calling UpdateTransactionsFromBlock(). Note that * until this is called, the mempool state is not consistent, and in particular * mapLinks may not be correct (and therefore functions like * CalculateMemPoolAncestors() and CalculateDescendants() that rely on them to * walk the mempool are not generally safe to use). * * Computational limits: * * Updating all in-mempool ancestors of a newly added transaction can be slow, * if no bound exists on how many in-mempool ancestors there may be. * CalculateMemPoolAncestors() takes configurable limits that are designed to * prevent these calculations from being too CPU intensive. * * Adding transactions from a disconnected block can be very time consuming, * because we don't have a way to limit the number of in-mempool descendants. To * bound CPU processing, we limit the amount of work we're willing to do to * properly update the descendant information for a tx being added from a * disconnected block. If we would exceed the limit, then we instead mark the * entry as "dirty", and set the feerate for sorting purposes to be equal the * feerate of the transaction without any descendants. */ class CTxMemPool { private: //!< Value n means that n times in 2^32 we check. uint32_t nCheckFrequency; unsigned int nTransactionsUpdated; - CBlockPolicyEstimator *minerPolicyEstimator; //!< sum of all mempool tx's virtual sizes. uint64_t totalTxSize; //!< sum of dynamic memory usage of all the map elements (NOT the maps //! themselves) uint64_t cachedInnerUsage; mutable int64_t lastRollingFeeUpdate; mutable bool blockSinceLastRollingFeeBump; //!< minimum fee to get into the pool, decreases exponentially mutable double rollingMinimumFeeRate; void trackPackageRemoved(const CFeeRate &rate); public: // public only for testing static const int ROLLING_FEE_HALFLIFE = 60 * 60 * 12; typedef boost::multi_index_container< CTxMemPoolEntry, boost::multi_index::indexed_by< // sorted by txid boost::multi_index::hashed_unique< mempoolentry_txid, SaltedTxidHasher>, // sorted by fee rate boost::multi_index::ordered_non_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByDescendantScore>, // sorted by entry time boost::multi_index::ordered_non_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByEntryTime>, // sorted by score (for mining prioritization) boost::multi_index::ordered_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByScore>, // sorted by fee rate with ancestors boost::multi_index::ordered_non_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByAncestorFee>>> indexed_transaction_set; mutable CCriticalSection cs; indexed_transaction_set mapTx; typedef indexed_transaction_set::nth_index<0>::type::iterator txiter; //!< All tx hashes/entries in mapTx, in random order std::vector> vTxHashes; struct CompareIteratorByHash { bool operator()(const txiter &a, const txiter &b) const { return a->GetTx().GetId() < b->GetTx().GetId(); } }; typedef std::set setEntries; const setEntries &GetMemPoolParents(txiter entry) const; const setEntries &GetMemPoolChildren(txiter entry) const; private: typedef std::map cacheMap; struct TxLinks { setEntries parents; setEntries children; }; typedef std::map txlinksMap; txlinksMap mapLinks; void UpdateParent(txiter entry, txiter parent, bool add); void UpdateChild(txiter entry, txiter child, bool add); std::vector GetSortedDepthAndScore() const; public: indirectmap mapNextTx; std::map mapDeltas; /** Create a new CTxMemPool. */ CTxMemPool(); ~CTxMemPool(); /** * If sanity-checking is turned on, check makes sure the pool is consistent * (does not contain two transactions that spend the same inputs, all inputs * are in the mapNextTx array). If sanity-checking is turned off, check does * nothing. */ void check(const CCoinsViewCache *pcoins) const; void setSanityCheck(double dFrequency = 1.0) { nCheckFrequency = dFrequency * 4294967295.0; } // addUnchecked must updated state for all ancestors of a given transaction, // to track size/count of descendant transactions. First version of // addUnchecked can be used to have it call CalculateMemPoolAncestors(), and // then invoke the second version. // Note that addUnchecked is ONLY called from ATMP outside of tests // and any other callers may break wallet's in-mempool tracking (due to // lack of CValidationInterface::TransactionAddedToMempool callbacks). bool addUnchecked(const uint256 &hash, const CTxMemPoolEntry &entry, bool validFeeEstimate = true); bool addUnchecked(const uint256 &hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool validFeeEstimate = true); void removeRecursive( const CTransaction &tx, MemPoolRemovalReason reason = MemPoolRemovalReason::UNKNOWN); void removeForReorg(const Config &config, const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags); void removeConflicts(const CTransaction &tx); void removeForBlock(const std::vector &vtx, unsigned int nBlockHeight); void clear(); // lock free void _clear(); bool CompareDepthAndScore(const uint256 &hasha, const uint256 &hashb); void queryHashes(std::vector &vtxid); bool isSpent(const COutPoint &outpoint); unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); /** * Check that none of this transactions inputs are in the mempool, and thus * the tx is not dependent on other mempool transactions to be included in a * block. */ bool HasNoInputsOf(const CTransaction &tx) const; /** Affect CreateNewBlock prioritisation of transactions */ void PrioritiseTransaction(const uint256 hash, const std::string strHash, double dPriorityDelta, const Amount nFeeDelta); void ApplyDeltas(const uint256 hash, double &dPriorityDelta, Amount &nFeeDelta) const; void ClearPrioritisation(const uint256 hash); public: /** * Remove a set of transactions from the mempool. If a transaction is in * this set, then all in-mempool descendants must also be in the set, unless * this transaction is being removed for being in a block. Set * updateDescendants to true when removing a tx that was in a block, so that * any in-mempool descendants have their ancestor state updated. */ void RemoveStaged(setEntries &stage, bool updateDescendants, MemPoolRemovalReason reason = MemPoolRemovalReason::UNKNOWN); /** * When adding transactions from a disconnected block back to the mempool, * new mempool entries may have children in the mempool (which is generally * not the case when otherwise adding transactions). * UpdateTransactionsFromBlock() will find child transactions and update the * descendant state for each transaction in txidsToUpdate (excluding any * child transactions present in txidsToUpdate, which are already accounted * for). * Note: txidsToUpdate should be the set of transactions from the * disconnected block that have been accepted back into the mempool. */ void UpdateTransactionsFromBlock(const std::vector &txidsToUpdate); /** * Try to calculate all in-mempool ancestors of entry. * (these are all calculated including the tx itself) * limitAncestorCount = max number of ancestors * limitAncestorSize = max size of ancestors * limitDescendantCount = max number of descendants any ancestor can have * limitDescendantSize = max size of descendants any ancestor can have * errString = populated with error reason if any limits are hit * fSearchForParents = whether to search a tx's vin for in-mempool parents, * or look up parents from mapLinks. Must be true for entries not in the * mempool */ bool CalculateMemPoolAncestors( const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents = true) const; /** * Populate setDescendants with all in-mempool descendants of hash. * Assumes that setDescendants includes all in-mempool descendants of * anything already in it. */ void CalculateDescendants(txiter it, setEntries &setDescendants) const; /** * The minimum fee to get into the mempool, which may itself not be enough * for larger-sized transactions. The incrementalRelayFee policy variable is * used to bound the time it takes the fee rate to go back down all the way * to 0. When the feerate would otherwise be half of this, it is set to 0 * instead. */ CFeeRate GetMinFee(size_t sizelimit) const; /** * Remove transactions from the mempool until its dynamic size is <= * sizelimit. pvNoSpendsRemaining, if set, will be populated with the list * of outpoints which are not in mempool which no longer have any spends in * this mempool. */ void TrimToSize(size_t sizelimit, std::vector *pvNoSpendsRemaining = nullptr); /** * Expire all transaction (and their dependencies) in the mempool older than * time. Return the number of removed transactions. */ int Expire(int64_t time); /** * Reduce the size of the mempool by expiring and then trimming the mempool. */ void LimitSize(size_t limit, unsigned long age); /** * Returns false if the transaction is in the mempool and not within the * chain limit specified. */ bool TransactionWithinChainLimit(const uint256 &txid, size_t chainLimit) const; unsigned long size() { LOCK(cs); return mapTx.size(); } uint64_t GetTotalTxSize() const { LOCK(cs); return totalTxSize; } bool exists(uint256 hash) const { LOCK(cs); return mapTx.count(hash) != 0; } bool exists(const COutPoint &outpoint) const { LOCK(cs); auto it = mapTx.find(outpoint.GetTxId()); return it != mapTx.end() && outpoint.GetN() < it->GetTx().vout.size(); } CTransactionRef get(const uint256 &hash) const; TxMempoolInfo info(const uint256 &hash) const; std::vector infoAll() const; /** * Estimate fee rate needed to get into the next nBlocks. If no answer can * be given at nBlocks, return an estimate at the lowest number of blocks * where one can be given. */ CFeeRate estimateSmartFee(int nBlocks, int *answerFoundAtBlocks = nullptr) const; /** Estimate fee rate needed to get into the next nBlocks */ CFeeRate estimateFee(int nBlocks) const; - /** Write/Read estimates to disk */ - bool WriteFeeEstimates(CAutoFile &fileout) const; - bool ReadFeeEstimates(CAutoFile &filein); - size_t DynamicMemoryUsage() const; boost::signals2::signal NotifyEntryAdded; boost::signals2::signal NotifyEntryRemoved; private: /** * UpdateForDescendants is used by UpdateTransactionsFromBlock to update the * descendants for a single transaction that has been added to the mempool * but may have child transactions in the mempool, eg during a chain reorg. * setExclude is the set of descendant transactions in the mempool that must * not be accounted for (because any descendants in setExclude were added to * the mempool after the transaction being updated and hence their state is * already reflected in the parent state). * * cachedDescendants will be updated with the descendants of the transaction * being updated, so that future invocations don't need to walk the same * transaction again, if encountered in another transaction chain. */ void UpdateForDescendants(txiter updateIt, cacheMap &cachedDescendants, const std::set &setExclude); /** * Update ancestors of hash to add/remove it as a descendant transaction. */ void UpdateAncestorsOf(bool add, txiter hash, setEntries &setAncestors); /** Set ancestor state for an entry */ void UpdateEntryForAncestors(txiter it, const setEntries &setAncestors); /** * For each transaction being removed, update ancestors and any direct * children. If updateDescendants is true, then also update in-mempool * descendants' ancestor state. */ void UpdateForRemoveFromMempool(const setEntries &entriesToRemove, bool updateDescendants); /** Sever link between specified transaction and direct children. */ void UpdateChildrenForRemoval(txiter entry); /** * Before calling removeUnchecked for a given transaction, * UpdateForRemoveFromMempool must be called on the entire (dependent) set * of transactions being removed at the same time. We use each * CTxMemPoolEntry's setMemPoolParents in order to walk ancestors of a given * transaction that is removed, so we can't remove intermediate transactions * in a chain before we've updated all the state for the removal. */ void removeUnchecked(txiter entry, MemPoolRemovalReason reason = MemPoolRemovalReason::UNKNOWN); }; /** * CCoinsView that brings transactions from a memorypool into view. * It does not check for spendings by memory pool transactions. */ class CCoinsViewMemPool : public CCoinsViewBacked { protected: const CTxMemPool &mempool; public: CCoinsViewMemPool(CCoinsView *baseIn, const CTxMemPool &mempoolIn); bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; bool HaveCoin(const COutPoint &outpoint) const override; }; // We want to sort transactions by coin age priority typedef std::pair TxCoinAgePriority; struct TxCoinAgePriorityCompare { bool operator()(const TxCoinAgePriority &a, const TxCoinAgePriority &b) { if (a.first == b.first) { // Reverse order to make sort less than return CompareTxMemPoolEntryByScore()(*(b.second), *(a.second)); } return a.first < b.first; } }; /** * DisconnectedBlockTransactions * * During the reorg, it's desirable to re-add previously confirmed transactions * to the mempool, so that anything not re-confirmed in the new chain is * available to be mined. However, it's more efficient to wait until the reorg * is complete and process all still-unconfirmed transactions at that time, * since we expect most confirmed transactions to (typically) still be * confirmed in the new chain, and re-accepting to the memory pool is expensive * (and therefore better to not do in the middle of reorg-processing). * Instead, store the disconnected transactions (in order!) as we go, remove any * that are included in blocks in the new chain, and then process the remaining * still-unconfirmed transactions at the end. */ // multi_index tag names struct txid_index {}; struct insertion_order {}; class DisconnectedBlockTransactions { private: typedef boost::multi_index_container< CTransactionRef, boost::multi_index::indexed_by< // sorted by txid boost::multi_index::hashed_unique< boost::multi_index::tag, mempoolentry_txid, SaltedTxidHasher>, // sorted by order in the blockchain boost::multi_index::sequenced< boost::multi_index::tag>>> indexed_disconnected_transactions; indexed_disconnected_transactions queuedTx; uint64_t cachedInnerUsage = 0; void addTransaction(const CTransactionRef &tx) { queuedTx.insert(tx); cachedInnerUsage += RecursiveDynamicUsage(tx); } public: // It's almost certainly a logic bug if we don't clear out queuedTx before // destruction, as we add to it while disconnecting blocks, and then we // need to re-process remaining transactions to ensure mempool consistency. // For now, assert() that we've emptied out this object on destruction. // This assert() can always be removed if the reorg-processing code were // to be refactored such that this assumption is no longer true (for // instance if there was some other way we cleaned up the mempool after a // reorg, besides draining this object). ~DisconnectedBlockTransactions() { assert(queuedTx.empty()); } // Estimate the overhead of queuedTx to be 6 pointers + an allocation, as // no exact formula for boost::multi_index_contained is implemented. size_t DynamicMemoryUsage() const { return memusage::MallocUsage(sizeof(CTransactionRef) + 6 * sizeof(void *)) * queuedTx.size() + cachedInnerUsage; } const indexed_disconnected_transactions &GetQueuedTx() const { return queuedTx; } // Add entries for a block while reconstructing the topological ordering so // they can be added back to the mempool simply. void addForBlock(const std::vector &vtx); // Remove entries based on txid_index, and update memory usage. void removeForBlock(const std::vector &vtx) { // Short-circuit in the common case of a block being added to the tip if (queuedTx.empty()) { return; } for (auto const &tx : vtx) { auto it = queuedTx.find(tx->GetId()); if (it != queuedTx.end()) { cachedInnerUsage -= RecursiveDynamicUsage(*it); queuedTx.erase(it); } } } // Remove an entry by insertion_order index, and update memory usage. void removeEntry(indexed_disconnected_transactions::index< insertion_order>::type::iterator entry) { cachedInnerUsage -= RecursiveDynamicUsage(*entry); queuedTx.get().erase(entry); } void clear() { cachedInnerUsage = 0; queuedTx.clear(); } /** * Make mempool consistent after a reorg, by re-adding or recursively * erasing disconnected block transactions from the mempool, and also * removing any other transactions from the mempool that are no longer valid * given the new tip/height. * * Note: we assume that disconnectpool only contains transactions that are * NOT confirmed in the current chain nor already in the mempool (otherwise, * in-mempool descendants of such transactions would be removed). * * Passing fAddToMempool=false will skip trying to add the transactions * back, and instead just erase from the mempool as needed. */ void updateMempoolForReorg(const Config &config, bool fAddToMempool); }; #endif // BITCOIN_TXMEMPOOL_H diff --git a/src/wallet/fees.h b/src/wallet/fees.h index 029fd56673..a7b57f700f 100644 --- a/src/wallet/fees.h +++ b/src/wallet/fees.h @@ -1,33 +1,32 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2017 The Bitcoin Core developers // Copyright (c) 2018 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_WALLET_FEES_H #define BITCOIN_WALLET_FEES_H #include "amount.h" -class CBlockPolicyEstimator; class CCoinControl; class CFeeRate; class CTxMemPool; struct FeeCalculation; /** * Estimate the minimum fee considering user set parameters * and the required fee */ Amount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool &pool); /** * Estimate the minimum fee considering required fee and targetFee or if 0 * then fee estimation for nConfirmTarget */ Amount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool &pool, Amount targetFee); #endif // BITCOIN_WALLET_FEES_H diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 9f3a58ed6b..c15ced2b6d 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,517 +1,515 @@ #!/usr/bin/env python3 # Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" from collections import deque from enum import Enum import logging import argparse import os import pdb import shutil import sys import tempfile import time import traceback from .authproxy import JSONRPCException from . import coverage from .test_node import TestNode from .util import ( MAX_NODES, PortSeed, assert_equal, check_json_precision, connect_nodes_bi, disconnect_nodes, initialize_datadir, log_filename, p2p_port, rpc_port, set_node_times, sync_blocks, sync_mempools, ) class TestStatus(Enum): PASSED = 1 FAILED = 2 SKIPPED = 3 TEST_EXIT_PASSED = 0 TEST_EXIT_FAILED = 1 TEST_EXIT_SKIPPED = 77 class BitcoinTestFramework(): """Base class for a bitcoin test script. Individual bitcoin test scripts should subclass this class and override the set_test_params() and run_test() methods. Individual tests can also override the following methods to customize the test setup: - add_options() - setup_chain() - setup_network() - setup_nodes() The __init__() and main() methods should not be overridden. This class also contains various public and private helper methods.""" def __init__(self): """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method""" self.setup_clean_chain = False self.nodes = [] self.mocktime = 0 self.supports_cli = False def main(self): """Main function. This should not be overridden by the subclass test scripts.""" parser = argparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") parser.add_argument("--noshutdown", dest="noshutdown", default=False, action="store_true", help="Don't stop bitcoinds after the test execution") parser.add_argument("--srcdir", dest="srcdir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../../../src"), help="Source directory containing bitcoind/bitcoin-cli (default: %(default)s)") parser.add_argument("--cachedir", dest="cachedir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../../cache"), help="Directory for caching pregenerated datadirs") parser.add_argument("--tmpdir", dest="tmpdir", help="Root directory for datadirs") parser.add_argument("-l", "--loglevel", dest="loglevel", default="INFO", help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory.") parser.add_argument("--tracerpc", dest="trace_rpc", default=False, action="store_true", help="Print out all RPC calls as they are made") parser.add_argument("--portseed", dest="port_seed", default=os.getpid(), type=int, help="The seed to use for assigning port numbers (default: current process id)") parser.add_argument("--coveragedir", dest="coveragedir", help="Write tested RPC commands into this directory") parser.add_argument("--configfile", dest="configfile", help="Location of the test framework config file") parser.add_argument("--pdbonfailure", dest="pdbonfailure", default=False, action="store_true", help="Attach a python debugger if test fails") parser.add_argument("--usecli", dest="usecli", default=False, action="store_true", help="use bitcoin-cli instead of RPC for all commands") self.add_options(parser) self.options = parser.parse_args() self.set_test_params() assert hasattr( self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" PortSeed.n = self.options.port_seed os.environ['PATH'] = self.options.srcdir + ":" + \ self.options.srcdir + "/qt:" + os.environ['PATH'] check_json_precision() self.options.cachedir = os.path.abspath(self.options.cachedir) # Set up temp directory and start logging if self.options.tmpdir: self.options.tmpdir = os.path.abspath(self.options.tmpdir) os.makedirs(self.options.tmpdir, exist_ok=False) else: self.options.tmpdir = tempfile.mkdtemp(prefix="test") self._start_logging() success = TestStatus.FAILED try: if self.options.usecli and not self.supports_cli: raise SkipTest( "--usecli specified but test does not support using CLI") self.setup_chain() self.setup_network() self.run_test() success = TestStatus.PASSED except JSONRPCException as e: self.log.exception("JSONRPC error") except SkipTest as e: self.log.warning("Test Skipped: %s" % e.message) success = TestStatus.SKIPPED except AssertionError as e: self.log.exception("Assertion failed") except KeyError as e: self.log.exception("Key error") except Exception as e: self.log.exception("Unexpected exception caught during testing") except KeyboardInterrupt as e: self.log.warning("Exiting after keyboard interrupt") if success == TestStatus.FAILED and self.options.pdbonfailure: print("Testcase failed. Attaching python debugger. Enter ? for help") pdb.set_trace() if not self.options.noshutdown: self.log.info("Stopping nodes") if self.nodes: self.stop_nodes() else: self.log.info( "Note: bitcoinds were not stopped and may still be running") if not self.options.nocleanup and not self.options.noshutdown and success != TestStatus.FAILED: self.log.info("Cleaning up") shutil.rmtree(self.options.tmpdir) else: self.log.warning("Not cleaning up dir %s" % self.options.tmpdir) if os.getenv("PYTHON_DEBUG", ""): # Dump the end of the debug logs, to aid in debugging rare # travis failures. import glob filenames = [self.options.tmpdir + "/test_framework.log"] filenames += glob.glob(self.options.tmpdir + "/node*/regtest/debug.log") MAX_LINES_TO_PRINT = 1000 for fn in filenames: try: with open(fn, 'r') as f: print("From", fn, ":") print("".join(deque(f, MAX_LINES_TO_PRINT))) except OSError: print("Opening file %s failed." % fn) traceback.print_exc() if success == TestStatus.PASSED: self.log.info("Tests successful") sys.exit(TEST_EXIT_PASSED) elif success == TestStatus.SKIPPED: self.log.info("Test skipped") sys.exit(TEST_EXIT_SKIPPED) else: self.log.error( "Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) logging.shutdown() sys.exit(TEST_EXIT_FAILED) # Methods to override in subclass test scripts. def set_test_params(self): """Tests must this method to change default values for number of nodes, topology, etc""" raise NotImplementedError def add_options(self, parser): """Override this method to add command-line options to the test""" pass def setup_chain(self): """Override this method to customize blockchain setup""" self.log.info("Initializing test directory " + self.options.tmpdir) if self.setup_clean_chain: self._initialize_chain_clean() else: self._initialize_chain() def setup_network(self): """Override this method to customize test network topology""" self.setup_nodes() # Connect the nodes as a "chain". This allows us # to split the network between nodes 1 and 2 to get # two halves that can work on competing chains. for i in range(self.num_nodes - 1): connect_nodes_bi(self.nodes[i], self.nodes[i + 1]) self.sync_all() def setup_nodes(self): """Override this method to customize test node setup""" extra_args = None if hasattr(self, "extra_args"): extra_args = self.extra_args self.add_nodes(self.num_nodes, extra_args) self.start_nodes() def run_test(self): """Tests must override this method to define test logic""" raise NotImplementedError # Public helper methods. These can be accessed by the subclass test scripts. def add_nodes(self, num_nodes, extra_args=None, rpchost=None, timewait=None, binary=None): """Instantiate TestNode objects""" if extra_args is None: extra_args = [[]] * num_nodes if binary is None: binary = [None] * num_nodes assert_equal(len(extra_args), num_nodes) assert_equal(len(binary), num_nodes) for i in range(num_nodes): self.nodes.append(TestNode(i, self.options.tmpdir, extra_args[i], rpchost, rpc_port=rpc_port(i), p2p_port=p2p_port(i), timewait=timewait, binary=binary[i], stderr=None, mocktime=self.mocktime, coverage_dir=self.options.coveragedir, use_cli=self.options.usecli)) def start_node(self, i, *args, **kwargs): """Start a bitcoind""" node = self.nodes[i] node.start(*args, **kwargs) node.wait_for_rpc_connection() if self.options.coveragedir is not None: coverage.write_all_rpc_commands(self.options.coveragedir, node.rpc) def start_nodes(self, extra_args=None, *args, **kwargs): """Start multiple bitcoinds""" if extra_args is None: extra_args = [None] * self.num_nodes assert_equal(len(extra_args), self.num_nodes) try: for i, node in enumerate(self.nodes): node.start(extra_args[i], *args, **kwargs) for node in self.nodes: node.wait_for_rpc_connection() except: # If one node failed to start, stop the others self.stop_nodes() raise if self.options.coveragedir is not None: for node in self.nodes: coverage.write_all_rpc_commands( self.options.coveragedir, node.rpc) def stop_node(self, i): """Stop a bitcoind test node""" self.nodes[i].stop_node() self.nodes[i].wait_until_stopped() def stop_nodes(self): """Stop multiple bitcoind test nodes""" for node in self.nodes: # Issue RPC to stop nodes node.stop_node() for node in self.nodes: # Wait for nodes to stop node.wait_until_stopped() def restart_node(self, i, extra_args=None): """Stop and start a test node""" self.stop_node(i) self.start_node(i, extra_args) def assert_start_raises_init_error(self, i, extra_args=None, expected_msg=None, *args, **kwargs): with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr: try: self.start_node( i, extra_args, stderr=log_stderr, *args, **kwargs) self.stop_node(i) except Exception as e: assert 'bitcoind exited' in str(e) # node must have shutdown self.nodes[i].running = False self.nodes[i].process = None if expected_msg is not None: log_stderr.seek(0) stderr = log_stderr.read().decode('utf-8') if expected_msg not in stderr: raise AssertionError( "Expected error \"" + expected_msg + "\" not found in:\n" + stderr) else: if expected_msg is None: assert_msg = "bitcoind should have exited with an error" else: assert_msg = "bitcoind should have exited with expected error " + expected_msg raise AssertionError(assert_msg) def wait_for_node_exit(self, i, timeout): self.nodes[i].process.wait(timeout) def split_network(self): """ Split the network of four nodes into nodes 0/1 and 2/3. """ disconnect_nodes(self.nodes[1], self.nodes[2]) disconnect_nodes(self.nodes[2], self.nodes[1]) self.sync_all([self.nodes[:2], self.nodes[2:]]) def join_network(self): """ Join the (previously split) network halves together. """ connect_nodes_bi(self.nodes[1], self.nodes[2]) self.sync_all() def sync_all(self, node_groups=None): if not node_groups: node_groups = [self.nodes] for group in node_groups: sync_blocks(group) sync_mempools(group) def enable_mocktime(self, mocktime): """Enable mocktime for the script. mocktime may be needed for scripts that use the cached version of the blockchain. If the cached version of the blockchain is used without mocktime then the mempools will not sync due to IBD. """ if self.mocktime == 0: self.mocktime = mocktime else: self.log.warning("mocktime overriden by test.") def disable_mocktime(self): self.mocktime = 0 # Private helper methods. These should not be accessed by the subclass test scripts. def _start_logging(self): # Add logger and logging handlers self.log = logging.getLogger('TestFramework') self.log.setLevel(logging.DEBUG) # Create file handler to log all messages fh = logging.FileHandler(self.options.tmpdir + '/test_framework.log') fh.setLevel(logging.DEBUG) # Create console handler to log messages to stderr. By default this # logs only error messages, but can be configured with --loglevel. ch = logging.StreamHandler(sys.stdout) # User can provide log level as a number or string (eg DEBUG). loglevel # was caught as a string, so try to convert it to an int ll = int(self.options.loglevel) if self.options.loglevel.isdigit( ) else self.options.loglevel.upper() ch.setLevel(ll) # Format logs the same as bitcoind's debug.log with microprecision (so log files can be concatenated and sorted) formatter = logging.Formatter( fmt='%(asctime)s.%(msecs)03d000 %(name)s (%(levelname)s): %(message)s', datefmt='%Y-%m-%d %H:%M:%S') formatter.converter = time.gmtime fh.setFormatter(formatter) ch.setFormatter(formatter) # add the handlers to the logger self.log.addHandler(fh) self.log.addHandler(ch) if self.options.trace_rpc: rpc_logger = logging.getLogger("BitcoinRPC") rpc_logger.setLevel(logging.DEBUG) rpc_handler = logging.StreamHandler(sys.stdout) rpc_handler.setLevel(logging.DEBUG) rpc_logger.addHandler(rpc_handler) def _initialize_chain(self): """Initialize a pre-mined blockchain for use by the test. Create a cache of a 200-block-long chain (with wallet) for MAX_NODES Afterward, create num_nodes copies from the cache.""" assert self.num_nodes <= MAX_NODES create_cache = False for i in range(MAX_NODES): if not os.path.isdir(os.path.join(self.options.cachedir, 'node' + str(i))): create_cache = True break if create_cache: self.log.debug("Creating data directories from cached datadir") # find and delete old cache directories if any exist for i in range(MAX_NODES): if os.path.isdir(os.path.join(self.options.cachedir, "node" + str(i))): shutil.rmtree(os.path.join( self.options.cachedir, "node" + str(i))) # Create cache directories, run bitcoinds: for i in range(MAX_NODES): datadir = initialize_datadir(self.options.cachedir, i) args = [os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir=" + datadir, "-discover=0"] if i > 0: args.append("-connect=127.0.0.1:" + str(p2p_port(0))) self.nodes.append(TestNode(i, self.options.cachedir, extra_args=[], host=None, rpc_port=rpc_port(i), p2p_port=p2p_port(i), timewait=None, binary=None, stderr=None, mocktime=self.mocktime, coverage_dir=None)) self.nodes[i].args = args self.start_node(i) # Wait for RPC connections to be ready for node in self.nodes: node.wait_for_rpc_connection() # Create a 200-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. # Note: To preserve compatibility with older versions of # initialize_chain, only 4 nodes will generate coins. # # blocks are created with timestamps 10 minutes apart # starting from 2010 minutes in the past self.enable_mocktime(int(time.time()) - (201 * 10 * 60)) for i in range(2): for peer in range(4): for j in range(25): set_node_times(self.nodes, self.mocktime) self.nodes[peer].generate(1) self.mocktime += 10 * 60 # Must sync before next peer starts generating blocks sync_blocks(self.nodes) # Shut them down, and clean up cache directories: self.stop_nodes() self.nodes = [] self.disable_mocktime() for i in range(MAX_NODES): os.remove(log_filename(self.options.cachedir, i, "debug.log")) os.remove(log_filename( self.options.cachedir, i, "wallets/db.log")) os.remove(log_filename(self.options.cachedir, i, "peers.dat")) - os.remove(log_filename( - self.options.cachedir, i, "fee_estimates.dat")) for i in range(self.num_nodes): from_dir = os.path.join(self.options.cachedir, "node" + str(i)) to_dir = os.path.join(self.options.tmpdir, "node" + str(i)) shutil.copytree(from_dir, to_dir) # Overwrite port/rpcport in bitcoin.conf initialize_datadir(self.options.tmpdir, i) def _initialize_chain_clean(self): """Initialize empty blockchain for use by the test. Create an empty blockchain and num_nodes wallets. Useful if a test case wants complete control over initialization.""" for i in range(self.num_nodes): initialize_datadir(self.options.tmpdir, i) class ComparisonTestFramework(BitcoinTestFramework): """Test framework for doing p2p comparison testing Sets up some bitcoind binaries: - 1 binary: test binary - 2 binaries: 1 test binary, 1 ref binary - n>2 binaries: 1 test binary, n-1 ref binaries""" def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True def add_options(self, parser): parser.add_argument("--testbinary", dest="testbinary", default=os.getenv("BITCOIND", "bitcoind"), help="bitcoind binary to test") parser.add_argument("--refbinary", dest="refbinary", default=os.getenv("BITCOIND", "bitcoind"), help="bitcoind binary to use for reference nodes (if any)") def setup_network(self): extra_args = [['-whitelist=127.0.0.1']] * self.num_nodes if hasattr(self, "extra_args"): extra_args = self.extra_args self.add_nodes(self.num_nodes, extra_args, binary=[self.options.testbinary] + [self.options.refbinary] * (self.num_nodes - 1)) self.start_nodes() class SkipTest(Exception): """This exception is raised to skip a test""" def __init__(self, message): self.message = message