diff --git a/doc/REST-interface.md b/doc/REST-interface.md --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -84,6 +84,7 @@ Returns various information about the TX mempool. Only supports JSON as output format. +* loaded : (boolean) if the mempool is fully loaded * size : (numeric) the number of transactions in the TX mempool * bytes : (numeric) size of the TX mempool in bytes * usage : (numeric) total TX mempool memory usage diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -26,6 +26,7 @@ bench/gcs_filter.cpp \ bench/merkle_root.cpp \ bench/mempool_eviction.cpp \ + bench/rpc_mempool.cpp \ bench/base58.cpp \ bench/lockedpool.cpp \ bench/prevector.cpp @@ -45,7 +46,9 @@ $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ - $(LIBUNIVALUE) + $(LIBUNIVALUE) \ + $(EVENT_PTHREADS_LIBS) \ + $(EVENT_LIBS) if ENABLE_ZMQ bench_bench_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt --- a/src/bench/CMakeLists.txt +++ b/src/bench/CMakeLists.txt @@ -51,6 +51,7 @@ merkle_root.cpp prevector.cpp rollingbloom.cpp + rpc_mempool.cpp # Add the generated headers to trigger the conversion command ${BENCH_DATA_GENERATED_HEADERS} diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp new file mode 100644 --- /dev/null +++ b/src/bench/rpc_mempool.cpp @@ -0,0 +1,45 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include + +#include + +#include +#include + +static void AddTx(const CTransactionRef &tx, const Amount &fee, + CTxMemPool &pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) { + LockPoints lp; + pool.addUnchecked(tx->GetId(), + CTxMemPoolEntry(tx, fee, /* time */ 0, /*priority*/ 10.0, + /* height */ 1, tx->GetValueOut(), + /* spendsCoinbase */ false, + /* sigOpCost */ 4, lp)); +} + +static void RpcMempool(benchmark::State &state) { + CTxMemPool pool; + LOCK2(cs_main, pool.cs); + + for (int i = 0; i < 1000; ++i) { + CMutableTransaction tx = CMutableTransaction(); + tx.vin.resize(1); + tx.vin[0].scriptSig = CScript() << OP_1; + tx.vout.resize(1); + tx.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; + tx.vout[0].nValue = i * CENT; + const CTransactionRef tx_r{MakeTransactionRef(tx)}; + AddTx(tx_r, /* fee */ i * CENT, pool); + } + + while (state.KeepRunning()) { + (void)MempoolToJSON(pool, /*verbose*/ true); + } +} + +BENCHMARK(RpcMempool, 40); diff --git a/src/init.cpp b/src/init.cpp --- a/src/init.cpp +++ b/src/init.cpp @@ -237,9 +237,9 @@ g_banman.reset(); g_txindex.reset(); - if (g_is_mempool_loaded && + if (::g_mempool.IsLoaded() && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - DumpMempool(); + DumpMempool(::g_mempool); } // FlushStateToDisk generates a ChainStateFlushed callback, which we should @@ -1207,9 +1207,9 @@ } } // End scope of CImportingNow if (gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - LoadMempool(config); + LoadMempool(config, ::g_mempool); } - g_is_mempool_loaded = !fRequestShutdown; + ::g_mempool.SetIsLoaded(!fRequestShutdown); } /** Sanity checks diff --git a/src/rest.cpp b/src/rest.cpp --- a/src/rest.cpp +++ b/src/rest.cpp @@ -330,7 +330,7 @@ switch (rf) { case RetFormat::JSON: { - UniValue mempoolInfoObject = mempoolInfoToJSON(); + UniValue mempoolInfoObject = MempoolInfoToJSON(::g_mempool); std::string strJSON = mempoolInfoObject.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); @@ -355,7 +355,7 @@ switch (rf) { case RetFormat::JSON: { - UniValue mempoolObject = mempoolToJSON(true); + UniValue mempoolObject = MempoolToJSON(::g_mempool, true); std::string strJSON = mempoolObject.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -11,6 +11,7 @@ class CBlockIndex; class Config; class JSONRPCRequest; +class CTxMemPool; UniValue getblockchaininfo(const Config &config, const JSONRPCRequest &request); @@ -30,10 +31,10 @@ const CBlockIndex *blockindex, bool txDetails = false); /** Mempool information to JSON */ -UniValue mempoolInfoToJSON(); +UniValue MempoolInfoToJSON(const CTxMemPool &pool); /** Mempool to JSON */ -UniValue mempoolToJSON(bool fVerbose = false); +UniValue MempoolToJSON(const CTxMemPool &pool, bool verbose = false); /** Block header to JSON */ UniValue blockheaderToJSON(const CBlockIndex *tip, diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -457,9 +457,10 @@ " ... ]\n"; } -static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) - EXCLUSIVE_LOCKS_REQUIRED(g_mempool.cs) { - AssertLockHeld(g_mempool.cs); +static void entryToJSON(const CTxMemPool &pool, UniValue &info, + const CTxMemPoolEntry &e) + EXCLUSIVE_LOCKS_REQUIRED(pool.cs) { + AssertLockHeld(pool.cs); UniValue fees(UniValue::VOBJ); fees.pushKV("base", ValueFromAmount(e.GetFee())); @@ -484,7 +485,7 @@ const CTransaction &tx = e.GetTx(); std::set setDepends; for (const CTxIn &txin : tx.vin) { - if (g_mempool.exists(txin.prevout.GetTxId())) { + if (pool.exists(txin.prevout.GetTxId())) { setDepends.insert(txin.prevout.GetTxId().ToString()); } } @@ -497,9 +498,8 @@ info.pushKV("depends", depends); UniValue spent(UniValue::VARR); - const CTxMemPool::txiter &it = g_mempool.mapTx.find(tx.GetId()); - const CTxMemPool::setEntries &setChildren = - g_mempool.GetMemPoolChildren(it); + const CTxMemPool::txiter &it = pool.mapTx.find(tx.GetId()); + const CTxMemPool::setEntries &setChildren = pool.GetMemPoolChildren(it); for (CTxMemPool::txiter childiter : setChildren) { spent.push_back(childiter->GetTx().GetId().ToString()); } @@ -507,20 +507,20 @@ info.pushKV("spentby", spent); } -UniValue mempoolToJSON(bool fVerbose) { - if (fVerbose) { - LOCK(g_mempool.cs); +UniValue MempoolToJSON(const CTxMemPool &pool, bool verbose) { + if (verbose) { + LOCK(pool.cs); UniValue o(UniValue::VOBJ); - for (const CTxMemPoolEntry &e : g_mempool.mapTx) { + for (const CTxMemPoolEntry &e : pool.mapTx) { const uint256 &txid = e.GetTx().GetId(); UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(pool, info, e); o.pushKV(txid.ToString(), info); } return o; } else { std::vector vtxids; - g_mempool.queryHashes(vtxids); + pool.queryHashes(vtxids); UniValue a(UniValue::VARR); for (const uint256 &txid : vtxids) { @@ -564,7 +564,7 @@ fVerbose = request.params[0].get_bool(); } - return mempoolToJSON(fVerbose); + return MempoolToJSON(::g_mempool, fVerbose); } static UniValue getmempoolancestors(const Config &config, @@ -630,7 +630,7 @@ const CTxMemPoolEntry &e = *ancestorIt; const uint256 &_hash = e.GetTx().GetId(); UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(::g_mempool, info, e); o.pushKV(_hash.ToString(), info); } return o; @@ -699,7 +699,7 @@ const CTxMemPoolEntry &e = *descendantIt; const uint256 &_hash = e.GetTx().GetId(); UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(::g_mempool, info, e); o.pushKV(_hash.ToString(), info); } return o; @@ -736,7 +736,7 @@ const CTxMemPoolEntry &e = *it; UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(::g_mempool, info, e); return info; } @@ -1485,18 +1485,19 @@ return res; } -UniValue mempoolInfoToJSON() { +UniValue MempoolInfoToJSON(const CTxMemPool &pool) { UniValue ret(UniValue::VOBJ); - ret.pushKV("size", (int64_t)g_mempool.size()); - ret.pushKV("bytes", (int64_t)g_mempool.GetTotalTxSize()); - ret.pushKV("usage", (int64_t)g_mempool.DynamicMemoryUsage()); + ret.pushKV("loaded", pool.IsLoaded()); + ret.pushKV("size", (int64_t)pool.size()); + ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); + ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); size_t maxmempool = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; ret.pushKV("maxmempool", (int64_t)maxmempool); - ret.pushKV("mempoolminfee", - ValueFromAmount( - std::max(g_mempool.GetMinFee(maxmempool), ::minRelayTxFee) - .GetFeePerK())); + ret.pushKV( + "mempoolminfee", + ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee) + .GetFeePerK())); ret.pushKV("minrelaytxfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); return ret; @@ -1510,6 +1511,8 @@ "\nReturns details on the active state of the TX memory pool.\n" "\nResult:\n" "{\n" + " \"loaded\": true|false (boolean) True if the mempool is " + "fully loaded\n" " \"size\": xxxxx, (numeric) Current tx count\n" " \"bytes\": xxxxx, (numeric) Transaction size.\n" " \"usage\": xxxxx, (numeric) Total memory usage for " @@ -1528,7 +1531,7 @@ HelpExampleRpc("getmempoolinfo", "")); } - return mempoolInfoToJSON(); + return MempoolInfoToJSON(::g_mempool); } static UniValue preciousblock(const Config &config, @@ -2197,11 +2200,11 @@ HelpExampleRpc("savemempool", "")); } - if (!g_is_mempool_loaded) { + if (!::g_mempool.IsLoaded()) { throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); } - if (!DumpMempool()) { + if (!DumpMempool(::g_mempool)) { throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk"); } diff --git a/src/txmempool.h b/src/txmempool.h --- a/src/txmempool.h +++ b/src/txmempool.h @@ -225,7 +225,7 @@ const LockPoints &lp; }; -// extracts a transaction hash from CTxMempoolEntry or CTransactionRef +// extracts a transaction hash from CTxMemPoolEntry or CTransactionRef struct mempoolentry_txid { typedef uint256 result_type; result_type operator()(const CTxMemPoolEntry &entry) const { @@ -496,6 +496,8 @@ void trackPackageRemoved(const CFeeRate &rate) EXCLUSIVE_LOCKS_REQUIRED(cs); + bool m_is_loaded GUARDED_BY(cs){false}; + public: // public only for testing static const int ROLLING_FEE_HALFLIFE = 60 * 60 * 12; @@ -645,7 +647,7 @@ // lock free void _clear() EXCLUSIVE_LOCKS_REQUIRED(cs); bool CompareDepthAndScore(const uint256 &hasha, const uint256 &hashb); - void queryHashes(std::vector &vtxid); + void queryHashes(std::vector &vtxid) const; bool isSpent(const COutPoint &outpoint) const; unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); @@ -753,6 +755,12 @@ void GetTransactionAncestry(const uint256 &txid, size_t &ancestors, size_t &descendants) const; + /** @returns true if the mempool is fully loaded */ + bool IsLoaded() const; + + /** Sets the current loaded state */ + void SetIsLoaded(bool loaded); + unsigned long size() const { LOCK(cs); return mapTx.size(); diff --git a/src/txmempool.cpp b/src/txmempool.cpp --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -871,7 +871,7 @@ return iters; } -void CTxMemPool::queryHashes(std::vector &vtxid) { +void CTxMemPool::queryHashes(std::vector &vtxid) const { LOCK(cs); auto iters = GetSortedDepthAndScore(); @@ -1238,6 +1238,16 @@ } } +bool CTxMemPool::IsLoaded() const { + LOCK(cs); + return m_is_loaded; +} + +void CTxMemPool::SetIsLoaded(bool loaded) { + LOCK(cs); + m_is_loaded = loaded; +} + SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} diff --git a/src/validation.h b/src/validation.h --- a/src/validation.h +++ b/src/validation.h @@ -211,7 +211,6 @@ extern CScript COINBASE_FLAGS; extern CCriticalSection cs_main; extern CTxMemPool g_mempool; -extern std::atomic_bool g_is_mempool_loaded; extern uint64_t nLastBlockTx; extern uint64_t nLastBlockSize; extern const std::string strMessageMagic; @@ -730,9 +729,9 @@ CBlockFileInfo *GetBlockFileInfo(size_t n); /** Dump the mempool to disk. */ -bool DumpMempool(); +bool DumpMempool(const CTxMemPool &pool); /** Load the mempool from disk. */ -bool LoadMempool(const Config &config); +bool LoadMempool(const Config &config, CTxMemPool &pool); #endif // BITCOIN_VALIDATION_H diff --git a/src/validation.cpp b/src/validation.cpp --- a/src/validation.cpp +++ b/src/validation.cpp @@ -263,7 +263,6 @@ Amount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; CTxMemPool g_mempool; -std::atomic_bool g_is_mempool_loaded{false}; /** Constant stuff for coinbase transactions we create: */ CScript COINBASE_FLAGS; @@ -5590,7 +5589,7 @@ static const uint64_t MEMPOOL_DUMP_VERSION = 1; -bool LoadMempool(const Config &config) { +bool LoadMempool(const Config &config, CTxMemPool &pool) { int64_t nExpiryTimeout = gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; FILE *filestr = fsbridge::fopen(GetDataDir() / "mempool.dat", "rb"); @@ -5626,14 +5625,14 @@ Amount amountdelta = nFeeDelta * SATOSHI; if (amountdelta != Amount::zero()) { - g_mempool.PrioritiseTransaction(tx->GetId(), prioritydummy, - amountdelta); + pool.PrioritiseTransaction(tx->GetId(), prioritydummy, + amountdelta); } CValidationState state; if (nTime + nExpiryTimeout > nNow) { LOCK(cs_main); AcceptToMemoryPoolWithTime( - config, g_mempool, state, tx, true /* fLimitFree */, + config, pool, state, tx, true /* fLimitFree */, nullptr /* pfMissingInputs */, nTime, false /* fOverrideMempoolLimit */, Amount::zero() /* nAbsurdFee */, false /* test_accept */); @@ -5654,7 +5653,7 @@ file >> mapDeltas; for (const auto &i : mapDeltas) { - g_mempool.PrioritiseTransaction(i.first, prioritydummy, i.second); + pool.PrioritiseTransaction(i.first, prioritydummy, i.second); } } catch (const std::exception &e) { LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing " @@ -5669,7 +5668,7 @@ return true; } -bool DumpMempool() { +bool DumpMempool(const CTxMemPool &pool) { int64_t start = GetTimeMicros(); std::map mapDeltas; @@ -5679,12 +5678,12 @@ LOCK(dump_mutex); { - LOCK(g_mempool.cs); - for (const auto &i : g_mempool.mapDeltas) { + LOCK(pool.cs); + for (const auto &i : pool.mapDeltas) { mapDeltas[i.first] = i.second.second; } - vinfo = g_mempool.infoAll(); + vinfo = pool.infoAll(); } int64_t mid = GetTimeMicros(); diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -37,7 +37,6 @@ """ from decimal import Decimal import os -import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -81,9 +80,10 @@ self.start_node(1) self.start_node(0) self.start_node(2) - # Give bitcoind a second to reload the mempool - wait_until(lambda: len(self.nodes[0].getrawmempool()) == 5, timeout=1) - wait_until(lambda: len(self.nodes[2].getrawmempool()) == 5, timeout=1) + wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"], timeout=1) + wait_until(lambda: self.nodes[2].getmempoolinfo()["loaded"], timeout=1) + assert_equal(len(self.nodes[0].getrawmempool()), 5) + assert_equal(len(self.nodes[2].getrawmempool()), 5) # The others have loaded their mempool. If node_1 loaded anything, we'd probably notice by now: assert_equal(len(self.nodes[1].getrawmempool()), 0) @@ -96,15 +96,15 @@ "Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") self.stop_nodes() self.start_node(0, extra_args=["-persistmempool=0"]) - # Give bitcoind a second to reload the mempool - time.sleep(1) + wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"]) assert_equal(len(self.nodes[0].getrawmempool()), 0) self.log.debug( "Stop-start node0. Verify that it has the transactions in its mempool.") self.stop_nodes() self.start_node(0) - wait_until(lambda: len(self.nodes[0].getrawmempool()) == 5) + wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"]) + assert_equal(len(self.nodes[0].getrawmempool()), 5) mempooldat0 = os.path.join( self.nodes[0].datadir, 'regtest', 'mempool.dat') @@ -121,7 +121,8 @@ os.rename(mempooldat0, mempooldat1) self.stop_nodes() self.start_node(1, extra_args=[]) - wait_until(lambda: len(self.nodes[1].getrawmempool()) == 5) + wait_until(lambda: self.nodes[1].getmempoolinfo()["loaded"]) + assert_equal(len(self.nodes[1].getrawmempool()), 5) self.log.debug( "Prevent bitcoind from writing mempool.dat to disk. Verify that `savemempool` fails")