diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 6bb0f0cf8..166167be4 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -1,129 +1,129 @@ // Copyright (c) 2011-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include static void AddTx(const CTransactionRef &tx, const Amount &nFee, CTxMemPool &pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) { int64_t nTime = 0; unsigned int nHeight = 1; bool spendsCoinbase = false; unsigned int nSigChecks = 1; LockPoints lp; pool.addUnchecked(CTxMemPoolEntry(tx, nFee, nTime, nHeight, spendsCoinbase, nSigChecks, lp)); } // Right now this is only testing eviction performance in an extremely small // mempool. Code needs to be written to generate a much wider variety of // unique transactions for a more meaningful performance measurement. static void MempoolEviction(benchmark::Bench &bench) { TestingSetup test_setup{ CBaseChainParams::REGTEST, /* extra_args */ { "-nodebuglogfile", "-nodebug", }, }; CMutableTransaction tx1 = CMutableTransaction(); tx1.vin.resize(1); tx1.vin[0].scriptSig = CScript() << OP_1; tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; CMutableTransaction tx2 = CMutableTransaction(); tx2.vin.resize(1); tx2.vin[0].scriptSig = CScript() << OP_2; tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_2 << OP_EQUAL; tx2.vout[0].nValue = 10 * COIN; CMutableTransaction tx3 = CMutableTransaction(); tx3.vin.resize(1); tx3.vin[0].prevout = COutPoint(tx2.GetId(), 0); tx3.vin[0].scriptSig = CScript() << OP_2; tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_3 << OP_EQUAL; tx3.vout[0].nValue = 10 * COIN; CMutableTransaction tx4 = CMutableTransaction(); tx4.vin.resize(2); tx4.vin[0].prevout = COutPoint(); tx4.vin[0].scriptSig = CScript() << OP_4; tx4.vin[1].prevout = COutPoint(); tx4.vin[1].scriptSig = CScript() << OP_4; tx4.vout.resize(2); tx4.vout[0].scriptPubKey = CScript() << OP_4 << OP_EQUAL; tx4.vout[0].nValue = 10 * COIN; tx4.vout[1].scriptPubKey = CScript() << OP_4 << OP_EQUAL; tx4.vout[1].nValue = 10 * COIN; CMutableTransaction tx5 = CMutableTransaction(); tx5.vin.resize(2); tx5.vin[0].prevout = COutPoint(tx4.GetId(), 0); tx5.vin[0].scriptSig = CScript() << OP_4; tx5.vin[1].prevout = COutPoint(); tx5.vin[1].scriptSig = CScript() << OP_5; tx5.vout.resize(2); tx5.vout[0].scriptPubKey = CScript() << OP_5 << OP_EQUAL; tx5.vout[0].nValue = 10 * COIN; tx5.vout[1].scriptPubKey = CScript() << OP_5 << OP_EQUAL; tx5.vout[1].nValue = 10 * COIN; CMutableTransaction tx6 = CMutableTransaction(); tx6.vin.resize(2); tx6.vin[0].prevout = COutPoint(tx4.GetId(), 1); tx6.vin[0].scriptSig = CScript() << OP_4; tx6.vin[1].prevout = COutPoint(); tx6.vin[1].scriptSig = CScript() << OP_6; tx6.vout.resize(2); tx6.vout[0].scriptPubKey = CScript() << OP_6 << OP_EQUAL; tx6.vout[0].nValue = 10 * COIN; tx6.vout[1].scriptPubKey = CScript() << OP_6 << OP_EQUAL; tx6.vout[1].nValue = 10 * COIN; CMutableTransaction tx7 = CMutableTransaction(); tx7.vin.resize(2); tx7.vin[0].prevout = COutPoint(tx5.GetId(), 0); tx7.vin[0].scriptSig = CScript() << OP_5; tx7.vin[1].prevout = COutPoint(tx6.GetId(), 0); tx7.vin[1].scriptSig = CScript() << OP_6; tx7.vout.resize(2); tx7.vout[0].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[0].nValue = 10 * COIN; tx7.vout[1].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[1].nValue = 10 * COIN; - CTxMemPool pool; + CTxMemPool &pool = *Assert(test_setup.m_node.mempool); LOCK2(cs_main, pool.cs); // Create transaction references outside the "hot loop" const CTransactionRef tx1_r{MakeTransactionRef(tx1)}; const CTransactionRef tx2_r{MakeTransactionRef(tx2)}; const CTransactionRef tx3_r{MakeTransactionRef(tx3)}; const CTransactionRef tx4_r{MakeTransactionRef(tx4)}; const CTransactionRef tx5_r{MakeTransactionRef(tx5)}; const CTransactionRef tx6_r{MakeTransactionRef(tx6)}; const CTransactionRef tx7_r{MakeTransactionRef(tx7)}; bench.run([&]() NO_THREAD_SAFETY_ANALYSIS { AddTx(tx1_r, 10000 * SATOSHI, pool); AddTx(tx2_r, 5000 * SATOSHI, pool); AddTx(tx3_r, 20000 * SATOSHI, pool); AddTx(tx4_r, 7000 * SATOSHI, pool); AddTx(tx5_r, 1000 * SATOSHI, pool); AddTx(tx6_r, 1100 * SATOSHI, pool); AddTx(tx7_r, 9000 * SATOSHI, pool); pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); pool.TrimToSize(GetSerializeSize(*tx1_r, PROTOCOL_VERSION)); }); } BENCHMARK(MempoolEviction); diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index 04cf28997..262769eed 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -1,39 +1,49 @@ // Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include #include +#include #include #include static void AddTx(const CTransactionRef &tx, const Amount &fee, CTxMemPool &pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) { LockPoints lp; pool.addUnchecked(CTxMemPoolEntry(tx, fee, /* time */ 0, /* height */ 1, /* spendsCoinbase */ false, /*_sigChecks=*/1, lp)); } static void RpcMempool(benchmark::Bench &bench) { - CTxMemPool pool; + const TestingSetup test_setup{ + CBaseChainParams::MAIN, + /* extra_args */ + { + "-nodebuglogfile", + "-nodebug", + }, + }; + CTxMemPool &pool = *Assert(test_setup.m_node.mempool); LOCK2(cs_main, pool.cs); for (int i = 0; i < 1000; ++i) { CMutableTransaction tx = CMutableTransaction(); tx.vin.resize(1); tx.vin[0].scriptSig = CScript() << OP_1; tx.vout.resize(1); tx.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; tx.vout[0].nValue = i * COIN; const CTransactionRef tx_r{MakeTransactionRef(tx)}; AddTx(tx_r, /* fee */ i * COIN, pool); } bench.run([&] { (void)MempoolToJSON(pool, /*verbose*/ true); }); } BENCHMARK(RpcMempool); diff --git a/src/rest.cpp b/src/rest.cpp index 49a64da02..1f199d5b4 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -1,849 +1,849 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using node::GetTransaction; using node::NodeContext; using node::ReadBlockFromDisk; // Allow a max of 15 outpoints to be queried at once. static const size_t MAX_GETUTXOS_OUTPOINTS = 15; enum class RetFormat { UNDEF, BINARY, HEX, JSON, }; static const struct { RetFormat rf; const char *name; } rf_names[] = { {RetFormat::UNDEF, ""}, {RetFormat::BINARY, "bin"}, {RetFormat::HEX, "hex"}, {RetFormat::JSON, "json"}, }; struct CCoin { uint32_t nHeight; CTxOut out; CCoin() : nHeight(0) {} explicit CCoin(Coin in) : nHeight(in.GetHeight()), out(std::move(in.GetTxOut())) {} SERIALIZE_METHODS(CCoin, obj) { uint32_t nTxVerDummy = 0; READWRITE(nTxVerDummy, obj.nHeight, obj.out); } }; static bool RESTERR(HTTPRequest *req, enum HTTPStatusCode status, std::string message) { req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(status, message + "\r\n"); return false; } /** * Get the node context. * * @param[in] req The HTTP request, whose status code will be set if node * context is not found. * @returns Pointer to the node context or nullptr if not found. */ static NodeContext *GetNodeContext(const std::any &context, HTTPRequest *req) { auto node_context = util::AnyPtr(context); if (!node_context) { RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, strprintf("%s:%d (%s)\n" "Internal bug detected: Node context not found!\n" "You may report this issue here: %s\n", __FILE__, __LINE__, __func__, PACKAGE_BUGREPORT)); return nullptr; } return node_context; } /** * Get the node context mempool. * * @param[in] req The HTTP request, whose status code will be set if node * context mempool is not found. * @returns Pointer to the mempool or nullptr if no mempool found. */ static CTxMemPool *GetMemPool(const std::any &context, HTTPRequest *req) { auto node_context = util::AnyPtr(context); if (!node_context || !node_context->mempool) { RESTERR(req, HTTP_NOT_FOUND, "Mempool disabled or instance not found"); return nullptr; } return node_context->mempool.get(); } /** * Get the node context chainstatemanager. * * @param[in] req The HTTP request, whose status code will be set if node * context chainstatemanager is not found. * @returns Pointer to the chainstatemanager or nullptr if none found. */ static ChainstateManager *GetChainman(const std::any &context, HTTPRequest *req) { auto node_context = util::AnyPtr(context); if (!node_context || !node_context->chainman) { RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, strprintf("%s:%d (%s)\n" "Internal bug detected: Chainman disabled or instance" " not found!\n" "You may report this issue here: %s\n", __FILE__, __LINE__, __func__, PACKAGE_BUGREPORT)); return nullptr; } return node_context->chainman.get(); } static RetFormat ParseDataFormat(std::string ¶m, const std::string &strReq) { const std::string::size_type pos = strReq.rfind('.'); if (pos == std::string::npos) { param = strReq; return rf_names[0].rf; } param = strReq.substr(0, pos); const std::string suff(strReq, pos + 1); for (const auto &rf_name : rf_names) { if (suff == rf_name.name) { return rf_name.rf; } } /* If no suffix is found, return original string. */ param = strReq; return rf_names[0].rf; } static std::string AvailableDataFormatsString() { std::string formats; for (const auto &rf_name : rf_names) { if (strlen(rf_name.name) > 0) { formats.append("."); formats.append(rf_name.name); formats.append(", "); } } if (formats.length() > 0) { return formats.substr(0, formats.length() - 2); } return formats; } static bool CheckWarmup(HTTPRequest *req) { std::string statusmessage; if (RPCIsInWarmup(&statusmessage)) { return RESTERR(req, HTTP_SERVICE_UNAVAILABLE, "Service temporarily unavailable: " + statusmessage); } return true; } static bool rest_headers(Config &config, const std::any &context, HTTPRequest *req, const std::string &strURIPart) { if (!CheckWarmup(req)) { return false; } std::string param; const RetFormat rf = ParseDataFormat(param, strURIPart); std::vector path = SplitString(param, '/'); if (path.size() != 2) { return RESTERR(req, HTTP_BAD_REQUEST, "No header count specified. Use " "/rest/headers//.."); } long count = strtol(path[0].c_str(), nullptr, 10); if (count < 1 || count > 2000) { return RESTERR(req, HTTP_BAD_REQUEST, "Header count out of range: " + path[0]); } std::string hashStr = path[1]; uint256 rawHash; if (!ParseHashStr(hashStr, rawHash)) { return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); } const BlockHash hash(rawHash); const CBlockIndex *tip = nullptr; std::vector headers; headers.reserve(count); { ChainstateManager *maybe_chainman = GetChainman(context, req); if (!maybe_chainman) { return false; } ChainstateManager &chainman = *maybe_chainman; LOCK(cs_main); CChain &active_chain = chainman.ActiveChain(); tip = active_chain.Tip(); const CBlockIndex *pindex = chainman.m_blockman.LookupBlockIndex(hash); while (pindex != nullptr && active_chain.Contains(pindex)) { headers.push_back(pindex); if (headers.size() == size_t(count)) { break; } pindex = active_chain.Next(pindex); } } switch (rf) { case RetFormat::BINARY: { CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); for (const CBlockIndex *pindex : headers) { ssHeader << pindex->GetBlockHeader(); } std::string binaryHeader = ssHeader.str(); req->WriteHeader("Content-Type", "application/octet-stream"); req->WriteReply(HTTP_OK, binaryHeader); return true; } case RetFormat::HEX: { CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); for (const CBlockIndex *pindex : headers) { ssHeader << pindex->GetBlockHeader(); } std::string strHex = HexStr(ssHeader) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); return true; } case RetFormat::JSON: { UniValue jsonHeaders(UniValue::VARR); for (const CBlockIndex *pindex : headers) { jsonHeaders.push_back(blockheaderToJSON(tip, pindex)); } std::string strJSON = jsonHeaders.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); return true; } default: { return RESTERR( req, HTTP_NOT_FOUND, "output format not found (available: .bin, .hex, .json)"); } } } static bool rest_block(const Config &config, const std::any &context, HTTPRequest *req, const std::string &strURIPart, bool showTxDetails) { if (!CheckWarmup(req)) { return false; } std::string hashStr; const RetFormat rf = ParseDataFormat(hashStr, strURIPart); uint256 rawHash; if (!ParseHashStr(hashStr, rawHash)) { return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); } const BlockHash hash(rawHash); CBlock block; const CBlockIndex *pblockindex = nullptr; const CBlockIndex *tip = nullptr; ChainstateManager *maybe_chainman = GetChainman(context, req); if (!maybe_chainman) { return false; } ChainstateManager &chainman = *maybe_chainman; { LOCK(cs_main); tip = chainman.ActiveTip(); pblockindex = chainman.m_blockman.LookupBlockIndex(hash); if (!pblockindex) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } if (chainman.m_blockman.IsBlockPruned(pblockindex)) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); } } if (!ReadBlockFromDisk(block, pblockindex, config.GetChainParams().GetConsensus())) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } switch (rf) { case RetFormat::BINARY: { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; std::string binaryBlock = ssBlock.str(); req->WriteHeader("Content-Type", "application/octet-stream"); req->WriteReply(HTTP_OK, binaryBlock); return true; } case RetFormat::HEX: { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; std::string strHex = HexStr(ssBlock) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); return true; } case RetFormat::JSON: { UniValue objBlock = blockToJSON(chainman.m_blockman, block, tip, pblockindex, showTxDetails); std::string strJSON = objBlock.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); return true; } default: { return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } } static bool rest_block_extended(Config &config, const std::any &context, HTTPRequest *req, const std::string &strURIPart) { return rest_block(config, context, req, strURIPart, true); } static bool rest_block_notxdetails(Config &config, const std::any &context, HTTPRequest *req, const std::string &strURIPart) { return rest_block(config, context, req, strURIPart, false); } static bool rest_chaininfo(Config &config, const std::any &context, HTTPRequest *req, const std::string &strURIPart) { if (!CheckWarmup(req)) { return false; } std::string param; const RetFormat rf = ParseDataFormat(param, strURIPart); switch (rf) { case RetFormat::JSON: { JSONRPCRequest jsonRequest; jsonRequest.context = context; jsonRequest.params = UniValue(UniValue::VARR); UniValue chainInfoObject = getblockchaininfo().HandleRequest(config, jsonRequest); std::string strJSON = chainInfoObject.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); return true; } default: { return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); } } } static bool rest_mempool_info(Config &config, const std::any &context, HTTPRequest *req, const std::string &strURIPart) { if (!CheckWarmup(req)) { return false; } const CTxMemPool *mempool = GetMemPool(context, req); if (!mempool) { return false; } std::string param; const RetFormat rf = ParseDataFormat(param, strURIPart); switch (rf) { case RetFormat::JSON: { UniValue mempoolInfoObject = MempoolInfoToJSON(*mempool); std::string strJSON = mempoolInfoObject.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); return true; } default: { return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); } } } static bool rest_mempool_contents(Config &config, const std::any &context, HTTPRequest *req, const std::string &strURIPart) { if (!CheckWarmup(req)) { return false; } const CTxMemPool *mempool = GetMemPool(context, req); if (!mempool) { return false; } std::string param; const RetFormat rf = ParseDataFormat(param, strURIPart); switch (rf) { case RetFormat::JSON: { UniValue mempoolObject = MempoolToJSON(*mempool, true); std::string strJSON = mempoolObject.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); return true; } default: { return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); } } } static bool rest_tx(Config &config, const std::any &context, HTTPRequest *req, const std::string &strURIPart) { if (!CheckWarmup(req)) { return false; } std::string hashStr; const RetFormat rf = ParseDataFormat(hashStr, strURIPart); uint256 hash; if (!ParseHashStr(hashStr, hash)) { return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); } const TxId txid(hash); if (g_txindex) { g_txindex->BlockUntilSyncedToCurrentChain(); } const NodeContext *const node = GetNodeContext(context, req); if (!node) { return false; } BlockHash hashBlock; const CTransactionRef tx = GetTransaction(/* block_index */ nullptr, node->mempool.get(), txid, Params().GetConsensus(), hashBlock); if (!tx) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } switch (rf) { case RetFormat::BINARY: { CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssTx << tx; std::string binaryTx = ssTx.str(); req->WriteHeader("Content-Type", "application/octet-stream"); req->WriteReply(HTTP_OK, binaryTx); return true; } case RetFormat::HEX: { CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssTx << tx; std::string strHex = HexStr(ssTx) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); return true; } case RetFormat::JSON: { UniValue objTx(UniValue::VOBJ); TxToUniv(*tx, hashBlock, objTx); std::string strJSON = objTx.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); return true; } default: { return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } } static bool rest_getutxos(Config &config, const std::any &context, HTTPRequest *req, const std::string &strURIPart) { if (!CheckWarmup(req)) { return false; } std::string param; const RetFormat rf = ParseDataFormat(param, strURIPart); std::vector uriParts; if (param.length() > 1) { std::string strUriParams = param.substr(1); uriParts = SplitString(strUriParams, '/'); } // throw exception in case of an empty request std::string strRequestMutable = req->ReadBody(); if (strRequestMutable.length() == 0 && uriParts.size() == 0) { return RESTERR(req, HTTP_BAD_REQUEST, "Error: empty request"); } bool fInputParsed = false; bool fCheckMemPool = false; std::vector vOutPoints; // parse/deserialize input // input-format = output-format, rest/getutxos/bin requires binary input, // gives binary output, ... if (uriParts.size() > 0) { // inputs is sent over URI scheme // (/rest/getutxos/checkmempool/txid1-n/txid2-n/...) if (uriParts[0] == "checkmempool") { fCheckMemPool = true; } for (size_t i = (fCheckMemPool) ? 1 : 0; i < uriParts.size(); i++) { int32_t nOutput; std::string strTxid = uriParts[i].substr(0, uriParts[i].find('-')); std::string strOutput = uriParts[i].substr(uriParts[i].find('-') + 1); if (!ParseInt32(strOutput, &nOutput) || !IsHex(strTxid)) { return RESTERR(req, HTTP_BAD_REQUEST, "Parse error"); } TxId txid; txid.SetHex(strTxid); vOutPoints.push_back(COutPoint(txid, uint32_t(nOutput))); } if (vOutPoints.size() > 0) { fInputParsed = true; } else { return RESTERR(req, HTTP_BAD_REQUEST, "Error: empty request"); } } switch (rf) { case RetFormat::HEX: { // convert hex to bin, continue then with bin part std::vector strRequestV = ParseHex(strRequestMutable); strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); } // FALLTHROUGH case RetFormat::BINARY: { try { // deserialize only if user sent a request if (strRequestMutable.size() > 0) { // don't allow sending input over URI and HTTP RAW DATA if (fInputParsed) { return RESTERR(req, HTTP_BAD_REQUEST, "Combination of URI scheme inputs and " "raw post data is not allowed"); } CDataStream oss(SER_NETWORK, PROTOCOL_VERSION); oss << strRequestMutable; oss >> fCheckMemPool; oss >> vOutPoints; } } catch (const std::ios_base::failure &) { // abort in case of unreadable binary data return RESTERR(req, HTTP_BAD_REQUEST, "Parse error"); } break; } case RetFormat::JSON: { if (!fInputParsed) { return RESTERR(req, HTTP_BAD_REQUEST, "Error: empty request"); } break; } default: { return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } // limit max outpoints if (vOutPoints.size() > MAX_GETUTXOS_OUTPOINTS) { return RESTERR( req, HTTP_BAD_REQUEST, strprintf("Error: max outpoints exceeded (max: %d, tried: %d)", MAX_GETUTXOS_OUTPOINTS, vOutPoints.size())); } // check spentness and form a bitmap (as well as a JSON capable // human-readable string representation) std::vector bitmap; std::vector outs; std::string bitmapStringRepresentation; std::vector hits; bitmap.resize((vOutPoints.size() + 7) / 8); ChainstateManager *maybe_chainman = GetChainman(context, req); if (!maybe_chainman) { return false; } ChainstateManager &chainman = *maybe_chainman; decltype(chainman.ActiveHeight()) active_height; BlockHash active_hash; { auto process_utxos = [&vOutPoints, &outs, &hits, &active_height, &active_hash, - &chainman](const CCoinsView &view, const CTxMemPool &mempool) + &chainman](const CCoinsView &view, const CTxMemPool *mempool) EXCLUSIVE_LOCKS_REQUIRED(chainman.GetMutex()) { for (const COutPoint &vOutPoint : vOutPoints) { Coin coin; - bool hit = !mempool.isSpent(vOutPoint) && + bool hit = (!mempool || !mempool->isSpent(vOutPoint)) && view.GetCoin(vOutPoint, coin); hits.push_back(hit); if (hit) { outs.emplace_back(std::move(coin)); } } active_height = chainman.ActiveHeight(); active_hash = chainman.ActiveTip()->GetBlockHash(); }; if (fCheckMemPool) { const CTxMemPool *mempool = GetMemPool(context, req); if (!mempool) { return false; } // use db+mempool as cache backend in case user likes to query // mempool LOCK2(cs_main, mempool->cs); CCoinsViewCache &viewChain = chainman.ActiveChainstate().CoinsTip(); CCoinsViewMemPool viewMempool(&viewChain, *mempool); - process_utxos(viewMempool, *mempool); + process_utxos(viewMempool, mempool); } else { // no need to lock mempool! LOCK(cs_main); - process_utxos(chainman.ActiveChainstate().CoinsTip(), CTxMemPool()); + process_utxos(chainman.ActiveChainstate().CoinsTip(), nullptr); } for (size_t i = 0; i < hits.size(); ++i) { const bool hit = hits[i]; // form a binary string representation (human-readable for json // output) bitmapStringRepresentation.append(hit ? "1" : "0"); bitmap[i / 8] |= ((uint8_t)hit) << (i % 8); } } switch (rf) { case RetFormat::BINARY: { // serialize data // use exact same output as mentioned in Bip64 CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); ssGetUTXOResponse << active_height << active_hash << bitmap << outs; std::string ssGetUTXOResponseString = ssGetUTXOResponse.str(); req->WriteHeader("Content-Type", "application/octet-stream"); req->WriteReply(HTTP_OK, ssGetUTXOResponseString); return true; } case RetFormat::HEX: { CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); ssGetUTXOResponse << active_height << active_hash << bitmap << outs; std::string strHex = HexStr(ssGetUTXOResponse) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); return true; } case RetFormat::JSON: { UniValue objGetUTXOResponse(UniValue::VOBJ); // pack in some essentials // use more or less the same output as mentioned in Bip64 objGetUTXOResponse.pushKV("chainHeight", active_height); objGetUTXOResponse.pushKV("chaintipHash", active_hash.GetHex()); objGetUTXOResponse.pushKV("bitmap", bitmapStringRepresentation); UniValue utxos(UniValue::VARR); for (const CCoin &coin : outs) { UniValue utxo(UniValue::VOBJ); utxo.pushKV("height", int32_t(coin.nHeight)); utxo.pushKV("value", coin.out.nValue); // include the script in a json output UniValue o(UniValue::VOBJ); ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); utxo.pushKV("scriptPubKey", o); utxos.push_back(utxo); } objGetUTXOResponse.pushKV("utxos", utxos); // return json string std::string strJSON = objGetUTXOResponse.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); return true; } default: { return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } } static bool rest_blockhash_by_height(Config &config, const std::any &context, HTTPRequest *req, const std::string &str_uri_part) { if (!CheckWarmup(req)) { return false; } std::string height_str; const RetFormat rf = ParseDataFormat(height_str, str_uri_part); int32_t blockheight; if (!ParseInt32(height_str, &blockheight) || blockheight < 0) { return RESTERR(req, HTTP_BAD_REQUEST, "Invalid height: " + SanitizeString(height_str)); } CBlockIndex *pblockindex = nullptr; { ChainstateManager *maybe_chainman = GetChainman(context, req); if (!maybe_chainman) { return false; } ChainstateManager &chainman = *maybe_chainman; LOCK(cs_main); const CChain &active_chain = chainman.ActiveChain(); if (blockheight > active_chain.Height()) { return RESTERR(req, HTTP_NOT_FOUND, "Block height out of range"); } pblockindex = active_chain[blockheight]; } switch (rf) { case RetFormat::BINARY: { CDataStream ss_blockhash(SER_NETWORK, PROTOCOL_VERSION); ss_blockhash << pblockindex->GetBlockHash(); req->WriteHeader("Content-Type", "application/octet-stream"); req->WriteReply(HTTP_OK, ss_blockhash.str()); return true; } case RetFormat::HEX: { req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, pblockindex->GetBlockHash().GetHex() + "\n"); return true; } case RetFormat::JSON: { req->WriteHeader("Content-Type", "application/json"); UniValue resp = UniValue(UniValue::VOBJ); resp.pushKV("blockhash", pblockindex->GetBlockHash().GetHex()); req->WriteReply(HTTP_OK, resp.write() + "\n"); return true; } default: { return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } } static const struct { const char *prefix; bool (*handler)(Config &config, const std::any &context, HTTPRequest *req, const std::string &strReq); } uri_prefixes[] = { {"/rest/tx/", rest_tx}, {"/rest/block/notxdetails/", rest_block_notxdetails}, {"/rest/block/", rest_block_extended}, {"/rest/chaininfo", rest_chaininfo}, {"/rest/mempool/info", rest_mempool_info}, {"/rest/mempool/contents", rest_mempool_contents}, {"/rest/headers/", rest_headers}, {"/rest/getutxos", rest_getutxos}, {"/rest/blockhashbyheight/", rest_blockhash_by_height}, }; void StartREST(const std::any &context) { for (const auto &up : uri_prefixes) { auto handler = [context, up](Config &config, HTTPRequest *req, const std::string &prefix) { return up.handler(config, context, req, prefix); }; RegisterHTTPHandler(up.prefix, false, handler); } } void InterruptREST() {} void StopREST() { for (const auto &up : uri_prefixes) { UnregisterHTTPHandler(up.prefix, false); } } diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 4984eabc8..ec31270d3 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -1,620 +1,620 @@ // Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include #include static std::vector> extra_txn; BOOST_FIXTURE_TEST_SUITE(blockencodings_tests, RegTestingSetup) static COutPoint InsecureRandOutPoint() { return COutPoint(TxId(InsecureRand256()), 0); } static CBlock BuildBlockTestCase() { CBlock block; CMutableTransaction tx; tx.vin.resize(1); tx.vin[0].scriptSig.resize(10); tx.vout.resize(1); tx.vout[0].nValue = 42 * SATOSHI; block.vtx.resize(3); block.vtx[0] = MakeTransactionRef(tx); block.nVersion = 42; block.hashPrevBlock = BlockHash(InsecureRand256()); block.nBits = 0x207fffff; tx.vin[0].prevout = InsecureRandOutPoint(); block.vtx[1] = MakeTransactionRef(tx); tx.vin.resize(10); for (size_t i = 0; i < tx.vin.size(); i++) { tx.vin[i].prevout = InsecureRandOutPoint(); } block.vtx[2] = MakeTransactionRef(tx); bool mutated; block.hashMerkleRoot = BlockMerkleRoot(block, &mutated); assert(!mutated); GlobalConfig config; const Consensus::Params ¶ms = config.GetChainParams().GetConsensus(); while (!CheckProofOfWork(block.GetHash(), block.nBits, params)) { ++block.nNonce; } return block; } // Number of shared use_counts we expect for a tx we haven't touched // (block + mempool + our copy from the GetSharedTx call) constexpr long SHARED_TX_OFFSET{3}; static void expectUseCount(const CTxMemPool &pool, const TxId &txid, long expectedCount) EXCLUSIVE_LOCKS_REQUIRED(pool.cs) { AssertLockHeld(pool.cs); BOOST_CHECK_EQUAL(pool.mapTx.find(txid)->GetSharedTx().use_count(), SHARED_TX_OFFSET + expectedCount); } BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) { - CTxMemPool pool; + CTxMemPool &pool = *Assert(m_node.mempool); TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[2])); const TxId block_txid2 = block.vtx[2]->GetId(); expectUseCount(pool, block_txid2, 0); // Do a simple ShortTxIDs RT { CBlockHeaderAndShortTxIDs shortIDs(block); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << shortIDs; CBlockHeaderAndShortTxIDs shortIDs2; stream >> shortIDs2; PartiallyDownloadedBlock partialBlock(GetConfig(), &pool); BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); BOOST_CHECK(partialBlock.IsTxAvailable(0)); BOOST_CHECK(!partialBlock.IsTxAvailable(1)); BOOST_CHECK(partialBlock.IsTxAvailable(2)); expectUseCount(pool, block_txid2, 1); size_t poolSize = pool.size(); pool.removeRecursive(*block.vtx[2], MemPoolRemovalReason::REPLACED); BOOST_CHECK_EQUAL(pool.size(), poolSize - 1); CBlock block2; { // No transactions. PartiallyDownloadedBlock tmp = partialBlock; BOOST_CHECK(partialBlock.FillBlock(block2, {}) == READ_STATUS_INVALID); partialBlock = tmp; } // Wrong transaction { // Current implementation doesn't check txn here, but don't require // that. PartiallyDownloadedBlock tmp = partialBlock; partialBlock.FillBlock(block2, {block.vtx[2]}); partialBlock = tmp; } bool mutated; BOOST_CHECK(block.hashMerkleRoot != BlockMerkleRoot(block2, &mutated)); CBlock block3; BOOST_CHECK(partialBlock.FillBlock(block3, {block.vtx[1]}) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block3.GetHash().ToString()); BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); BOOST_CHECK(!mutated); } } class TestHeaderAndShortIDs { // Utility to encode custom CBlockHeaderAndShortTxIDs public: CBlockHeader header; uint64_t nonce; std::vector shorttxids; std::vector prefilledtxn; explicit TestHeaderAndShortIDs(const CBlockHeaderAndShortTxIDs &orig) { CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << orig; stream >> *this; } explicit TestHeaderAndShortIDs(const CBlock &block) : TestHeaderAndShortIDs(CBlockHeaderAndShortTxIDs(block)) {} uint64_t GetShortID(const TxHash &txhash) const { CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << *this; CBlockHeaderAndShortTxIDs base; stream >> base; return base.GetShortID(txhash); } SERIALIZE_METHODS(TestHeaderAndShortIDs, obj) { READWRITE( obj.header, obj.nonce, Using>>(obj.shorttxids), Using>( obj.prefilledtxn)); } }; BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) { - CTxMemPool pool; + CTxMemPool &pool = *Assert(m_node.mempool); TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[2])); const TxId block_txid2 = block.vtx[2]->GetId(); expectUseCount(pool, block_txid2, 0); // Test with pre-forwarding tx 1, but not coinbase { TestHeaderAndShortIDs shortIDs(block); shortIDs.prefilledtxn.resize(1); shortIDs.prefilledtxn[0] = {1, block.vtx[1]}; shortIDs.shorttxids.resize(2); shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[0]->GetHash()); shortIDs.shorttxids[1] = shortIDs.GetShortID(block.vtx[2]->GetHash()); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << shortIDs; CBlockHeaderAndShortTxIDs shortIDs2; stream >> shortIDs2; PartiallyDownloadedBlock partialBlock(GetConfig(), &pool); BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); BOOST_CHECK(!partialBlock.IsTxAvailable(0)); BOOST_CHECK(partialBlock.IsTxAvailable(1)); BOOST_CHECK(partialBlock.IsTxAvailable(2)); // +1 because of partialBlock expectUseCount(pool, block_txid2, 1); CBlock block2; { // No transactions. PartiallyDownloadedBlock tmp = partialBlock; BOOST_CHECK(partialBlock.FillBlock(block2, {}) == READ_STATUS_INVALID); partialBlock = tmp; } // Wrong transaction { // Current implementation doesn't check txn here, but don't require // that. PartiallyDownloadedBlock tmp = partialBlock; partialBlock.FillBlock(block2, {block.vtx[1]}); partialBlock = tmp; } // +2 because of partialBlock and block2 expectUseCount(pool, block_txid2, 2); bool mutated; BOOST_CHECK(block.hashMerkleRoot != BlockMerkleRoot(block2, &mutated)); CBlock block3; PartiallyDownloadedBlock partialBlockCopy = partialBlock; BOOST_CHECK(partialBlock.FillBlock(block3, {block.vtx[0]}) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block3.GetHash().ToString()); BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); BOOST_CHECK(!mutated); // +3 because of partialBlock and block2 and block3 expectUseCount(pool, block_txid2, 3); block.vtx.clear(); block2.vtx.clear(); block3.vtx.clear(); // + 1 because of partialBlock; -1 because of block. expectUseCount(pool, block_txid2, 0); } // -1 because of block expectUseCount(pool, block_txid2, -1); } BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) { - CTxMemPool pool; + CTxMemPool &pool = *Assert(m_node.mempool); TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[1])); const TxId block_txid1 = block.vtx[1]->GetId(); expectUseCount(pool, block_txid1, 0); // Test with pre-forwarding coinbase + tx 2 with tx 1 in mempool { TestHeaderAndShortIDs shortIDs(block); shortIDs.prefilledtxn.resize(2); shortIDs.prefilledtxn[0] = {0, block.vtx[0]}; shortIDs.prefilledtxn[1] = {2, block.vtx[2]}; shortIDs.shorttxids.resize(1); shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[1]->GetHash()); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << shortIDs; CBlockHeaderAndShortTxIDs shortIDs2; stream >> shortIDs2; PartiallyDownloadedBlock partialBlock(GetConfig(), &pool); BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); BOOST_CHECK(partialBlock.IsTxAvailable(0)); BOOST_CHECK(partialBlock.IsTxAvailable(1)); BOOST_CHECK(partialBlock.IsTxAvailable(2)); expectUseCount(pool, block_txid1, 1); CBlock block2; PartiallyDownloadedBlock partialBlockCopy = partialBlock; BOOST_CHECK(partialBlock.FillBlock(block2, {}) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block2.GetHash().ToString()); bool mutated; BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block2, &mutated).ToString()); BOOST_CHECK(!mutated); block.vtx.clear(); block2.vtx.clear(); // + 1 because of partialBlock; -1 because of block. expectUseCount(pool, block_txid1, 0); } // -1 because of block expectUseCount(pool, block_txid1, -1); } BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) { - CTxMemPool pool; + CTxMemPool &pool = *Assert(m_node.mempool); CMutableTransaction coinbase; coinbase.vin.resize(1); coinbase.vin[0].scriptSig.resize(10); coinbase.vout.resize(1); coinbase.vout[0].nValue = 42 * SATOSHI; CBlock block; block.vtx.resize(1); block.vtx[0] = MakeTransactionRef(std::move(coinbase)); block.nVersion = 42; block.hashPrevBlock = BlockHash(InsecureRand256()); block.nBits = 0x207fffff; bool mutated; block.hashMerkleRoot = BlockMerkleRoot(block, &mutated); assert(!mutated); GlobalConfig config; const Consensus::Params ¶ms = config.GetChainParams().GetConsensus(); while (!CheckProofOfWork(block.GetHash(), block.nBits, params)) { ++block.nNonce; } // Test simple header round-trip with only coinbase { CBlockHeaderAndShortTxIDs shortIDs(block); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << shortIDs; CBlockHeaderAndShortTxIDs shortIDs2; stream >> shortIDs2; PartiallyDownloadedBlock partialBlock(GetConfig(), &pool); BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); BOOST_CHECK(partialBlock.IsTxAvailable(0)); CBlock block2; std::vector vtx_missing; BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block2.GetHash().ToString()); BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block2, &mutated).ToString()); BOOST_CHECK(!mutated); } } BOOST_AUTO_TEST_CASE(TransactionsRequestSerializationTest) { BlockTransactionsRequest req1; req1.blockhash = BlockHash(InsecureRand256()); req1.indices.resize(4); req1.indices[0] = 0; req1.indices[1] = 1; req1.indices[2] = 3; req1.indices[3] = 4; CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << req1; BlockTransactionsRequest req2; stream >> req2; BOOST_CHECK_EQUAL(req1.blockhash.ToString(), req2.blockhash.ToString()); BOOST_CHECK_EQUAL(req1.indices.size(), req2.indices.size()); BOOST_CHECK_EQUAL(req1.indices[0], req2.indices[0]); BOOST_CHECK_EQUAL(req1.indices[1], req2.indices[1]); BOOST_CHECK_EQUAL(req1.indices[2], req2.indices[2]); BOOST_CHECK_EQUAL(req1.indices[3], req2.indices[3]); } BOOST_AUTO_TEST_CASE(TransactionsRequestDeserializationMaxTest) { // Check that the highest legal index is decoded correctly BlockTransactionsRequest req0; req0.blockhash = BlockHash(InsecureRand256()); req0.indices.resize(1); using indiceType = decltype(req0.indices)::value_type; static_assert(MAX_SIZE < std::numeric_limits::max(), "The max payload size cannot fit into the indice type"); req0.indices[0] = MAX_SIZE; CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << req0; BlockTransactionsRequest req1; stream >> req1; BOOST_CHECK_EQUAL(req0.indices.size(), req1.indices.size()); BOOST_CHECK_EQUAL(req0.indices[0], req1.indices[0]); req0.indices[0] += 1; stream << req0; BlockTransactionsRequest req2; BOOST_CHECK_EXCEPTION(stream >> req2, std::ios_base::failure, HasReason("ReadCompactSize(): size too large")); } BOOST_AUTO_TEST_CASE(TransactionsRequestDeserializationOverflowTest) { // Any set of index deltas that starts with N values that sum to // (0x100000000 - N) causes the edge-case overflow that was originally not // checked for. Such a request cannot be created by serializing a real // BlockTransactionsRequest due to the overflow, so here we'll serialize // from raw deltas. This can only occur if MAX_SIZE is greater than the // maximum value for that the indice type can handle. BlockTransactionsRequest req0; req0.blockhash = BlockHash(InsecureRand256()); req0.indices.resize(3); using indiceType = decltype(req0.indices)::value_type; static_assert(std::is_same::value, "This test expects the indice type to be an uint32_t"); req0.indices[0] = 0x7000; req0.indices[1] = 0x100000000 - 0x7000 - 2; req0.indices[2] = 0; CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << req0.blockhash; WriteCompactSize(stream, req0.indices.size()); WriteCompactSize(stream, req0.indices[0]); WriteCompactSize(stream, req0.indices[1]); WriteCompactSize(stream, req0.indices[2]); BlockTransactionsRequest req1; // If MAX_SIZE is the limiting factor, the deserialization should throw. // Otherwise make sure that the overflow edge-case is under control. BOOST_CHECK_EXCEPTION(stream >> req1, std::ios_base::failure, HasReason((MAX_SIZE < req0.indices[1]) ? "ReadCompactSize(): size too large" : "differential value overflow")); } BOOST_AUTO_TEST_CASE(compactblock_overflow) { for (uint32_t firstIndex : {0u, 1u, std::numeric_limits::max()}) { TestHeaderAndShortIDs cb((CBlockHeaderAndShortTxIDs())); cb.prefilledtxn.push_back({firstIndex, MakeTransactionRef()}); cb.prefilledtxn.push_back({0u, MakeTransactionRef()}); CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); BOOST_CHECK_EXCEPTION(ss << cb, std::ios_base::failure, HasReason("differential value overflow")); } auto checkShortdTxIdsSizeException = [&](size_t compactSize, const std::string &reason) { CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); // header, nonce ss << CBlockHeader() << uint64_t(0); // shorttxids.size() WriteCompactSize(ss, compactSize); CBlockHeaderAndShortTxIDs cb; BOOST_CHECK_EXCEPTION(ss >> cb, std::ios_base::failure, HasReason(reason)); }; // Here we want to check against the max compact size, so there is no point // in building a valid compact block with MAX_SIZE + 1 shortid in it. // We just check the stream expects more data as a matter of verifying that // the overflow check did not trigger while saving test time and memory by // not constructing the large object. checkShortdTxIdsSizeException(MAX_SIZE, "CDataStream::read(): end of data"); checkShortdTxIdsSizeException(MAX_SIZE + 1, "ReadCompactSize(): size too large"); auto checkPrefilledTxnSizeException = [&](size_t compactSize, const std::string &reason) { CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); // header, nonce ss << CBlockHeader() << uint64_t(0); // shorttxids.size() WriteCompactSize(ss, 0); // prefilledtxn.size() WriteCompactSize(ss, compactSize); CBlockHeaderAndShortTxIDs cb; BOOST_CHECK_EXCEPTION(ss >> cb, std::ios_base::failure, HasReason(reason)); }; // Here we want to check against the max compact size, so there is no point // in building a valid compact block with MAX_SIZE + 1 transactions in it. // We just check the stream expects more data as a matter of verifying that // the overflow check did not trigger while saving test time and memory by // not constructing the large object. checkPrefilledTxnSizeException(MAX_SIZE, "CDataStream::read(): end of data"); checkPrefilledTxnSizeException(MAX_SIZE + 1, "ReadCompactSize(): size too large"); auto checkPrefilledTxnIndexSizeException = [&](size_t compactSize, const std::string &reason) { CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); // header, nonce ss << CBlockHeader() << uint64_t(0); // shorttxids.size() WriteCompactSize(ss, 0); // prefilledtxn.size() WriteCompactSize(ss, 1); // prefilledtxn[0].index WriteCompactSize(ss, compactSize); // prefilledtxn[0].tx ss << MakeTransactionRef(); CBlockHeaderAndShortTxIDs cb; BOOST_CHECK_EXCEPTION(ss >> cb, std::ios_base::failure, HasReason(reason)); }; // Here we want to check against the max compact size, so there is no point // in building a valid compact block with MAX_SIZE shortid in it. // We just check the stream expects more data as a matter of verifying that // the overflow check did not trigger while saving test time and memory by // not constructing the large object. checkPrefilledTxnIndexSizeException(MAX_SIZE, "non contiguous indexes"); checkPrefilledTxnIndexSizeException(MAX_SIZE + 1, "ReadCompactSize(): size too large"); // Compute the number of MAX_SIZE increment we need to cause an overflow const uint64_t overflow = uint64_t(std::numeric_limits::max()) + 1; // Due to differential encoding, a value of MAX_SIZE bumps the index by // MAX_SIZE + 1 BOOST_CHECK_GE(overflow, MAX_SIZE + 1); const uint64_t overflowIter = overflow / (MAX_SIZE + 1); // Make sure the iteration fits in an uint32_t and is <= MAX_SIZE BOOST_CHECK_LE(overflowIter, std::numeric_limits::max()); BOOST_CHECK_LE(overflowIter, MAX_SIZE); uint32_t remainder = uint32_t(overflow - ((MAX_SIZE + 1) * overflowIter)); { CDataStream ss(SER_DISK, PROTOCOL_VERSION); // header, nonce ss << CBlockHeader() << uint64_t(0); // shorttxids.size() WriteCompactSize(ss, 0); // prefilledtxn.size() WriteCompactSize(ss, overflowIter + 1); for (uint32_t i = 0; i < overflowIter; i++) { // prefilledtxn[i].index WriteCompactSize(ss, MAX_SIZE); // prefilledtxn[i].tx ss << MakeTransactionRef(); } // This is the prefilled tx causing the overflow WriteCompactSize(ss, remainder); ss << MakeTransactionRef(); CBlockHeaderAndShortTxIDs cb; BOOST_CHECK_EXCEPTION(ss >> cb, std::ios_base::failure, HasReason("differential value overflow")); } { CDataStream ss(SER_DISK, PROTOCOL_VERSION); // header, nonce ss << CBlockHeader() << uint64_t(0); // shorttxids.size() WriteCompactSize(ss, 1); // shorttxids[0] CustomUintFormatter().Ser( ss, 0u); // prefilledtxn.size() WriteCompactSize(ss, overflowIter + 1); for (uint32_t i = 0; i < overflowIter; i++) { // prefilledtxn[i].index WriteCompactSize(ss, MAX_SIZE); // prefilledtxn[i].tx ss << MakeTransactionRef(); } // This prefilled tx isn't enough to cause the overflow alone, but it // overflows due to the extra shortid. WriteCompactSize(ss, remainder - 1); ss << MakeTransactionRef(); CBlockHeaderAndShortTxIDs cb; // ss >> cp; BOOST_CHECK_EXCEPTION(ss >> cb, std::ios_base::failure, HasReason("indexes overflowed 32 bits")); } { CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); // header, nonce ss << CBlockHeader() << uint64_t(0); // shorttxids.size() WriteCompactSize(ss, 0); // prefilledtxn.size() WriteCompactSize(ss, 2); // prefilledtxn[0].index WriteCompactSize(ss, 0); // prefilledtxn[0].tx ss << MakeTransactionRef(); // prefilledtxn[1].index = 1 is differentially encoded, which means // it has an absolute index of 2. This leaves no tx at index 1. WriteCompactSize(ss, 1); // prefilledtxn[1].tx ss << MakeTransactionRef(); CBlockHeaderAndShortTxIDs cb; BOOST_CHECK_EXCEPTION(ss >> cb, std::ios_base::failure, HasReason("non contiguous indexes")); } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 1557bd96e..eda30a3ec 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -1,584 +1,581 @@ // Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include #include BOOST_FIXTURE_TEST_SUITE(mempool_tests, TestingSetup) static constexpr auto REMOVAL_REASON_DUMMY = MemPoolRemovalReason::REPLACED; BOOST_AUTO_TEST_CASE(MempoolRemoveTest) { // Test CTxMemPool::remove functionality TestMemPoolEntryHelper entry; // Parent transaction with three children, and three grand-children: CMutableTransaction txParent; txParent.vin.resize(1); txParent.vin[0].scriptSig = CScript() << OP_11; txParent.vout.resize(3); for (int i = 0; i < 3; i++) { txParent.vout[i].scriptPubKey = CScript() << OP_11 << OP_EQUAL; txParent.vout[i].nValue = 33000 * SATOSHI; } CMutableTransaction txChild[3]; for (int i = 0; i < 3; i++) { txChild[i].vin.resize(1); txChild[i].vin[0].scriptSig = CScript() << OP_11; txChild[i].vin[0].prevout = COutPoint(txParent.GetId(), i); txChild[i].vout.resize(1); txChild[i].vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; txChild[i].vout[0].nValue = 11000 * SATOSHI; } CMutableTransaction txGrandChild[3]; for (int i = 0; i < 3; i++) { txGrandChild[i].vin.resize(1); txGrandChild[i].vin[0].scriptSig = CScript() << OP_11; txGrandChild[i].vin[0].prevout = COutPoint(txChild[i].GetId(), 0); txGrandChild[i].vout.resize(1); txGrandChild[i].vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; txGrandChild[i].vout[0].nValue = 11000 * SATOSHI; } - CTxMemPool testPool; + CTxMemPool &testPool = *Assert(m_node.mempool); LOCK2(cs_main, testPool.cs); // Nothing in pool, remove should do nothing: unsigned int poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txParent), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize); // Just the parent: testPool.addUnchecked(entry.FromTx(txParent)); poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txParent), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize - 1); // Parent, children, grandchildren: testPool.addUnchecked(entry.FromTx(txParent)); for (int i = 0; i < 3; i++) { testPool.addUnchecked(entry.FromTx(txChild[i])); testPool.addUnchecked(entry.FromTx(txGrandChild[i])); } // Remove Child[0], GrandChild[0] should be removed: poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txChild[0]), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize - 2); // ... make sure grandchild and child are gone: poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txGrandChild[0]), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize); poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txChild[0]), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize); // Remove parent, all children/grandchildren should go: poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txParent), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize - 5); BOOST_CHECK_EQUAL(testPool.size(), 0UL); // Add children and grandchildren, but NOT the parent (simulate the parent // being in a block) for (int i = 0; i < 3; i++) { testPool.addUnchecked(entry.FromTx(txChild[i])); testPool.addUnchecked(entry.FromTx(txGrandChild[i])); } // Now remove the parent, as might happen if a block-re-org occurs but the // parent cannot be put into the mempool (maybe because it is non-standard): poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txParent), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize - 6); BOOST_CHECK_EQUAL(testPool.size(), 0UL); } BOOST_AUTO_TEST_CASE(MempoolClearTest) { // Test CTxMemPool::clear functionality TestMemPoolEntryHelper entry; // Create a transaction CMutableTransaction txParent; txParent.vin.resize(1); txParent.vin[0].scriptSig = CScript() << OP_11; txParent.vout.resize(3); for (int i = 0; i < 3; i++) { txParent.vout[i].scriptPubKey = CScript() << OP_11 << OP_EQUAL; txParent.vout[i].nValue = 33000 * SATOSHI; } - CTxMemPool testPool; + CTxMemPool &testPool = *Assert(m_node.mempool); LOCK2(cs_main, testPool.cs); // Nothing in pool, clear should do nothing: testPool.clear(); BOOST_CHECK_EQUAL(testPool.size(), 0UL); // Add the transaction testPool.addUnchecked(entry.FromTx(txParent)); BOOST_CHECK_EQUAL(testPool.size(), 1UL); BOOST_CHECK_EQUAL(testPool.mapTx.size(), 1UL); BOOST_CHECK_EQUAL(testPool.mapNextTx.size(), 1UL); // CTxMemPool's members should be empty after a clear testPool.clear(); BOOST_CHECK_EQUAL(testPool.size(), 0UL); BOOST_CHECK_EQUAL(testPool.mapTx.size(), 0UL); BOOST_CHECK_EQUAL(testPool.mapNextTx.size(), 0UL); } template static void CheckSort(CTxMemPool &pool, std::vector &sortedOrder, const std::string &testcase) EXCLUSIVE_LOCKS_REQUIRED(pool.cs) { BOOST_CHECK_EQUAL(pool.size(), sortedOrder.size()); typename CTxMemPool::indexed_transaction_set::index::type::iterator it = pool.mapTx.get().begin(); int count = 0; for (; it != pool.mapTx.get().end(); ++it, ++count) { BOOST_CHECK_MESSAGE(it->GetTx().GetId().ToString() == sortedOrder[count], it->GetTx().GetId().ToString() << " != " << sortedOrder[count] << " in test " << testcase << ":" << count); } } BOOST_AUTO_TEST_CASE(MempoolIndexingTest) { - CTxMemPool pool; + CTxMemPool &pool = *Assert(m_node.mempool); LOCK2(cs_main, pool.cs); TestMemPoolEntryHelper entry; /** * Remove the default nonzero sigChecks, since the below tests are * focussing on fee-based ordering and involve some artificially very tiny * 21-byte transactions without any inputs. */ entry.SigChecks(0); /* 3rd highest fee */ CMutableTransaction tx1 = CMutableTransaction(); tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; /* highest fee */ CMutableTransaction tx2 = CMutableTransaction(); tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx2.vout[0].nValue = 2 * COIN; pool.addUnchecked(entry.Fee(20000 * SATOSHI).FromTx(tx2)); /* lowest fee */ CMutableTransaction tx3 = CMutableTransaction(); tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx3.vout[0].nValue = 5 * COIN; pool.addUnchecked(entry.Fee(Amount::zero()).FromTx(tx3)); /* 2nd highest fee */ CMutableTransaction tx4 = CMutableTransaction(); tx4.vout.resize(1); tx4.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx4.vout[0].nValue = 6 * COIN; pool.addUnchecked(entry.Fee(15000 * SATOSHI).FromTx(tx4)); /* equal fee rate to tx1, but arrived later to the mempool */ CMutableTransaction tx5 = CMutableTransaction(); tx5.vout.resize(1); tx5.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx5.vout[0].nValue = 11 * COIN; pool.addUnchecked(entry.Fee(10000 * SATOSHI).Time(100).FromTx(tx1)); pool.addUnchecked(entry.Fee(10000 * SATOSHI).Time(200).FromTx(tx5)); BOOST_CHECK_EQUAL(pool.size(), 5UL); std::vector sortedOrder; sortedOrder.resize(5); sortedOrder[0] = tx2.GetId().ToString(); // 20000 sortedOrder[1] = tx4.GetId().ToString(); // 15000 sortedOrder[3] = tx5.GetId().ToString(); // 10000 sortedOrder[2] = tx1.GetId().ToString(); // 10000 sortedOrder[4] = tx3.GetId().ToString(); // 0 CheckSort(pool, sortedOrder, "MempoolIndexingTest1"); } BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) { - CTxMemPool pool; + CTxMemPool &pool = *Assert(m_node.mempool); LOCK2(cs_main, pool.cs); TestMemPoolEntryHelper entry; Amount feeIncrement = MEMPOOL_FULL_FEE_INCREMENT.GetFeePerK(); CMutableTransaction tx1 = CMutableTransaction(); tx1.vin.resize(1); tx1.vin[0].scriptSig = CScript() << OP_1; tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; pool.addUnchecked(entry.Fee(20000 * SATOSHI).FromTx(tx1)); CMutableTransaction tx2 = CMutableTransaction(); tx2.vin.resize(1); tx2.vin[0].scriptSig = CScript() << OP_2; tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_2 << OP_EQUAL; tx2.vout[0].nValue = 10 * COIN; pool.addUnchecked(entry.Fee(4000 * SATOSHI).FromTx(tx2)); // should do nothing pool.TrimToSize(pool.DynamicMemoryUsage()); BOOST_CHECK(pool.exists(tx1.GetId())); BOOST_CHECK(pool.exists(tx2.GetId())); // should remove the lower-feerate transaction pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); BOOST_CHECK(pool.exists(tx1.GetId())); BOOST_CHECK(!pool.exists(tx2.GetId())); pool.addUnchecked(entry.FromTx(tx2)); CMutableTransaction tx3 = CMutableTransaction(); tx3.vin.resize(1); tx3.vin[0].prevout = COutPoint(tx2.GetId(), 0); tx3.vin[0].scriptSig = CScript() << OP_2; tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_3 << OP_EQUAL; tx3.vout[0].nValue = 10 * COIN; pool.addUnchecked(entry.Fee(16000 * SATOSHI).FromTx(tx3)); // tx2 should be removed, tx3 is a child of tx2, so it should be removed // even though it has highest fee. pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); BOOST_CHECK(pool.exists(tx1.GetId())); BOOST_CHECK(!pool.exists(tx2.GetId())); BOOST_CHECK(!pool.exists(tx3.GetId())); // mempool is limited to tx1's size in memory usage, so nothing fits std::vector vNoSpendsRemaining; pool.TrimToSize(CTransaction(tx1).GetTotalSize(), &vNoSpendsRemaining); BOOST_CHECK(!pool.exists(tx1.GetId())); BOOST_CHECK(!pool.exists(tx2.GetId())); BOOST_CHECK(!pool.exists(tx3.GetId())); // This vector should only contain 'root' (not unconfirmed) outpoints // Though both tx2 and tx3 were removed, tx3's input came from tx2. BOOST_CHECK_EQUAL(vNoSpendsRemaining.size(), 1); BOOST_CHECK(vNoSpendsRemaining == std::vector{COutPoint()}); // maxFeeRateRemoved was set by the transaction with the highest fee, // that was not removed because it was a child of another tx. CFeeRate maxFeeRateRemoved(20000 * SATOSHI, CTransaction(tx1).GetTotalSize()); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + feeIncrement); CMutableTransaction tx4 = CMutableTransaction(); tx4.vin.resize(2); tx4.vin[0].prevout = COutPoint(); tx4.vin[0].scriptSig = CScript() << OP_4; tx4.vin[1].prevout = COutPoint(); tx4.vin[1].scriptSig = CScript() << OP_4; tx4.vout.resize(2); tx4.vout[0].scriptPubKey = CScript() << OP_4 << OP_EQUAL; tx4.vout[0].nValue = 10 * COIN; tx4.vout[1].scriptPubKey = CScript() << OP_4 << OP_EQUAL; tx4.vout[1].nValue = 10 * COIN; CMutableTransaction tx5 = CMutableTransaction(); tx5.vin.resize(2); tx5.vin[0].prevout = COutPoint(tx4.GetId(), 0); tx5.vin[0].scriptSig = CScript() << OP_4; tx5.vin[1].prevout = COutPoint(); tx5.vin[1].scriptSig = CScript() << OP_5; tx5.vout.resize(2); tx5.vout[0].scriptPubKey = CScript() << OP_5 << OP_EQUAL; tx5.vout[0].nValue = 10 * COIN; tx5.vout[1].scriptPubKey = CScript() << OP_5 << OP_EQUAL; tx5.vout[1].nValue = 10 * COIN; CMutableTransaction tx6 = CMutableTransaction(); tx6.vin.resize(2); tx6.vin[0].prevout = COutPoint(tx4.GetId(), 1); tx6.vin[0].scriptSig = CScript() << OP_4; tx6.vin[1].prevout = COutPoint(); tx6.vin[1].scriptSig = CScript() << OP_6; tx6.vout.resize(2); tx6.vout[0].scriptPubKey = CScript() << OP_6 << OP_EQUAL; tx6.vout[0].nValue = 10 * COIN; tx6.vout[1].scriptPubKey = CScript() << OP_6 << OP_EQUAL; tx6.vout[1].nValue = 10 * COIN; CMutableTransaction tx7 = CMutableTransaction(); tx7.vin.resize(2); tx7.vin[0].prevout = COutPoint(tx5.GetId(), 0); tx7.vin[0].scriptSig = CScript() << OP_5; tx7.vin[1].prevout = COutPoint(tx6.GetId(), 0); tx7.vin[1].scriptSig = CScript() << OP_6; tx7.vout.resize(2); tx7.vout[0].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[0].nValue = 10 * COIN; tx7.vout[1].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[1].nValue = 10 * COIN; pool.addUnchecked(entry.Fee(7000 * SATOSHI).FromTx(tx4)); pool.addUnchecked(entry.Fee(1000 * SATOSHI).FromTx(tx5)); pool.addUnchecked(entry.Fee(1100 * SATOSHI).FromTx(tx6)); pool.addUnchecked(entry.Fee(9000 * SATOSHI).FromTx(tx7)); // we only require this to remove, at max, 2 txn, because it's not clear // what we're really optimizing for aside from that pool.TrimToSize(pool.DynamicMemoryUsage() - 1); BOOST_CHECK(pool.exists(tx4.GetId())); BOOST_CHECK(pool.exists(tx6.GetId())); BOOST_CHECK(!pool.exists(tx7.GetId())); if (!pool.exists(tx5.GetId())) { pool.addUnchecked(entry.Fee(1000 * SATOSHI).FromTx(tx5)); } pool.addUnchecked(entry.Fee(9000 * SATOSHI).FromTx(tx7)); // should maximize mempool size by only removing 5/7 pool.TrimToSize(pool.DynamicMemoryUsage() / 2); BOOST_CHECK(pool.exists(tx4.GetId())); BOOST_CHECK(!pool.exists(tx5.GetId())); BOOST_CHECK(pool.exists(tx6.GetId())); BOOST_CHECK(!pool.exists(tx7.GetId())); pool.addUnchecked(entry.Fee(1000 * SATOSHI).FromTx(tx5)); pool.addUnchecked(entry.Fee(9000 * SATOSHI).FromTx(tx7)); std::vector vtx; SetMockTime(42); SetMockTime(42 + CTxMemPool::ROLLING_FEE_HALFLIFE); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + feeIncrement); // ... we should keep the same min fee until we get a block pool.removeForBlock(vtx); SetMockTime(42 + 2 * CTxMemPool::ROLLING_FEE_HALFLIFE); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + feeIncrement) / 2); // ... then feerate should drop 1/2 each halflife SetMockTime(42 + 2 * CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE / 2); BOOST_CHECK_EQUAL( pool.GetMinFee(pool.DynamicMemoryUsage() * 5 / 2).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + feeIncrement) / 4); // ... with a 1/2 halflife when mempool is < 1/2 its target size SetMockTime(42 + 2 * CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE / 2 + CTxMemPool::ROLLING_FEE_HALFLIFE / 4); BOOST_CHECK_EQUAL( pool.GetMinFee(pool.DynamicMemoryUsage() * 9 / 2).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + feeIncrement) / 8 + SATOSHI); // ... with a 1/4 halflife when mempool is < 1/4 its target size } // expectedSize can be smaller than correctlyOrderedIds.size(), since we // might be testing intermediary states. Just avoiding some slice operations, void CheckDisconnectPoolOrder(DisconnectedBlockTransactions &disconnectPool, std::vector correctlyOrderedIds, unsigned int expectedSize) { int i = 0; BOOST_CHECK_EQUAL(disconnectPool.GetQueuedTx().size(), expectedSize); // Txns in queuedTx's insertion_order index are sorted from children to // parent txn for (const CTransactionRef &tx : reverse_iterate(disconnectPool.GetQueuedTx().get())) { BOOST_CHECK(tx->GetId() == correctlyOrderedIds[i]); i++; } } typedef std::vector vecptx; BOOST_AUTO_TEST_CASE(TestImportMempool) { CMutableTransaction chainedTxn[5]; std::vector correctlyOrderedIds; COutPoint lastOutpoint; // Construct a chain of 5 transactions for (int i = 0; i < 5; i++) { chainedTxn[i].vin.emplace_back(lastOutpoint); chainedTxn[i].vout.emplace_back(10 * SATOSHI, CScript() << OP_TRUE); correctlyOrderedIds.push_back(chainedTxn[i].GetId()); lastOutpoint = COutPoint(correctlyOrderedIds[i], 0); } // The first 3 txns simulate once confirmed transactions that have been // disconnected. We test 3 different orders: in order, one case of mixed // order and inverted order. vecptx disconnectedTxnsInOrder = {&chainedTxn[0], &chainedTxn[1], &chainedTxn[2]}; vecptx disconnectedTxnsMixedOrder = {&chainedTxn[1], &chainedTxn[2], &chainedTxn[0]}; vecptx disconnectedTxnsInvertedOrder = {&chainedTxn[2], &chainedTxn[1], &chainedTxn[0]}; // The last 2 txns simulate a chain of unconfirmed transactions in the // mempool. We test 2 different orders: in and out of order. vecptx unconfTxnsInOrder = {&chainedTxn[3], &chainedTxn[4]}; vecptx unconfTxnsOutOfOrder = {&chainedTxn[4], &chainedTxn[3]}; // Now we test all combinations of the previously defined orders for // disconnected and unconfirmed txns. The expected outcome is to have these // transactions in the correct order in queuedTx, as defined in // correctlyOrderedIds. for (auto &disconnectedTxns : {disconnectedTxnsInOrder, disconnectedTxnsMixedOrder, disconnectedTxnsInvertedOrder}) { for (auto &unconfTxns : {unconfTxnsInOrder, unconfTxnsOutOfOrder}) { - CTxMemPool testPool; + CTxMemPool &testPool = *Assert(m_node.mempool); // addForBlock inserts disconnectTxns in disconnectPool. They // simulate transactions that were once confirmed in a block std::vector vtx; for (auto tx : disconnectedTxns) { vtx.push_back(MakeTransactionRef(*tx)); } DisconnectedBlockTransactions disconnectPool; - LOCK(testPool.cs); + LOCK2(cs_main, testPool.cs); { disconnectPool.addForBlock(vtx, testPool); CheckDisconnectPoolOrder(disconnectPool, correctlyOrderedIds, disconnectedTxns.size()); // If the mempool is empty, importMempool doesn't change // disconnectPool disconnectPool.importMempool(testPool); CheckDisconnectPoolOrder(disconnectPool, correctlyOrderedIds, disconnectedTxns.size()); - { - LOCK(cs_main); - // Add all unconfirmed transactions in testPool - for (auto tx : unconfTxns) { - TestMemPoolEntryHelper entry; - testPool.addUnchecked(entry.FromTx(*tx)); - } + // Add all unconfirmed transactions in testPool + for (auto tx : unconfTxns) { + TestMemPoolEntryHelper entry; + testPool.addUnchecked(entry.FromTx(*tx)); } // Now we test importMempool with a non empty mempool disconnectPool.importMempool(testPool); } CheckDisconnectPoolOrder(disconnectPool, correctlyOrderedIds, disconnectedTxns.size() + unconfTxns.size()); // We must clear disconnectPool to not trigger the assert in its // destructor disconnectPool.clear(); } } } inline CTransactionRef make_tx(std::vector &&output_values, std::vector &&inputs = std::vector(), std::vector &&input_indices = std::vector()) { CMutableTransaction tx = CMutableTransaction(); tx.vin.resize(inputs.size()); tx.vout.resize(output_values.size()); for (size_t i = 0; i < inputs.size(); ++i) { tx.vin[i].prevout = COutPoint(inputs[i]->GetId(), input_indices.size() > i ? input_indices[i] : 0); } for (size_t i = 0; i < output_values.size(); ++i) { tx.vout[i].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx.vout[i].nValue = output_values[i]; } return MakeTransactionRef(tx); } BOOST_AUTO_TEST_CASE(GetModifiedFeeRateTest) { CMutableTransaction tx = CMutableTransaction(); tx.vin.resize(1); // Make tx exactly 1000 bytes. const size_t dummyDataSize = 1000 - (GetSerializeSize(tx, PROTOCOL_VERSION) + 5 /* OP_PUSHDATA2 and ?? */); tx.vin[0].scriptSig << std::vector(dummyDataSize); assert(GetSerializeSize(tx, PROTOCOL_VERSION) == 1000); TestMemPoolEntryHelper entry; auto entryNormal = entry.Fee(1000 * SATOSHI).FromTx(tx); BOOST_CHECK_EQUAL(1000 * SATOSHI, entryNormal.GetModifiedFeeRate().GetFee(1000)); // Add modified fee CTxMemPoolEntry entryFeeModified = entry.Fee(1000 * SATOSHI).FromTx(tx); entryFeeModified.UpdateFeeDelta(1000 * SATOSHI); BOOST_CHECK_EQUAL(2000 * SATOSHI, entryFeeModified.GetModifiedFeeRate().GetFee(1000)); // Excessive sigop count "modifies" size CTxMemPoolEntry entrySizeModified = entry.Fee(1000 * SATOSHI) .SigChecks(2000 / DEFAULT_BYTES_PER_SIGCHECK) .FromTx(tx); BOOST_CHECK_EQUAL(500 * SATOSHI, entrySizeModified.GetModifiedFeeRate().GetFee(1000)); } BOOST_AUTO_TEST_CASE(CompareTxMemPoolEntryByModifiedFeeRateTest) { CTransactionRef a = make_tx(/* output_values */ {1 * COIN}); CTransactionRef b = make_tx(/* output_values */ {2 * COIN}); // For this test, we want b to have lower txid. if (a->GetId() < b->GetId()) { std::swap(a, b); } BOOST_CHECK_GT(a->GetId(), b->GetId()); TestMemPoolEntryHelper entry; CompareTxMemPoolEntryByModifiedFeeRate compare; auto checkOrdering = [&compare](const auto &a, const auto &b) { BOOST_CHECK(compare(a, b)); BOOST_CHECK(!compare(b, a)); }; // If the fees and entryId are the same, lower TxId should sort before checkOrdering(entry.Fee(100 * SATOSHI).FromTx(b), entry.Fee(100 * SATOSHI).FromTx(a)); // Earlier entryId, same fee, should sort before checkOrdering(entry.Fee(100 * SATOSHI).EntryId(1).FromTx(a), entry.Fee(100 * SATOSHI).EntryId(2).FromTx(b)); // Higher fee, earlier entryId should sort before checkOrdering(entry.Fee(101 * SATOSHI).EntryId(1).FromTx(a), entry.Fee(100 * SATOSHI).EntryId(2).FromTx(b)); // Higher fee, same entryId should sort before checkOrdering(entry.Fee(101 * SATOSHI).FromTx(a), entry.Fee(100 * SATOSHI).FromTx(b)); // Same with fee delta. { CTxMemPoolEntry entryA = entry.Fee(100 * SATOSHI).FromTx(a); CTxMemPoolEntry entryB = entry.Fee(200 * SATOSHI).FromTx(b); // .. A and B have same modified fee, ordering is by lowest txid entryA.UpdateFeeDelta(100 * SATOSHI); checkOrdering(entryB, entryA); } // .. A is first entering the mempool CTxMemPoolEntry entryA = entry.Fee(100 * SATOSHI).EntryId(1).FromTx(a); CTxMemPoolEntry entryB = entry.Fee(100 * SATOSHI).EntryId(2).FromTx(b); checkOrdering(entryA, entryB); // .. B has higher modified fee. entryB.UpdateFeeDelta(1 * SATOSHI); checkOrdering(entryB, entryA); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index c8fd336a6..9232805e3 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -1,98 +1,98 @@ // Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, TestingSetup) BOOST_AUTO_TEST_CASE(MempoolMinimumFeeEstimate) { - CTxMemPool mpool; + CTxMemPool &mpool = *Assert(m_node.mempool); LOCK2(cs_main, mpool.cs); TestMemPoolEntryHelper entry; // Create a transaction template CScript garbage; for (unsigned int i = 0; i < 128; i++) { garbage.push_back('X'); } CMutableTransaction tx; tx.vin.resize(1); tx.vin[0].scriptSig = garbage; tx.vout.resize(1); tx.vout[0].nValue = Amount::zero(); // Create a fake block std::vector block; int blocknum = 0; // Loop through 200 blocks adding transactions so we have a estimateFee // that is calculable. while (blocknum < 200) { for (int64_t j = 0; j < 100; j++) { // make transaction unique tx.vin[0].nSequence = 10000 * blocknum + j; TxId txid = tx.GetId(); mpool.addUnchecked( entry.Fee((j + 1) * DEFAULT_BLOCK_MIN_TX_FEE_PER_KB) .Time(GetTime()) .Height(blocknum++) .FromTx(tx)); CTransactionRef ptx = mpool.get(txid); block.push_back(ptx); } mpool.removeForBlock(block); block.clear(); } // Check that the estimate is above the rolling minimum fee. This should be // true since we have not trimmed the mempool. BOOST_CHECK(mpool.GetMinFee(1) <= mpool.estimateFee()); // Check that estimateFee returns the minimum rolling fee even when the // mempool grows very quickly and no blocks have been mined. // Add a bunch of low fee transactions which are not in the mempool // And have zero fees. CMutableTransaction mtx; tx.vin.resize(1); tx.vin[0].scriptSig = garbage; tx.vout.resize(1); block.clear(); // Add tons of transactions to the mempool, // but don't mine them. for (int64_t i = 0; i < 10000; i++) { // Mutate the hash tx.vin[0].nSequence = 10000 * blocknum + i; // Add new transaction to the mempool with a increasing fee // The average should end up as 1/2 * 100 * // DEFAULT_BLOCK_MIN_TX_FEE_PER_KB mpool.addUnchecked(entry.Fee((i + 1) * DEFAULT_BLOCK_MIN_TX_FEE_PER_KB) .Time(GetTime()) .Height(blocknum) .FromTx(tx)); } // Trim to size. GetMinFee should be more than 10000 * // DEFAULT_BLOCK_MIN_TX_FEE_PER_KB, but the estimateFee should remain // unchanged. mpool.TrimToSize(1); BOOST_CHECK(mpool.GetMinFee(1) >= CFeeRate(10000 * DEFAULT_BLOCK_MIN_TX_FEE_PER_KB, CTransaction(tx).GetTotalSize())); BOOST_CHECK_MESSAGE(mpool.estimateFee() == mpool.GetMinFee(1), "Confirm blocks has failed"); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index e66da49bc..bf32390a9 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -1,162 +1,162 @@ // Copyright (c) 2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // #include #include #include #include #include #include #include #include #include #include #include #include BOOST_FIXTURE_TEST_SUITE(validation_chainstate_tests, TestingSetup) //! Test resizing coins-related Chainstate caches during runtime. //! BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) { const Config &config = GetConfig(); ChainstateManager manager(config); WITH_LOCK(::cs_main, manager.m_blockman.m_block_tree_db = std::make_unique(1 << 20, true)); - CTxMemPool mempool; + CTxMemPool &mempool = *Assert(m_node.mempool); //! Create and add a Coin with DynamicMemoryUsage of 80 bytes to the given //! view. auto add_coin = [](CCoinsViewCache &coins_view) -> COutPoint { TxId txid{InsecureRand256()}; COutPoint outp{txid, 0}; Amount nValue = static_cast(InsecureRand32()) * SATOSHI; CScript scriptPubKey; scriptPubKey.assign((uint32_t)56, 1); Coin newcoin(CTxOut(nValue, std::move(scriptPubKey)), 1, false); coins_view.AddCoin(outp, std::move(newcoin), false); return outp; }; Chainstate &c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool)); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); // Need at least one block loaded to be able to flush caches BOOST_REQUIRE(c1.LoadGenesisBlock()); // Add a coin to the in-memory cache, upsize once, then downsize. { LOCK(::cs_main); auto outpoint = add_coin(c1.CoinsTip()); // Set a meaningless bestblock value in the coinsview cache - otherwise // we won't flush during ResizecoinsCaches() and will subsequently hit // an assertion. c1.CoinsTip().SetBestBlock(BlockHash{InsecureRand256()}); BOOST_CHECK(c1.CoinsTip().HaveCoinInCache(outpoint)); c1.ResizeCoinsCaches(/* upsizing the coinsview cache */ 1 << 24, /* downsizing the coinsdb cache */ 1 << 22); // View should still have the coin cached, since we haven't destructed // the cache on upsize. BOOST_CHECK(c1.CoinsTip().HaveCoinInCache(outpoint)); c1.ResizeCoinsCaches(/* upsizing the coinsview cache */ 1 << 22, /* downsizing the coinsdb cache */ 1 << 23); // The view cache should be empty since we had to destruct to downsize. BOOST_CHECK(!c1.CoinsTip().HaveCoinInCache(outpoint)); } } //! Test UpdateTip behavior for both active and background chainstates. //! //! When run on the background chainstate, UpdateTip should do a subset //! of what it does for the active chainstate. BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) { ChainstateManager &chainman = *Assert(m_node.chainman); BlockHash curr_tip = BlockHash{::g_best_block}; // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo // value can be found. mineBlocks(10); // After adding some blocks to the tip, best block should have changed. BOOST_CHECK(::g_best_block != curr_tip); BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(this, NoMalleation, /*reset_chainstate=*/true)); // Ensure our active chain is the snapshot chainstate. BOOST_CHECK(WITH_LOCK(::cs_main, return chainman.IsSnapshotActive())); curr_tip = BlockHash{::g_best_block}; // Mine a new block on top of the activated snapshot chainstate. // Defined in TestChain100Setup. mineBlocks(1); // After adding some blocks to the snapshot tip, best block should have // changed. BOOST_CHECK(::g_best_block != curr_tip); curr_tip = BlockHash{::g_best_block}; Chainstate *background_cs = nullptr; auto chainstates = chainman.GetAll(); BOOST_CHECK_EQUAL(chainstates.size(), 2); for (Chainstate *cs : chainman.GetAll()) { BOOST_CHECK(cs); if (cs != &chainman.ActiveChainstate()) { background_cs = cs; } } BOOST_CHECK(background_cs); // Create a block to append to the validation chain. std::vector noTxns; CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; CBlock validation_block = this->CreateBlock(noTxns, scriptPubKey, *background_cs); auto pblock = std::make_shared(validation_block); BlockValidationState state; bool newblock = false; const Config &config = GetConfig(); // TODO: much of this is inlined from ProcessNewBlock(); just reuse PNB() // once it is changed to support multiple chainstates. { LOCK(::cs_main); BlockValidationOptions options{config}; bool checked = CheckBlock( *pblock, state, config.GetChainParams().GetConsensus(), options); BOOST_CHECK(checked); bool accepted = background_cs->AcceptBlock(config, pblock, state, true, nullptr, &newblock); BOOST_CHECK(accepted); } // UpdateTip is called here bool block_added = background_cs->ActivateBestChain(config, state, pblock); // Ensure tip is as expected BOOST_CHECK_EQUAL(background_cs->m_chain.Tip()->GetBlockHash(), validation_block.GetHash()); // g_best_block should be unchanged after adding a block to the background // validation chain. BOOST_CHECK(block_added); BOOST_CHECK_EQUAL(curr_tip, ::g_best_block); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index edf3e6ede..1ab64b507 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -1,178 +1,178 @@ // Copyright (c) 2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // #include #include #include #include #include using node::BlockManager; BOOST_FIXTURE_TEST_SUITE(validation_flush_tests, ChainTestingSetup) //! Test utilities for detecting when we need to flush the coins cache based //! on estimated memory usage. //! //! @sa Chainstate::GetCoinsCacheSizeState() //! BOOST_AUTO_TEST_CASE(getcoinscachesizestate) { - CTxMemPool mempool; + CTxMemPool &mempool = *Assert(m_node.mempool); BlockManager blockman{}; Chainstate chainstate{&mempool, blockman, *Assert(m_node.chainman)}; chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, /*should_wipe*/ false); WITH_LOCK(::cs_main, chainstate.InitCoinsCache(1 << 10)); constexpr bool is_64_bit = sizeof(void *) == 8; LOCK(::cs_main); auto &view = chainstate.CoinsTip(); //! Create and add a Coin with DynamicMemoryUsage of 80 bytes to the given //! view. auto add_coin = [](CCoinsViewCache &coins_view) -> COutPoint { ; auto txid = TxId(InsecureRand256()); COutPoint outp{txid, 0}; CScript scriptPubKey; scriptPubKey.assign(static_cast(62), 1); Coin newcoin(CTxOut(500 * SATOSHI, std::move(scriptPubKey)), 1, false); coins_view.AddCoin(std::move(outp), std::move(newcoin), false); return outp; }; // The number of bytes consumed by coin's heap data, i.e. CScript // (prevector<28, uint8_t>) when assigned 56 bytes of data per above. // // See also: Coin::DynamicMemoryUsage(). constexpr unsigned int COIN_SIZE = is_64_bit ? 80 : 72; auto print_view_mem_usage = [](CCoinsViewCache &_view) { BOOST_TEST_MESSAGE( "CCoinsViewCache memory usage: " << _view.DynamicMemoryUsage()); }; constexpr size_t MAX_COINS_CACHE_BYTES = 1024; // Without any coins in the cache, we shouldn't need to flush. BOOST_CHECK_EQUAL(chainstate.GetCoinsCacheSizeState( MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::OK); // If the initial memory allocations of cacheCoins don't match these common // cases, we can't really continue to make assertions about memory usage. // End the test early. if (view.DynamicMemoryUsage() != 32 && view.DynamicMemoryUsage() != 16) { // Add a bunch of coins to see that we at least flip over to CRITICAL. for (int i{0}; i < 1000; ++i) { COutPoint res = add_coin(view); BOOST_CHECK_EQUAL(view.AccessCoin(res).DynamicMemoryUsage(), COIN_SIZE); } BOOST_CHECK_EQUAL( chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::CRITICAL); BOOST_TEST_MESSAGE( "Exiting cache flush tests early due to unsupported arch"); return; } print_view_mem_usage(view); BOOST_CHECK_EQUAL(view.DynamicMemoryUsage(), is_64_bit ? 32U : 16U); // We should be able to add COINS_UNTIL_CRITICAL coins to the cache before // going CRITICAL. This is contingent not only on the dynamic memory usage // of the Coins that we're adding (COIN_SIZE bytes per), but also on how // much memory the cacheCoins (unordered_map) preallocates. constexpr int COINS_UNTIL_CRITICAL{3}; for (int i{0}; i < COINS_UNTIL_CRITICAL; ++i) { COutPoint res = add_coin(view); print_view_mem_usage(view); BOOST_CHECK_EQUAL(view.AccessCoin(res).DynamicMemoryUsage(), COIN_SIZE); BOOST_CHECK_EQUAL( chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::OK); } // Adding some additional coins will push us over the edge to CRITICAL. for (int i{0}; i < 4; ++i) { add_coin(view); print_view_mem_usage(view); if (chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0) == CoinsCacheSizeState::CRITICAL) { break; } } BOOST_CHECK_EQUAL(chainstate.GetCoinsCacheSizeState( MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::CRITICAL); // Passing non-zero max mempool usage should allow us more headroom. BOOST_CHECK_EQUAL( chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), CoinsCacheSizeState::OK); for (int i{0}; i < 3; ++i) { add_coin(view); print_view_mem_usage(view); BOOST_CHECK_EQUAL( chainstate.GetCoinsCacheSizeState( MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), CoinsCacheSizeState::OK); } // Adding another coin with the additional mempool room will put us >90% // but not yet critical. add_coin(view); print_view_mem_usage(view); // Only perform these checks on 64 bit hosts; I haven't done the math // for 32. if (is_64_bit) { float usage_percentage = (float)view.DynamicMemoryUsage() / (MAX_COINS_CACHE_BYTES + (1 << 10)); BOOST_TEST_MESSAGE("CoinsTip usage percentage: " << usage_percentage); BOOST_CHECK(usage_percentage >= 0.9); BOOST_CHECK(usage_percentage < 1); BOOST_CHECK_EQUAL( chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, 1 << 10), CoinsCacheSizeState::LARGE); } // Using the default max_* values permits way more coins to be added. for (int i{0}; i < 1000; ++i) { add_coin(view); BOOST_CHECK_EQUAL(chainstate.GetCoinsCacheSizeState(), CoinsCacheSizeState::OK); } // Flushing the view doesn't take us back to OK because cacheCoins has // preallocated memory that doesn't get reclaimed even after flush. BOOST_CHECK_EQUAL( chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, 0), CoinsCacheSizeState::CRITICAL); view.SetBestBlock(BlockHash(InsecureRand256())); BOOST_CHECK(view.Flush()); print_view_mem_usage(view); BOOST_CHECK_EQUAL( chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, 0), CoinsCacheSizeState::CRITICAL); } BOOST_AUTO_TEST_SUITE_END()