diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 8f2d37dc4f..b5ebfde8bb 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -1,123 +1,123 @@ // Copyright (c) 2011-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "bench.h" #include "policy/policy.h" #include "txmempool.h" #include #include static void AddTx(const CTransaction &tx, const Amount &nFee, CTxMemPool &pool) { int64_t nTime = 0; double dPriority = 10.0; unsigned int nHeight = 1; bool spendsCoinbase = false; unsigned int sigOpCost = 4; LockPoints lp; pool.addUnchecked(tx.GetId(), CTxMemPoolEntry(MakeTransactionRef(tx), nFee, nTime, dPriority, nHeight, tx.GetValueOut(), spendsCoinbase, sigOpCost, lp)); } // Right now this is only testing eviction performance in an extremely small // mempool. Code needs to be written to generate a much wider variety of // unique transactions for a more meaningful performance measurement. static void MempoolEviction(benchmark::State &state) { CMutableTransaction tx1 = CMutableTransaction(); tx1.vin.resize(1); tx1.vin[0].scriptSig = CScript() << OP_1; tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; CMutableTransaction tx2 = CMutableTransaction(); tx2.vin.resize(1); tx2.vin[0].scriptSig = CScript() << OP_2; tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_2 << OP_EQUAL; tx2.vout[0].nValue = 10 * COIN; CMutableTransaction tx3 = CMutableTransaction(); tx3.vin.resize(1); tx3.vin[0].prevout = COutPoint(tx2.GetId(), 0); tx3.vin[0].scriptSig = CScript() << OP_2; tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_3 << OP_EQUAL; tx3.vout[0].nValue = 10 * COIN; CMutableTransaction tx4 = CMutableTransaction(); tx4.vin.resize(2); tx4.vin[0].prevout.SetNull(); tx4.vin[0].scriptSig = CScript() << OP_4; tx4.vin[1].prevout.SetNull(); tx4.vin[1].scriptSig = CScript() << OP_4; tx4.vout.resize(2); tx4.vout[0].scriptPubKey = CScript() << OP_4 << OP_EQUAL; tx4.vout[0].nValue = 10 * COIN; tx4.vout[1].scriptPubKey = CScript() << OP_4 << OP_EQUAL; tx4.vout[1].nValue = 10 * COIN; CMutableTransaction tx5 = CMutableTransaction(); tx5.vin.resize(2); tx5.vin[0].prevout = COutPoint(tx4.GetId(), 0); tx5.vin[0].scriptSig = CScript() << OP_4; tx5.vin[1].prevout.SetNull(); tx5.vin[1].scriptSig = CScript() << OP_5; tx5.vout.resize(2); tx5.vout[0].scriptPubKey = CScript() << OP_5 << OP_EQUAL; tx5.vout[0].nValue = 10 * COIN; tx5.vout[1].scriptPubKey = CScript() << OP_5 << OP_EQUAL; tx5.vout[1].nValue = 10 * COIN; CMutableTransaction tx6 = CMutableTransaction(); tx6.vin.resize(2); tx6.vin[0].prevout = COutPoint(tx4.GetId(), 1); tx6.vin[0].scriptSig = CScript() << OP_4; tx6.vin[1].prevout.SetNull(); tx6.vin[1].scriptSig = CScript() << OP_6; tx6.vout.resize(2); tx6.vout[0].scriptPubKey = CScript() << OP_6 << OP_EQUAL; tx6.vout[0].nValue = 10 * COIN; tx6.vout[1].scriptPubKey = CScript() << OP_6 << OP_EQUAL; tx6.vout[1].nValue = 10 * COIN; CMutableTransaction tx7 = CMutableTransaction(); tx7.vin.resize(2); tx7.vin[0].prevout = COutPoint(tx5.GetId(), 0); tx7.vin[0].scriptSig = CScript() << OP_5; tx7.vin[1].prevout = COutPoint(tx6.GetId(), 0); tx7.vin[1].scriptSig = CScript() << OP_6; tx7.vout.resize(2); tx7.vout[0].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[0].nValue = 10 * COIN; tx7.vout[1].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[1].nValue = 10 * COIN; - CTxMemPool pool(CFeeRate(Amount(1000))); + CTxMemPool pool; CTransaction t1(tx1); CTransaction t2(tx2); CTransaction t3(tx3); CTransaction t4(tx4); CTransaction t5(tx5); CTransaction t6(tx6); CTransaction t7(tx1); while (state.KeepRunning()) { AddTx(t1, Amount(10000LL), pool); AddTx(t2, Amount(5000LL), pool); AddTx(t3, Amount(20000LL), pool); AddTx(t4, Amount(7000LL), pool); AddTx(t5, Amount(1000LL), pool); AddTx(t6, Amount(1100LL), pool); AddTx(t7, Amount(9000LL), pool); pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); pool.TrimToSize(t1.GetTotalSize()); } } BENCHMARK(MempoolEviction); diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 4070a67946..0386adb7d7 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -1,578 +1,577 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "policy/fees.h" #include "policy/policy.h" #include "amount.h" #include "primitives/transaction.h" #include "random.h" #include "streams.h" #include "txmempool.h" #include "util.h" void TxConfirmStats::Initialize(std::vector &defaultBuckets, unsigned int maxConfirms, double _decay) { decay = _decay; for (size_t i = 0; i < defaultBuckets.size(); i++) { buckets.push_back(defaultBuckets[i]); bucketMap[defaultBuckets[i]] = i; } confAvg.resize(maxConfirms); curBlockConf.resize(maxConfirms); unconfTxs.resize(maxConfirms); for (unsigned int i = 0; i < maxConfirms; i++) { confAvg[i].resize(buckets.size()); curBlockConf[i].resize(buckets.size()); unconfTxs[i].resize(buckets.size()); } oldUnconfTxs.resize(buckets.size()); curBlockTxCt.resize(buckets.size()); txCtAvg.resize(buckets.size()); curBlockVal.resize(buckets.size()); avg.resize(buckets.size()); } // Zero out the data for the current block void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight) { for (size_t j = 0; j < buckets.size(); j++) { oldUnconfTxs[j] += unconfTxs[nBlockHeight % unconfTxs.size()][j]; unconfTxs[nBlockHeight % unconfTxs.size()][j] = 0; for (size_t i = 0; i < curBlockConf.size(); i++) { curBlockConf[i][j] = 0; } curBlockTxCt[j] = 0; curBlockVal[j] = 0; } } void TxConfirmStats::Record(int blocksToConfirm, double val) { // blocksToConfirm is 1-based if (blocksToConfirm < 1) { return; } unsigned int bucketindex = bucketMap.lower_bound(val)->second; for (size_t i = blocksToConfirm; i <= curBlockConf.size(); i++) { curBlockConf[i - 1][bucketindex]++; } curBlockTxCt[bucketindex]++; curBlockVal[bucketindex] += val; } void TxConfirmStats::UpdateMovingAverages() { for (unsigned int j = 0; j < buckets.size(); j++) { for (unsigned int i = 0; i < confAvg.size(); i++) { confAvg[i][j] = confAvg[i][j] * decay + curBlockConf[i][j]; } avg[j] = avg[j] * decay + curBlockVal[j]; txCtAvg[j] = txCtAvg[j] * decay + curBlockTxCt[j]; } } // returns -1 on error conditions double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, bool requireGreater, unsigned int nBlockHeight) { // Counters for a bucket (or range of buckets) // Number of tx's confirmed within the confTarget double nConf = 0; // Total number of tx's that were ever confirmed double totalNum = 0; // Number of tx's still in mempool for confTarget or longer int extraNum = 0; int maxbucketindex = buckets.size() - 1; // requireGreater means we are looking for the lowest feerate such that all // higher values pass, so we start at maxbucketindex (highest feerate) and // look at successively smaller buckets until we reach failure. Otherwise, // we are looking for the highest feerate such that all lower values fail, // and we go in the opposite direction. unsigned int startbucket = requireGreater ? maxbucketindex : 0; int step = requireGreater ? -1 : 1; // We'll combine buckets until we have enough samples. // The near and far variables will define the range we've combined // The best variables are the last range we saw which still had a high // enough confirmation rate to count as success. // The cur variables are the current range we're counting. unsigned int curNearBucket = startbucket; unsigned int bestNearBucket = startbucket; unsigned int curFarBucket = startbucket; unsigned int bestFarBucket = startbucket; bool foundAnswer = false; unsigned int bins = unconfTxs.size(); // Start counting from highest(default) or lowest feerate transactions for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) { curFarBucket = bucket; nConf += confAvg[confTarget - 1][bucket]; totalNum += txCtAvg[bucket]; for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++) { extraNum += unconfTxs[(nBlockHeight - confct) % bins][bucket]; } extraNum += oldUnconfTxs[bucket]; // If we have enough transaction data points in this range of buckets, // we can test for success (Only count the confirmed data points, so // that each confirmation count will be looking at the same amount of // data and same bucket breaks) if (totalNum >= sufficientTxVal / (1 - decay)) { double curPct = nConf / (totalNum + extraNum); // Check to see if we are no longer getting confirmed at the success // rate if (requireGreater && curPct < successBreakPoint) { break; } if (!requireGreater && curPct > successBreakPoint) { break; } // Otherwise update the cumulative stats, and the bucket variables // and reset the counters foundAnswer = true; nConf = 0; totalNum = 0; extraNum = 0; bestNearBucket = curNearBucket; bestFarBucket = curFarBucket; curNearBucket = bucket + step; } } double median = -1; double txSum = 0; // Calculate the "average" feerate of the best bucket range that met success // conditions. Find the bucket with the median transaction and then report // the average feerate from that bucket. This is a compromise between // finding the median which we can't since we don't save all tx's and // reporting the average which is less accurate unsigned int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; unsigned int maxBucket = bestNearBucket > bestFarBucket ? bestNearBucket : bestFarBucket; for (unsigned int j = minBucket; j <= maxBucket; j++) { txSum += txCtAvg[j]; } if (foundAnswer && txSum != 0) { txSum = txSum / 2; for (unsigned int j = minBucket; j <= maxBucket; j++) { if (txCtAvg[j] < txSum) { txSum -= txCtAvg[j]; } else { // we're in the right bucket median = avg[j] / txCtAvg[j]; break; } } } LogPrint(BCLog::ESTIMATEFEE, "%3d: For conf success %s %4.2f need feerate " "%s: %12.5g from buckets %8g - %8g Cur " "Bucket stats %6.2f%% %8.1f/(%.1f+%d " "mempool)\n", confTarget, requireGreater ? ">" : "<", successBreakPoint, requireGreater ? ">" : "<", median, buckets[minBucket], buckets[maxBucket], 100 * nConf / (totalNum + extraNum), nConf, totalNum, extraNum); return median; } void TxConfirmStats::Write(CAutoFile &fileout) { fileout << decay; fileout << buckets; fileout << avg; fileout << txCtAvg; fileout << confAvg; } void TxConfirmStats::Read(CAutoFile &filein) { // Read data file into temporary variables and do some very basic sanity // checking std::vector fileBuckets; std::vector fileAvg; std::vector> fileConfAvg; std::vector fileTxCtAvg; double fileDecay; size_t maxConfirms; size_t numBuckets; filein >> fileDecay; if (fileDecay <= 0 || fileDecay >= 1) { throw std::runtime_error("Corrupt estimates file. Decay must be " "between 0 and 1 (non-inclusive)"); } filein >> fileBuckets; numBuckets = fileBuckets.size(); if (numBuckets <= 1 || numBuckets > 1000) { throw std::runtime_error("Corrupt estimates file. Must have between 2 " "and 1000 feerate buckets"); } filein >> fileAvg; if (fileAvg.size() != numBuckets) { throw std::runtime_error( "Corrupt estimates file. Mismatch in feerate average bucket count"); } filein >> fileTxCtAvg; if (fileTxCtAvg.size() != numBuckets) { throw std::runtime_error( "Corrupt estimates file. Mismatch in tx count bucket count"); } filein >> fileConfAvg; maxConfirms = fileConfAvg.size(); if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) { // one week throw std::runtime_error("Corrupt estimates file. Must maintain " "estimates for between 1 and 1008 (one week) " "confirms"); } for (unsigned int i = 0; i < maxConfirms; i++) { if (fileConfAvg[i].size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in " "feerate conf average bucket count"); } } // Now that we've processed the entire feerate estimate data file and not // thrown any errors, we can copy it to our data structures decay = fileDecay; buckets = fileBuckets; avg = fileAvg; confAvg = fileConfAvg; txCtAvg = fileTxCtAvg; bucketMap.clear(); // Resize the current block variables which aren't stored in the data file // to match the number of confirms and buckets curBlockConf.resize(maxConfirms); for (unsigned int i = 0; i < maxConfirms; i++) { curBlockConf[i].resize(buckets.size()); } curBlockTxCt.resize(buckets.size()); curBlockVal.resize(buckets.size()); unconfTxs.resize(maxConfirms); for (unsigned int i = 0; i < maxConfirms; i++) { unconfTxs[i].resize(buckets.size()); } oldUnconfTxs.resize(buckets.size()); for (size_t i = 0; i < buckets.size(); i++) { bucketMap[buckets[i]] = i; } LogPrint( BCLog::ESTIMATEFEE, "Reading estimates: %u buckets counting confirms up to %u blocks\n", numBuckets, maxConfirms); } unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val) { unsigned int bucketindex = bucketMap.lower_bound(val)->second; unsigned int blockIndex = nBlockHeight % unconfTxs.size(); unconfTxs[blockIndex][bucketindex]++; return bucketindex; } void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex) { // nBestSeenHeight is not updated yet for the new block int blocksAgo = nBestSeenHeight - entryHeight; if (nBestSeenHeight == 0) { // the BlockPolicyEstimator hasn't seen any blocks yet blocksAgo = 0; } if (blocksAgo < 0) { // This can't happen because we call this with our best seen height, no // entries can have higher LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, blocks ago is negative for mempool tx\n"); return; } if (blocksAgo >= (int)unconfTxs.size()) { if (oldUnconfTxs[bucketindex] > 0) { oldUnconfTxs[bucketindex]--; } else { LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx " "removed from >25 " "blocks,bucketIndex=%u already\n", bucketindex); } } else { unsigned int blockIndex = entryHeight % unconfTxs.size(); if (unconfTxs[blockIndex][bucketindex] > 0) { unconfTxs[blockIndex][bucketindex]--; } else { LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from " "blockIndex=%u,bucketIndex=%u already\n", blockIndex, bucketindex); } } } // This function is called from CTxMemPool::removeUnchecked to ensure txs // removed from the mempool for any reason are no longer tracked. Txs that were // part of a block have already been removed in processBlockTx to ensure they // are never double tracked, but it is of no harm to try to remove them again. bool CBlockPolicyEstimator::removeTx(uint256 hash) { std::map::iterator pos = mapMemPoolTxs.find(hash); if (pos == mapMemPoolTxs.end()) { return false; } feeStats.removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex); mapMemPoolTxs.erase(hash); return true; } -CBlockPolicyEstimator::CBlockPolicyEstimator(const CFeeRate &_minRelayFee) +CBlockPolicyEstimator::CBlockPolicyEstimator() : nBestSeenHeight(0), trackedTxs(0), untrackedTxs(0) { static_assert(MIN_FEERATE > Amount(0), "Min feerate must be nonzero"); CFeeRate minFeeRate(MIN_FEERATE); - minTrackedFee = _minRelayFee < minFeeRate ? minFeeRate : _minRelayFee; std::vector vfeelist; - for (double bucketBoundary = minTrackedFee.GetFeePerK().GetSatoshis(); + for (double bucketBoundary = minFeeRate.GetFeePerK().GetSatoshis(); bucketBoundary <= double(MAX_FEERATE.GetSatoshis()); bucketBoundary *= FEE_SPACING) { vfeelist.push_back(bucketBoundary); } vfeelist.push_back(double(INF_FEERATE.GetSatoshis())); feeStats.Initialize(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY); } void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry &entry, bool validFeeEstimate) { uint32_t txHeight = entry.GetHeight(); uint256 txid = entry.GetTx().GetId(); if (mapMemPoolTxs.count(txid)) { LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error mempool tx %s already being tracked\n", txid.ToString().c_str()); return; } if (txHeight != nBestSeenHeight) { // Ignore side chains and re-orgs; assuming they are random they don't // affect the estimate. We'll potentially double count transactions in // 1-block reorgs. Ignore txs if BlockPolicyEstimator is not in sync // with chainActive.Tip(). It will be synced next time a block is // processed. return; } // Only want to be updating estimates when our blockchain is synced, // otherwise we'll miscalculate how many blocks its taking to get included. if (!validFeeEstimate) { untrackedTxs++; return; } trackedTxs++; // Feerates are stored and reported as BCH-per-kb: CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); mapMemPoolTxs[txid].blockHeight = txHeight; mapMemPoolTxs[txid].bucketIndex = feeStats.NewTx(txHeight, double(feeRate.GetFeePerK().GetSatoshis())); } bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry *entry) { if (!removeTx(entry->GetTx().GetId())) { // This transaction wasn't being tracked for fee estimation return false; } // How many blocks did it take for miners to include this transaction? // blocksToConfirm is 1-based, so a transaction included in the earliest // possible block has confirmation count of 1 int blocksToConfirm = nBlockHeight - entry->GetHeight(); if (blocksToConfirm <= 0) { // This can't happen because we don't process transactions from a block // with a height lower than our greatest seen height LogPrint( BCLog::ESTIMATEFEE, "Blockpolicy error Transaction had negative blocksToConfirm\n"); return false; } // Feerates are stored and reported as BCH-per-kb: CFeeRate feeRate(entry->GetFee(), entry->GetTxSize()); feeStats.Record(blocksToConfirm, (double)feeRate.GetFeePerK().GetSatoshis()); return true; } void CBlockPolicyEstimator::processBlock( unsigned int nBlockHeight, std::vector &entries) { if (nBlockHeight <= nBestSeenHeight) { // Ignore side chains and re-orgs; assuming they are random they don't // affect the estimate. And if an attacker can re-org the chain at will, // then you've got much bigger problems than "attacker can influence // transaction fees." return; } // Must update nBestSeenHeight in sync with ClearCurrent so that calls to // removeTx (via processBlockTx) correctly calculate age of unconfirmed txs // to remove from tracking. nBestSeenHeight = nBlockHeight; // Clear the current block state and update unconfirmed circular buffer feeStats.ClearCurrent(nBlockHeight); unsigned int countedTxs = 0; // Repopulate the current block states for (size_t i = 0; i < entries.size(); i++) { if (processBlockTx(nBlockHeight, entries[i])) { countedTxs++; } } // Update all exponential averages with the current block state feeStats.UpdateMovingAverages(); LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy after updating estimates for %u " "of %u txs in block, since last block %u of " "%u tracked, new mempool map size %u\n", countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size()); trackedTxs = 0; untrackedTxs = 0; } CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) { // Return failure if trying to analyze a target we're not tracking // It's not possible to get reasonable estimates for confTarget of 1 if (confTarget <= 1 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) { return CFeeRate(Amount(0)); } double median = feeStats.EstimateMedianVal( confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); if (median < 0) { return CFeeRate(Amount(0)); } return CFeeRate(Amount(int64_t(median))); } CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool &pool) { if (answerFoundAtTarget) { *answerFoundAtTarget = confTarget; } // Return failure if trying to analyze a target we're not tracking if (confTarget <= 0 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) { return CFeeRate(Amount(0)); } // It's not possible to get reasonable estimates for confTarget of 1 if (confTarget == 1) { confTarget = 2; } double median = -1; while (median < 0 && (unsigned int)confTarget <= feeStats.GetMaxConfirms()) { median = feeStats.EstimateMedianVal(confTarget++, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); } if (answerFoundAtTarget) { *answerFoundAtTarget = confTarget - 1; } // If mempool is limiting txs , return at least the min feerate from the // mempool Amount minPoolFee = pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000) .GetFeePerK(); if (minPoolFee > Amount(0) && minPoolFee > Amount(int64_t(median))) { return CFeeRate(minPoolFee); } if (median < 0) { return CFeeRate(Amount(0)); } return CFeeRate(Amount(int64_t(median))); } double CBlockPolicyEstimator::estimateSmartPriority(int confTarget, int *answerFoundAtTarget, const CTxMemPool &pool) { if (answerFoundAtTarget) { *answerFoundAtTarget = confTarget; } // If mempool is limiting txs, no priority txs are allowed Amount minPoolFee = pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000) .GetFeePerK(); if (minPoolFee > Amount(0)) { return double(INF_PRIORITY.GetSatoshis()); } return -1; } void CBlockPolicyEstimator::Write(CAutoFile &fileout) { fileout << nBestSeenHeight; feeStats.Write(fileout); } void CBlockPolicyEstimator::Read(CAutoFile &filein, int nFileVersion) { int nFileBestSeenHeight; filein >> nFileBestSeenHeight; feeStats.Read(filein); nBestSeenHeight = nFileBestSeenHeight; if (nFileVersion < 139900) { TxConfirmStats priStats; priStats.Read(filein); } } FeeFilterRounder::FeeFilterRounder(const CFeeRate &minIncrementalFee) { Amount minFeeLimit = std::max(Amount(1), minIncrementalFee.GetFeePerK() / 2); feeset.insert(Amount(0)); for (double bucketBoundary = minFeeLimit.GetSatoshis(); bucketBoundary <= double(MAX_FEERATE.GetSatoshis()); bucketBoundary *= FEE_SPACING) { feeset.insert(Amount(int64_t(bucketBoundary))); } } Amount FeeFilterRounder::round(const Amount currentMinFee) { auto it = feeset.lower_bound(currentMinFee); if ((it != feeset.begin() && insecure_rand.rand32() % 3 != 0) || it == feeset.end()) { it--; } return *it; } diff --git a/src/policy/fees.h b/src/policy/fees.h index 8a3b00bb37..3cb8560998 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -1,299 +1,298 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_POLICYESTIMATOR_H #define BITCOIN_POLICYESTIMATOR_H #include "amount.h" #include "random.h" #include "uint256.h" #include #include #include class CAutoFile; class CFeeRate; class CTxMemPoolEntry; class CTxMemPool; /** \class CBlockPolicyEstimator * The BlockPolicyEstimator is used for estimating the feerate needed for a * transaction to be included in a block within a certain number of blocks. * * At a high level the algorithm works by grouping transactions into buckets * based on having similar feerates and then tracking how long it takes * transactions in the various buckets to be mined. It operates under the * assumption that in general transactions of higher feerate will be included in * blocks before transactions of lower feerate. So for example if you wanted to * know what feerate you should put on a transaction to be included in a block * within the next 5 blocks, you would start by looking at the bucket with the * highest feerate transactions and verifying that a sufficiently high * percentage of them were confirmed within 5 blocks and then you would look at * the next highest feerate bucket, and so on, stopping at the last bucket to * pass the test. The average feerate of transactions in this bucket will give * you an indication of the lowest feerate you can put on a transaction and * still have a sufficiently high chance of being confirmed within your desired * 5 blocks. * * Here is a brief description of the implementation: * When a transaction enters the mempool, we track the height of the block chain * at entry. Whenever a block comes in, we count the number of transactions in * each bucket and the total amount of feerate paid in each bucket. Then we * calculate how many blocks Y it took each transaction to be mined and we track * an array of counters in each bucket for how long it to took transactions to * get confirmed from 1 to a max of 25 and we increment all the counters from Y * up to 25. This is because for any number Z>=Y the transaction was * successfully mined within Z blocks. We want to save a history of this * information, so at any time we have a counter of the total number of * transactions that happened in a given feerate bucket and the total number * that were confirmed in each number 1-25 blocks or less for any bucket. We * save this history by keeping an exponentially decaying moving average of each * one of these stats. Furthermore we also keep track of the number unmined (in * mempool) transactions in each bucket and for how many blocks they have been * outstanding and use that to increase the number of transactions we've seen in * that feerate bucket when calculating an estimate for any number of * confirmations below the number of blocks they've been outstanding. */ /** * We will instantiate an instance of this class to track transactions that were * included in a block. We will lump transactions into a bucket according to * their approximate feerate and then track how long it took for those txs to be * included in a block. * * The tracking of unconfirmed (mempool) transactions is completely independent * of the historical tracking of transactions that have been confirmed in a * block. */ class TxConfirmStats { private: // Define the buckets we will group transactions into // The upper-bound of the range for the bucket (inclusive) std::vector buckets; // Map of bucket upper-bound to index into all vectors by bucket std::map bucketMap; // For each bucket X: // Count the total # of txs in each bucket // Track the historical moving average of this total over blocks std::vector txCtAvg; // and calculate the total for the current block to update the moving // average std::vector curBlockTxCt; // Count the total # of txs confirmed within Y blocks in each bucket // Track the historical moving average of theses totals over blocks // confAvg[Y][X] std::vector> confAvg; // and calculate the totals for the current block to update the moving // averages // curBlockConf[Y][X] std::vector> curBlockConf; // Sum the total feerate of all tx's in each bucket // Track the historical moving average of this total over blocks std::vector avg; // and calculate the total for the current block to update the moving // average std::vector curBlockVal; // Combine the conf counts with tx counts to calculate the confirmation % // for each Y,X. Combine the total value with the tx counts to calculate the // avg feerate per bucket double decay; // Mempool counts of outstanding transactions // For each bucket X, track the number of transactions in the mempool that // are unconfirmed for each possible confirmation value Y // unconfTxs[Y][X] std::vector> unconfTxs; // transactions still unconfirmed after MAX_CONFIRMS for each bucket std::vector oldUnconfTxs; public: /** * Initialize the data structures. This is called by BlockPolicyEstimator's * constructor with default values. * @param defaultBuckets contains the upper limits for the bucket boundaries * @param maxConfirms max number of confirms to track * @param decay how much to decay the historical moving average per block */ void Initialize(std::vector &defaultBuckets, unsigned int maxConfirms, double decay); /** * Clear the state of the curBlock variables to start counting for the new * block. */ void ClearCurrent(unsigned int nBlockHeight); /** * Record a new transaction data point in the current block stats * @param blocksToConfirm the number of blocks it took this transaction to * confirm * @param val the feerate of the transaction * @warning blocksToConfirm is 1-based and has to be >= 1 */ void Record(int blocksToConfirm, double val); /** Record a new transaction entering the mempool*/ unsigned int NewTx(unsigned int nBlockHeight, double val); /** Remove a transaction from mempool tracking stats*/ void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketIndex); /** * Update our estimates by decaying our historical moving average and * updating with the data gathered from the current block. */ void UpdateMovingAverages(); /** * Calculate a feerate estimate. Find the lowest value bucket (or range of * buckets to make sure we have enough data points) whose transactions still * have sufficient likelihood of being confirmed within the target number of * confirmations * @param confTarget target number of confirmations * @param sufficientTxVal required average number of transactions per block * in a bucket range * @param minSuccess the success probability we require * @param requireGreater return the lowest feerate such that all higher * values pass minSuccess OR * return the highest feerate such that all lower values fail * minSuccess * @param nBlockHeight the current block height */ double EstimateMedianVal(int confTarget, double sufficientTxVal, double minSuccess, bool requireGreater, unsigned int nBlockHeight); /** Return the max number of confirms we're tracking */ unsigned int GetMaxConfirms() { return confAvg.size(); } /** Write state of estimation data to a file*/ void Write(CAutoFile &fileout); /** * Read saved state of estimation data from a file and replace all internal * data structures and variables with this state. */ void Read(CAutoFile &filein); }; /** Track confirm delays up to 25 blocks, can't estimate beyond that */ static const unsigned int MAX_BLOCK_CONFIRMS = 25; /** Decay of .998 is a half-life of 346 blocks or about 2.4 days */ static const double DEFAULT_DECAY = .998; /** Require greater than 95% of X feerate transactions to be confirmed within Y * blocks for X to be big enough */ static const double MIN_SUCCESS_PCT = .95; /** Require an avg of 1 tx in the combined feerate bucket per block to have stat * significance */ static const double SUFFICIENT_FEETXS = 1; // Minimum and Maximum values for tracking feerates static constexpr Amount MIN_FEERATE(10); static const Amount MAX_FEERATE(int64_t(1e7)); static const Amount INF_FEERATE(MAX_MONEY); static const Amount INF_PRIORITY(int64_t(1e9) * MAX_MONEY); // We have to lump transactions into buckets based on feerate, but we want to be // able to give accurate estimates over a large range of potential feerates. // Therefore it makes sense to exponentially space the buckets /** Spacing of FeeRate buckets */ static const double FEE_SPACING = 1.1; /** * We want to be able to estimate feerates that are needed on tx's to be * included in a certain number of blocks. Every time a block is added to the * best chain, this class records stats on the transactions included in that * block */ class CBlockPolicyEstimator { public: /** * Create new BlockPolicyEstimator and initialize stats tracking classes * with default values. */ - CBlockPolicyEstimator(const CFeeRate &minRelayFee); + CBlockPolicyEstimator(); /** Process all the transactions that have been included in a block */ void processBlock(unsigned int nBlockHeight, std::vector &entries); /** Process a transaction confirmed in a block*/ bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry *entry); /** Process a transaction accepted to the mempool*/ void processTransaction(const CTxMemPoolEntry &entry, bool validFeeEstimate); /** Remove a transaction from the mempool tracking stats*/ bool removeTx(uint256 hash); /** Return a feerate estimate */ CFeeRate estimateFee(int confTarget); /** Estimate feerate needed to get be included in a block within * confTarget blocks. If no answer can be given at confTarget, return an * estimate at the lowest target where one can be given. */ CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool &pool); /** * Estimate priority needed to get be included in a block within confTarget * blocks. * DEPRECATED * Returns -1 unless mempool is currently limited then returns INF_PRIORITY * answerFoundAtTarget is set to confTarget */ double estimateSmartPriority(int confTarget, int *answerFoundAtTarget, const CTxMemPool &pool); /** Write estimation data to a file */ void Write(CAutoFile &fileout); /** Read estimation data from a file */ void Read(CAutoFile &filein, int nFileVersion); private: //!< Passed to constructor to avoid dependency on main - CFeeRate minTrackedFee; unsigned int nBestSeenHeight; struct TxStatsInfo { unsigned int blockHeight; unsigned int bucketIndex; TxStatsInfo() : blockHeight(0), bucketIndex(0) {} }; // map of txids to information about that transaction std::map mapMemPoolTxs; /** Classes to track historical data on transaction confirmations */ TxConfirmStats feeStats; unsigned int trackedTxs; unsigned int untrackedTxs; }; class FeeFilterRounder { public: /** Create new FeeFilterRounder */ FeeFilterRounder(const CFeeRate &minIncrementalFee); /** Quantize a minimum fee for privacy purpose before broadcast **/ Amount round(const Amount currentMinFee); private: std::set feeset; FastRandomContext insecure_rand; }; #endif /*BITCOIN_POLICYESTIMATOR_H */ diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 973ab4ebe0..4104375728 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -1,390 +1,390 @@ // Copyright (c) 2011-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "blockencodings.h" #include "chainparams.h" #include "config.h" #include "consensus/merkle.h" #include "random.h" #include "test/test_bitcoin.h" #include std::vector> extra_txn; struct RegtestingSetup : public TestingSetup { RegtestingSetup() : TestingSetup(CBaseChainParams::REGTEST) {} }; BOOST_FIXTURE_TEST_SUITE(blockencodings_tests, RegtestingSetup) static CBlock BuildBlockTestCase() { CBlock block; CMutableTransaction tx; tx.vin.resize(1); tx.vin[0].scriptSig.resize(10); tx.vout.resize(1); tx.vout[0].nValue = Amount(42); block.vtx.resize(3); block.vtx[0] = MakeTransactionRef(tx); block.nVersion = 42; block.hashPrevBlock = InsecureRand256(); block.nBits = 0x207fffff; tx.vin[0].prevout.hash = InsecureRand256(); tx.vin[0].prevout.n = 0; block.vtx[1] = MakeTransactionRef(tx); tx.vin.resize(10); for (size_t i = 0; i < tx.vin.size(); i++) { tx.vin[i].prevout.hash = InsecureRand256(); tx.vin[i].prevout.n = 0; } block.vtx[2] = MakeTransactionRef(tx); bool mutated; block.hashMerkleRoot = BlockMerkleRoot(block, &mutated); assert(!mutated); GlobalConfig config; while (!CheckProofOfWork(block.GetHash(), block.nBits, config)) { ++block.nNonce; } return block; } // Number of shared use_counts we expect for a tx we havent touched // == 2 (mempool + our copy from the GetSharedTx call) #define SHARED_TX_OFFSET 2 BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) { - CTxMemPool pool(CFeeRate(Amount(0))); + CTxMemPool pool; TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); pool.addUnchecked(block.vtx[2]->GetId(), entry.FromTx(*block.vtx[2])); BOOST_CHECK_EQUAL( pool.mapTx.find(block.vtx[2]->GetId())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); // Do a simple ShortTxIDs RT { CBlockHeaderAndShortTxIDs shortIDs(block); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << shortIDs; CBlockHeaderAndShortTxIDs shortIDs2; stream >> shortIDs2; PartiallyDownloadedBlock partialBlock(GetConfig(), &pool); BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); BOOST_CHECK(partialBlock.IsTxAvailable(0)); BOOST_CHECK(!partialBlock.IsTxAvailable(1)); BOOST_CHECK(partialBlock.IsTxAvailable(2)); BOOST_CHECK_EQUAL( pool.mapTx.find(block.vtx[2]->GetId())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); size_t poolSize = pool.size(); pool.removeRecursive(*block.vtx[2]); BOOST_CHECK_EQUAL(pool.size(), poolSize - 1); CBlock block2; { // No transactions. PartiallyDownloadedBlock tmp = partialBlock; BOOST_CHECK(partialBlock.FillBlock(block2, {}) == READ_STATUS_INVALID); partialBlock = tmp; } // Wrong transaction { // Current implementation doesn't check txn here, but don't require // that. PartiallyDownloadedBlock tmp = partialBlock; partialBlock.FillBlock(block2, {block.vtx[2]}); partialBlock = tmp; } bool mutated; BOOST_CHECK(block.hashMerkleRoot != BlockMerkleRoot(block2, &mutated)); CBlock block3; BOOST_CHECK(partialBlock.FillBlock(block3, {block.vtx[1]}) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block3.GetHash().ToString()); BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); BOOST_CHECK(!mutated); } } class TestHeaderAndShortIDs { // Utility to encode custom CBlockHeaderAndShortTxIDs public: CBlockHeader header; uint64_t nonce; std::vector shorttxids; std::vector prefilledtxn; TestHeaderAndShortIDs(const CBlockHeaderAndShortTxIDs &orig) { CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << orig; stream >> *this; } TestHeaderAndShortIDs(const CBlock &block) : TestHeaderAndShortIDs(CBlockHeaderAndShortTxIDs(block)) {} uint64_t GetShortID(const uint256 &txhash) const { CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << *this; CBlockHeaderAndShortTxIDs base; stream >> base; return base.GetShortID(txhash); } ADD_SERIALIZE_METHODS; template inline void SerializationOp(Stream &s, Operation ser_action) { READWRITE(header); READWRITE(nonce); size_t shorttxids_size = shorttxids.size(); READWRITE(VARINT(shorttxids_size)); shorttxids.resize(shorttxids_size); for (size_t i = 0; i < shorttxids.size(); i++) { uint32_t lsb = shorttxids[i] & 0xffffffff; uint16_t msb = (shorttxids[i] >> 32) & 0xffff; READWRITE(lsb); READWRITE(msb); shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb); } READWRITE(prefilledtxn); } }; BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) { - CTxMemPool pool(CFeeRate(Amount(0))); + CTxMemPool pool; TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); pool.addUnchecked(block.vtx[2]->GetId(), entry.FromTx(*block.vtx[2])); BOOST_CHECK_EQUAL( pool.mapTx.find(block.vtx[2]->GetId())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; // Test with pre-forwarding tx 1, but not coinbase { TestHeaderAndShortIDs shortIDs(block); shortIDs.prefilledtxn.resize(1); shortIDs.prefilledtxn[0] = {1, block.vtx[1]}; shortIDs.shorttxids.resize(2); shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[0]->GetId()); shortIDs.shorttxids[1] = shortIDs.GetShortID(block.vtx[2]->GetId()); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << shortIDs; CBlockHeaderAndShortTxIDs shortIDs2; stream >> shortIDs2; PartiallyDownloadedBlock partialBlock(GetConfig(), &pool); BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); BOOST_CHECK(!partialBlock.IsTxAvailable(0)); BOOST_CHECK(partialBlock.IsTxAvailable(1)); BOOST_CHECK(partialBlock.IsTxAvailable(2)); BOOST_CHECK_EQUAL( pool.mapTx.find(block.vtx[2]->GetId())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); CBlock block2; { // No transactions. PartiallyDownloadedBlock tmp = partialBlock; BOOST_CHECK(partialBlock.FillBlock(block2, {}) == READ_STATUS_INVALID); partialBlock = tmp; } // Wrong transaction { // Current implementation doesn't check txn here, but don't require // that. PartiallyDownloadedBlock tmp = partialBlock; partialBlock.FillBlock(block2, {block.vtx[1]}); partialBlock = tmp; } bool mutated; BOOST_CHECK(block.hashMerkleRoot != BlockMerkleRoot(block2, &mutated)); CBlock block3; PartiallyDownloadedBlock partialBlockCopy = partialBlock; BOOST_CHECK(partialBlock.FillBlock(block3, {block.vtx[0]}) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block3.GetHash().ToString()); BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); BOOST_CHECK(!mutated); txhash = block.vtx[2]->GetId(); block.vtx.clear(); block2.vtx.clear(); block3.vtx.clear(); // + 1 because of partialBlockCopy. BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); } BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); } BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) { - CTxMemPool pool(CFeeRate(Amount(0))); + CTxMemPool pool; TestMemPoolEntryHelper entry; CBlock block(BuildBlockTestCase()); pool.addUnchecked(block.vtx[1]->GetId(), entry.FromTx(*block.vtx[1])); BOOST_CHECK_EQUAL( pool.mapTx.find(block.vtx[1]->GetId())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; // Test with pre-forwarding coinbase + tx 2 with tx 1 in mempool { TestHeaderAndShortIDs shortIDs(block); shortIDs.prefilledtxn.resize(2); shortIDs.prefilledtxn[0] = {0, block.vtx[0]}; // id == 1 as it is 1 after index 1 shortIDs.prefilledtxn[1] = {1, block.vtx[2]}; shortIDs.shorttxids.resize(1); shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[1]->GetId()); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << shortIDs; CBlockHeaderAndShortTxIDs shortIDs2; stream >> shortIDs2; PartiallyDownloadedBlock partialBlock(GetConfig(), &pool); BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); BOOST_CHECK(partialBlock.IsTxAvailable(0)); BOOST_CHECK(partialBlock.IsTxAvailable(1)); BOOST_CHECK(partialBlock.IsTxAvailable(2)); BOOST_CHECK_EQUAL( pool.mapTx.find(block.vtx[1]->GetId())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); CBlock block2; PartiallyDownloadedBlock partialBlockCopy = partialBlock; BOOST_CHECK(partialBlock.FillBlock(block2, {}) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block2.GetHash().ToString()); bool mutated; BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block2, &mutated).ToString()); BOOST_CHECK(!mutated); txhash = block.vtx[1]->GetId(); block.vtx.clear(); block2.vtx.clear(); // + 1 because of partialBlockCopy. BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); } BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); } BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) { - CTxMemPool pool(CFeeRate(Amount(0))); + CTxMemPool pool; CMutableTransaction coinbase; coinbase.vin.resize(1); coinbase.vin[0].scriptSig.resize(10); coinbase.vout.resize(1); coinbase.vout[0].nValue = Amount(42); CBlock block; block.vtx.resize(1); block.vtx[0] = MakeTransactionRef(std::move(coinbase)); block.nVersion = 42; block.hashPrevBlock = InsecureRand256(); block.nBits = 0x207fffff; bool mutated; block.hashMerkleRoot = BlockMerkleRoot(block, &mutated); assert(!mutated); GlobalConfig config; while (!CheckProofOfWork(block.GetHash(), block.nBits, config)) { ++block.nNonce; } // Test simple header round-trip with only coinbase { CBlockHeaderAndShortTxIDs shortIDs(block); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << shortIDs; CBlockHeaderAndShortTxIDs shortIDs2; stream >> shortIDs2; PartiallyDownloadedBlock partialBlock(GetConfig(), &pool); BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK); BOOST_CHECK(partialBlock.IsTxAvailable(0)); CBlock block2; std::vector vtx_missing; BOOST_CHECK(partialBlock.FillBlock(block2, vtx_missing) == READ_STATUS_OK); BOOST_CHECK_EQUAL(block.GetHash().ToString(), block2.GetHash().ToString()); BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block2, &mutated).ToString()); BOOST_CHECK(!mutated); } } BOOST_AUTO_TEST_CASE(TransactionsRequestSerializationTest) { BlockTransactionsRequest req1; req1.blockhash = InsecureRand256(); req1.indices.resize(4); req1.indices[0] = 0; req1.indices[1] = 1; req1.indices[2] = 3; req1.indices[3] = 4; CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << req1; BlockTransactionsRequest req2; stream >> req2; BOOST_CHECK_EQUAL(req1.blockhash.ToString(), req2.blockhash.ToString()); BOOST_CHECK_EQUAL(req1.indices.size(), req2.indices.size()); BOOST_CHECK_EQUAL(req1.indices[0], req2.indices[0]); BOOST_CHECK_EQUAL(req1.indices[1], req2.indices[1]); BOOST_CHECK_EQUAL(req1.indices[2], req2.indices[2]); BOOST_CHECK_EQUAL(req1.indices[3], req2.indices[3]); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 65fcce3ec7..46d9c28b28 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -1,668 +1,668 @@ // Copyright (c) 2011-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "policy/policy.h" #include "txmempool.h" #include "util.h" #include "test/test_bitcoin.h" #include #include #include BOOST_FIXTURE_TEST_SUITE(mempool_tests, TestingSetup) BOOST_AUTO_TEST_CASE(MempoolRemoveTest) { // Test CTxMemPool::remove functionality TestMemPoolEntryHelper entry; // Parent transaction with three children, and three grand-children: CMutableTransaction txParent; txParent.vin.resize(1); txParent.vin[0].scriptSig = CScript() << OP_11; txParent.vout.resize(3); for (int i = 0; i < 3; i++) { txParent.vout[i].scriptPubKey = CScript() << OP_11 << OP_EQUAL; txParent.vout[i].nValue = Amount(33000LL); } CMutableTransaction txChild[3]; for (int i = 0; i < 3; i++) { txChild[i].vin.resize(1); txChild[i].vin[0].scriptSig = CScript() << OP_11; txChild[i].vin[0].prevout.hash = txParent.GetId(); txChild[i].vin[0].prevout.n = i; txChild[i].vout.resize(1); txChild[i].vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; txChild[i].vout[0].nValue = Amount(11000LL); } CMutableTransaction txGrandChild[3]; for (int i = 0; i < 3; i++) { txGrandChild[i].vin.resize(1); txGrandChild[i].vin[0].scriptSig = CScript() << OP_11; txGrandChild[i].vin[0].prevout.hash = txChild[i].GetId(); txGrandChild[i].vin[0].prevout.n = 0; txGrandChild[i].vout.resize(1); txGrandChild[i].vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; txGrandChild[i].vout[0].nValue = Amount(11000LL); } - CTxMemPool testPool(CFeeRate(Amount(0))); + CTxMemPool testPool; // Nothing in pool, remove should do nothing: unsigned int poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txParent)); BOOST_CHECK_EQUAL(testPool.size(), poolSize); // Just the parent: testPool.addUnchecked(txParent.GetId(), entry.FromTx(txParent)); poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txParent)); BOOST_CHECK_EQUAL(testPool.size(), poolSize - 1); // Parent, children, grandchildren: testPool.addUnchecked(txParent.GetId(), entry.FromTx(txParent)); for (int i = 0; i < 3; i++) { testPool.addUnchecked(txChild[i].GetId(), entry.FromTx(txChild[i])); testPool.addUnchecked(txGrandChild[i].GetId(), entry.FromTx(txGrandChild[i])); } // Remove Child[0], GrandChild[0] should be removed: poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txChild[0])); BOOST_CHECK_EQUAL(testPool.size(), poolSize - 2); // ... make sure grandchild and child are gone: poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txGrandChild[0])); BOOST_CHECK_EQUAL(testPool.size(), poolSize); poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txChild[0])); BOOST_CHECK_EQUAL(testPool.size(), poolSize); // Remove parent, all children/grandchildren should go: poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txParent)); BOOST_CHECK_EQUAL(testPool.size(), poolSize - 5); BOOST_CHECK_EQUAL(testPool.size(), 0UL); // Add children and grandchildren, but NOT the parent (simulate the parent // being in a block) for (int i = 0; i < 3; i++) { testPool.addUnchecked(txChild[i].GetId(), entry.FromTx(txChild[i])); testPool.addUnchecked(txGrandChild[i].GetId(), entry.FromTx(txGrandChild[i])); } // Now remove the parent, as might happen if a block-re-org occurs but the // parent cannot be put into the mempool (maybe because it is non-standard): poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txParent)); BOOST_CHECK_EQUAL(testPool.size(), poolSize - 6); BOOST_CHECK_EQUAL(testPool.size(), 0UL); } BOOST_AUTO_TEST_CASE(MempoolClearTest) { // Test CTxMemPool::clear functionality TestMemPoolEntryHelper entry; // Create a transaction CMutableTransaction txParent; txParent.vin.resize(1); txParent.vin[0].scriptSig = CScript() << OP_11; txParent.vout.resize(3); for (int i = 0; i < 3; i++) { txParent.vout[i].scriptPubKey = CScript() << OP_11 << OP_EQUAL; txParent.vout[i].nValue = Amount(33000LL); } - CTxMemPool testPool(CFeeRate(Amount(0))); + CTxMemPool testPool; // Nothing in pool, clear should do nothing: testPool.clear(); BOOST_CHECK_EQUAL(testPool.size(), 0UL); // Add the transaction testPool.addUnchecked(txParent.GetId(), entry.FromTx(txParent)); BOOST_CHECK_EQUAL(testPool.size(), 1UL); BOOST_CHECK_EQUAL(testPool.mapTx.size(), 1UL); BOOST_CHECK_EQUAL(testPool.mapNextTx.size(), 1UL); BOOST_CHECK_EQUAL(testPool.vTxHashes.size(), 1UL); // CTxMemPool's members should be empty after a clear testPool.clear(); BOOST_CHECK_EQUAL(testPool.size(), 0UL); BOOST_CHECK_EQUAL(testPool.mapTx.size(), 0UL); BOOST_CHECK_EQUAL(testPool.mapNextTx.size(), 0UL); BOOST_CHECK_EQUAL(testPool.vTxHashes.size(), 0UL); } template void CheckSort(CTxMemPool &pool, std::vector &sortedOrder) { BOOST_CHECK_EQUAL(pool.size(), sortedOrder.size()); typename CTxMemPool::indexed_transaction_set::index::type::iterator it = pool.mapTx.get().begin(); int count = 0; for (; it != pool.mapTx.get().end(); ++it, ++count) { BOOST_CHECK_EQUAL(it->GetTx().GetId().ToString(), sortedOrder[count]); } } BOOST_AUTO_TEST_CASE(MempoolIndexingTest) { - CTxMemPool pool(CFeeRate(Amount(0))); + CTxMemPool pool; TestMemPoolEntryHelper entry; /* 3rd highest fee */ CMutableTransaction tx1 = CMutableTransaction(); tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; pool.addUnchecked(tx1.GetId(), entry.Fee(Amount(10000LL)).Priority(10.0).FromTx(tx1)); /* highest fee */ CMutableTransaction tx2 = CMutableTransaction(); tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx2.vout[0].nValue = 2 * COIN; pool.addUnchecked(tx2.GetId(), entry.Fee(Amount(20000LL)).Priority(9.0).FromTx(tx2)); /* lowest fee */ CMutableTransaction tx3 = CMutableTransaction(); tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx3.vout[0].nValue = 5 * COIN; pool.addUnchecked(tx3.GetId(), entry.Fee(Amount(0LL)).Priority(100.0).FromTx(tx3)); /* 2nd highest fee */ CMutableTransaction tx4 = CMutableTransaction(); tx4.vout.resize(1); tx4.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx4.vout[0].nValue = 6 * COIN; pool.addUnchecked(tx4.GetId(), entry.Fee(Amount(15000LL)).Priority(1.0).FromTx(tx4)); /* equal fee rate to tx1, but newer */ CMutableTransaction tx5 = CMutableTransaction(); tx5.vout.resize(1); tx5.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx5.vout[0].nValue = 11 * COIN; entry.nTime = 1; entry.dPriority = 10.0; pool.addUnchecked(tx5.GetId(), entry.Fee(Amount(10000LL)).FromTx(tx5)); BOOST_CHECK_EQUAL(pool.size(), 5UL); std::vector sortedOrder; sortedOrder.resize(5); sortedOrder[0] = tx3.GetId().ToString(); // 0 sortedOrder[1] = tx5.GetId().ToString(); // 10000 sortedOrder[2] = tx1.GetId().ToString(); // 10000 sortedOrder[3] = tx4.GetId().ToString(); // 15000 sortedOrder[4] = tx2.GetId().ToString(); // 20000 CheckSort(pool, sortedOrder); /* low fee but with high fee child */ /* tx6 -> tx7 -> tx8, tx9 -> tx10 */ CMutableTransaction tx6 = CMutableTransaction(); tx6.vout.resize(1); tx6.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx6.vout[0].nValue = 20 * COIN; pool.addUnchecked(tx6.GetId(), entry.Fee(Amount(0LL)).FromTx(tx6)); BOOST_CHECK_EQUAL(pool.size(), 6UL); // Check that at this point, tx6 is sorted low sortedOrder.insert(sortedOrder.begin(), tx6.GetId().ToString()); CheckSort(pool, sortedOrder); CTxMemPool::setEntries setAncestors; setAncestors.insert(pool.mapTx.find(tx6.GetId())); CMutableTransaction tx7 = CMutableTransaction(); tx7.vin.resize(1); tx7.vin[0].prevout = COutPoint(tx6.GetId(), 0); tx7.vin[0].scriptSig = CScript() << OP_11; tx7.vout.resize(2); tx7.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx7.vout[0].nValue = 10 * COIN; tx7.vout[1].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx7.vout[1].nValue = 1 * COIN; CTxMemPool::setEntries setAncestorsCalculated; std::string dummy; BOOST_CHECK_EQUAL( pool.CalculateMemPoolAncestors(entry.Fee(Amount(2000000LL)).FromTx(tx7), setAncestorsCalculated, 100, 1000000, 1000, 1000000, dummy), true); BOOST_CHECK(setAncestorsCalculated == setAncestors); pool.addUnchecked(tx7.GetId(), entry.FromTx(tx7), setAncestors); BOOST_CHECK_EQUAL(pool.size(), 7UL); // Now tx6 should be sorted higher (high fee child): tx7, tx6, tx2, ... sortedOrder.erase(sortedOrder.begin()); sortedOrder.push_back(tx6.GetId().ToString()); sortedOrder.push_back(tx7.GetId().ToString()); CheckSort(pool, sortedOrder); /* low fee child of tx7 */ CMutableTransaction tx8 = CMutableTransaction(); tx8.vin.resize(1); tx8.vin[0].prevout = COutPoint(tx7.GetId(), 0); tx8.vin[0].scriptSig = CScript() << OP_11; tx8.vout.resize(1); tx8.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx8.vout[0].nValue = 10 * COIN; setAncestors.insert(pool.mapTx.find(tx7.GetId())); pool.addUnchecked(tx8.GetId(), entry.Fee(Amount(0LL)).Time(2).FromTx(tx8), setAncestors); // Now tx8 should be sorted low, but tx6/tx both high sortedOrder.insert(sortedOrder.begin(), tx8.GetId().ToString()); CheckSort(pool, sortedOrder); /* low fee child of tx7 */ CMutableTransaction tx9 = CMutableTransaction(); tx9.vin.resize(1); tx9.vin[0].prevout = COutPoint(tx7.GetId(), 1); tx9.vin[0].scriptSig = CScript() << OP_11; tx9.vout.resize(1); tx9.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx9.vout[0].nValue = 1 * COIN; pool.addUnchecked(tx9.GetId(), entry.Fee(Amount(0LL)).Time(3).FromTx(tx9), setAncestors); // tx9 should be sorted low BOOST_CHECK_EQUAL(pool.size(), 9UL); sortedOrder.insert(sortedOrder.begin(), tx9.GetId().ToString()); CheckSort(pool, sortedOrder); std::vector snapshotOrder = sortedOrder; setAncestors.insert(pool.mapTx.find(tx8.GetId())); setAncestors.insert(pool.mapTx.find(tx9.GetId())); /* tx10 depends on tx8 and tx9 and has a high fee*/ CMutableTransaction tx10 = CMutableTransaction(); tx10.vin.resize(2); tx10.vin[0].prevout = COutPoint(tx8.GetId(), 0); tx10.vin[0].scriptSig = CScript() << OP_11; tx10.vin[1].prevout = COutPoint(tx9.GetId(), 0); tx10.vin[1].scriptSig = CScript() << OP_11; tx10.vout.resize(1); tx10.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx10.vout[0].nValue = 10 * COIN; setAncestorsCalculated.clear(); BOOST_CHECK_EQUAL(pool.CalculateMemPoolAncestors( entry.Fee(Amount(200000LL)).Time(4).FromTx(tx10), setAncestorsCalculated, 100, 1000000, 1000, 1000000, dummy), true); BOOST_CHECK(setAncestorsCalculated == setAncestors); pool.addUnchecked(tx10.GetId(), entry.FromTx(tx10), setAncestors); /** * tx8 and tx9 should both now be sorted higher * Final order after tx10 is added: * * tx3 = 0 (1) * tx5 = 10000 (1) * tx1 = 10000 (1) * tx4 = 15000 (1) * tx2 = 20000 (1) * tx9 = 200k (2 txs) * tx8 = 200k (2 txs) * tx10 = 200k (1 tx) * tx6 = 2.2M (5 txs) * tx7 = 2.2M (4 txs) */ // take out tx9, tx8 from the beginning sortedOrder.erase(sortedOrder.begin(), sortedOrder.begin() + 2); sortedOrder.insert(sortedOrder.begin() + 5, tx9.GetId().ToString()); sortedOrder.insert(sortedOrder.begin() + 6, tx8.GetId().ToString()); // tx10 is just before tx6 sortedOrder.insert(sortedOrder.begin() + 7, tx10.GetId().ToString()); CheckSort(pool, sortedOrder); // there should be 10 transactions in the mempool BOOST_CHECK_EQUAL(pool.size(), 10UL); // Now try removing tx10 and verify the sort order returns to normal pool.removeRecursive(pool.mapTx.find(tx10.GetId())->GetTx()); CheckSort(pool, snapshotOrder); pool.removeRecursive(pool.mapTx.find(tx9.GetId())->GetTx()); pool.removeRecursive(pool.mapTx.find(tx8.GetId())->GetTx()); /* Now check the sort on the mining score index. * Final order should be: * * tx7 (2M) * tx2 (20k) * tx4 (15000) * tx1/tx5 (10000) * tx3/6 (0) * (Ties resolved by hash) */ sortedOrder.clear(); sortedOrder.push_back(tx7.GetId().ToString()); sortedOrder.push_back(tx2.GetId().ToString()); sortedOrder.push_back(tx4.GetId().ToString()); if (tx1.GetId() < tx5.GetId()) { sortedOrder.push_back(tx5.GetId().ToString()); sortedOrder.push_back(tx1.GetId().ToString()); } else { sortedOrder.push_back(tx1.GetId().ToString()); sortedOrder.push_back(tx5.GetId().ToString()); } if (tx3.GetId() < tx6.GetId()) { sortedOrder.push_back(tx6.GetId().ToString()); sortedOrder.push_back(tx3.GetId().ToString()); } else { sortedOrder.push_back(tx3.GetId().ToString()); sortedOrder.push_back(tx6.GetId().ToString()); } CheckSort(pool, sortedOrder); } BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) { - CTxMemPool pool(CFeeRate(Amount(0))); + CTxMemPool pool; TestMemPoolEntryHelper entry; /* 3rd highest fee */ CMutableTransaction tx1 = CMutableTransaction(); tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; pool.addUnchecked(tx1.GetId(), entry.Fee(Amount(10000LL)).Priority(10.0).FromTx(tx1)); /* highest fee */ CMutableTransaction tx2 = CMutableTransaction(); tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx2.vout[0].nValue = 2 * COIN; pool.addUnchecked(tx2.GetId(), entry.Fee(Amount(20000LL)).Priority(9.0).FromTx(tx2)); uint64_t tx2Size = CTransaction(tx2).GetTotalSize(); /* lowest fee */ CMutableTransaction tx3 = CMutableTransaction(); tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx3.vout[0].nValue = 5 * COIN; pool.addUnchecked(tx3.GetId(), entry.Fee(Amount(0LL)).Priority(100.0).FromTx(tx3)); /* 2nd highest fee */ CMutableTransaction tx4 = CMutableTransaction(); tx4.vout.resize(1); tx4.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx4.vout[0].nValue = 6 * COIN; pool.addUnchecked(tx4.GetId(), entry.Fee(Amount(15000LL)).Priority(1.0).FromTx(tx4)); /* equal fee rate to tx1, but newer */ CMutableTransaction tx5 = CMutableTransaction(); tx5.vout.resize(1); tx5.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx5.vout[0].nValue = 11 * COIN; pool.addUnchecked(tx5.GetId(), entry.Fee(Amount(10000LL)).FromTx(tx5)); BOOST_CHECK_EQUAL(pool.size(), 5UL); std::vector sortedOrder; sortedOrder.resize(5); sortedOrder[0] = tx2.GetId().ToString(); // 20000 sortedOrder[1] = tx4.GetId().ToString(); // 15000 // tx1 and tx5 are both 10000 // Ties are broken by hash, not timestamp, so determine which hash comes // first. if (tx1.GetId() < tx5.GetId()) { sortedOrder[2] = tx1.GetId().ToString(); sortedOrder[3] = tx5.GetId().ToString(); } else { sortedOrder[2] = tx5.GetId().ToString(); sortedOrder[3] = tx1.GetId().ToString(); } sortedOrder[4] = tx3.GetId().ToString(); // 0 CheckSort(pool, sortedOrder); /* low fee parent with high fee child */ /* tx6 (0) -> tx7 (high) */ CMutableTransaction tx6 = CMutableTransaction(); tx6.vout.resize(1); tx6.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx6.vout[0].nValue = 20 * COIN; uint64_t tx6Size = CTransaction(tx6).GetTotalSize(); pool.addUnchecked(tx6.GetId(), entry.Fee(Amount(0LL)).FromTx(tx6)); BOOST_CHECK_EQUAL(pool.size(), 6UL); // Ties are broken by hash if (tx3.GetId() < tx6.GetId()) { sortedOrder.push_back(tx6.GetId().ToString()); } else { sortedOrder.insert(sortedOrder.end() - 1, tx6.GetId().ToString()); } CheckSort(pool, sortedOrder); CMutableTransaction tx7 = CMutableTransaction(); tx7.vin.resize(1); tx7.vin[0].prevout = COutPoint(tx6.GetId(), 0); tx7.vin[0].scriptSig = CScript() << OP_11; tx7.vout.resize(1); tx7.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx7.vout[0].nValue = 10 * COIN; uint64_t tx7Size = CTransaction(tx7).GetTotalSize(); /* set the fee to just below tx2's feerate when including ancestor */ Amount fee((20000 / tx2Size) * (tx7Size + tx6Size) - 1); // CTxMemPoolEntry entry7(tx7, fee, 2, 10.0, 1, true); pool.addUnchecked(tx7.GetId(), entry.Fee(Amount(fee)).FromTx(tx7)); BOOST_CHECK_EQUAL(pool.size(), 7UL); sortedOrder.insert(sortedOrder.begin() + 1, tx7.GetId().ToString()); CheckSort(pool, sortedOrder); /* after tx6 is mined, tx7 should move up in the sort */ std::vector vtx; vtx.push_back(MakeTransactionRef(tx6)); pool.removeForBlock(vtx, 1); sortedOrder.erase(sortedOrder.begin() + 1); // Ties are broken by hash if (tx3.GetId() < tx6.GetId()) sortedOrder.pop_back(); else sortedOrder.erase(sortedOrder.end() - 2); sortedOrder.insert(sortedOrder.begin(), tx7.GetId().ToString()); CheckSort(pool, sortedOrder); } BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) { - CTxMemPool pool(CFeeRate(Amount(1000))); + CTxMemPool pool; TestMemPoolEntryHelper entry; entry.dPriority = 10.0; CMutableTransaction tx1 = CMutableTransaction(); tx1.vin.resize(1); tx1.vin[0].scriptSig = CScript() << OP_1; tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; pool.addUnchecked(tx1.GetId(), entry.Fee(Amount(10000LL)).FromTx(tx1, &pool)); CMutableTransaction tx2 = CMutableTransaction(); tx2.vin.resize(1); tx2.vin[0].scriptSig = CScript() << OP_2; tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_2 << OP_EQUAL; tx2.vout[0].nValue = 10 * COIN; pool.addUnchecked(tx2.GetId(), entry.Fee(Amount(5000LL)).FromTx(tx2, &pool)); // should do nothing pool.TrimToSize(pool.DynamicMemoryUsage()); BOOST_CHECK(pool.exists(tx1.GetId())); BOOST_CHECK(pool.exists(tx2.GetId())); // should remove the lower-feerate transaction pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); BOOST_CHECK(pool.exists(tx1.GetId())); BOOST_CHECK(!pool.exists(tx2.GetId())); pool.addUnchecked(tx2.GetId(), entry.FromTx(tx2, &pool)); CMutableTransaction tx3 = CMutableTransaction(); tx3.vin.resize(1); tx3.vin[0].prevout = COutPoint(tx2.GetId(), 0); tx3.vin[0].scriptSig = CScript() << OP_2; tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_3 << OP_EQUAL; tx3.vout[0].nValue = 10 * COIN; pool.addUnchecked(tx3.GetId(), entry.Fee(Amount(20000LL)).FromTx(tx3, &pool)); // tx3 should pay for tx2 (CPFP) pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); BOOST_CHECK(!pool.exists(tx1.GetId())); BOOST_CHECK(pool.exists(tx2.GetId())); BOOST_CHECK(pool.exists(tx3.GetId())); // mempool is limited to tx1's size in memory usage, so nothing fits pool.TrimToSize(CTransaction(tx1).GetTotalSize()); BOOST_CHECK(!pool.exists(tx1.GetId())); BOOST_CHECK(!pool.exists(tx2.GetId())); BOOST_CHECK(!pool.exists(tx3.GetId())); CFeeRate maxFeeRateRemoved(Amount(25000), CTransaction(tx3).GetTotalSize() + CTransaction(tx2).GetTotalSize()); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + Amount(1000)); CMutableTransaction tx4 = CMutableTransaction(); tx4.vin.resize(2); tx4.vin[0].prevout.SetNull(); tx4.vin[0].scriptSig = CScript() << OP_4; tx4.vin[1].prevout.SetNull(); tx4.vin[1].scriptSig = CScript() << OP_4; tx4.vout.resize(2); tx4.vout[0].scriptPubKey = CScript() << OP_4 << OP_EQUAL; tx4.vout[0].nValue = 10 * COIN; tx4.vout[1].scriptPubKey = CScript() << OP_4 << OP_EQUAL; tx4.vout[1].nValue = 10 * COIN; CMutableTransaction tx5 = CMutableTransaction(); tx5.vin.resize(2); tx5.vin[0].prevout = COutPoint(tx4.GetId(), 0); tx5.vin[0].scriptSig = CScript() << OP_4; tx5.vin[1].prevout.SetNull(); tx5.vin[1].scriptSig = CScript() << OP_5; tx5.vout.resize(2); tx5.vout[0].scriptPubKey = CScript() << OP_5 << OP_EQUAL; tx5.vout[0].nValue = 10 * COIN; tx5.vout[1].scriptPubKey = CScript() << OP_5 << OP_EQUAL; tx5.vout[1].nValue = 10 * COIN; CMutableTransaction tx6 = CMutableTransaction(); tx6.vin.resize(2); tx6.vin[0].prevout = COutPoint(tx4.GetId(), 1); tx6.vin[0].scriptSig = CScript() << OP_4; tx6.vin[1].prevout.SetNull(); tx6.vin[1].scriptSig = CScript() << OP_6; tx6.vout.resize(2); tx6.vout[0].scriptPubKey = CScript() << OP_6 << OP_EQUAL; tx6.vout[0].nValue = 10 * COIN; tx6.vout[1].scriptPubKey = CScript() << OP_6 << OP_EQUAL; tx6.vout[1].nValue = 10 * COIN; CMutableTransaction tx7 = CMutableTransaction(); tx7.vin.resize(2); tx7.vin[0].prevout = COutPoint(tx5.GetId(), 0); tx7.vin[0].scriptSig = CScript() << OP_5; tx7.vin[1].prevout = COutPoint(tx6.GetId(), 0); tx7.vin[1].scriptSig = CScript() << OP_6; tx7.vout.resize(2); tx7.vout[0].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[0].nValue = 10 * COIN; tx7.vout[1].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[1].nValue = 10 * COIN; pool.addUnchecked(tx4.GetId(), entry.Fee(Amount(7000LL)).FromTx(tx4, &pool)); pool.addUnchecked(tx5.GetId(), entry.Fee(Amount(1000LL)).FromTx(tx5, &pool)); pool.addUnchecked(tx6.GetId(), entry.Fee(Amount(1100LL)).FromTx(tx6, &pool)); pool.addUnchecked(tx7.GetId(), entry.Fee(Amount(9000LL)).FromTx(tx7, &pool)); // we only require this remove, at max, 2 txn, because its not clear what // we're really optimizing for aside from that pool.TrimToSize(pool.DynamicMemoryUsage() - 1); BOOST_CHECK(pool.exists(tx4.GetId())); BOOST_CHECK(pool.exists(tx6.GetId())); BOOST_CHECK(!pool.exists(tx7.GetId())); if (!pool.exists(tx5.GetId())) pool.addUnchecked(tx5.GetId(), entry.Fee(Amount(1000LL)).FromTx(tx5, &pool)); pool.addUnchecked(tx7.GetId(), entry.Fee(Amount(9000LL)).FromTx(tx7, &pool)); // should maximize mempool size by only removing 5/7 pool.TrimToSize(pool.DynamicMemoryUsage() / 2); BOOST_CHECK(pool.exists(tx4.GetId())); BOOST_CHECK(!pool.exists(tx5.GetId())); BOOST_CHECK(pool.exists(tx6.GetId())); BOOST_CHECK(!pool.exists(tx7.GetId())); pool.addUnchecked(tx5.GetId(), entry.Fee(Amount(1000LL)).FromTx(tx5, &pool)); pool.addUnchecked(tx7.GetId(), entry.Fee(Amount(9000LL)).FromTx(tx7, &pool)); std::vector vtx; SetMockTime(42); SetMockTime(42 + CTxMemPool::ROLLING_FEE_HALFLIFE); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + Amount(1000)); // ... we should keep the same min fee until we get a block pool.removeForBlock(vtx, 1); SetMockTime(42 + 2 * CTxMemPool::ROLLING_FEE_HALFLIFE); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + Amount(1000)) / 2); // ... then feerate should drop 1/2 each halflife SetMockTime(42 + 2 * CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE / 2); BOOST_CHECK_EQUAL( pool.GetMinFee(pool.DynamicMemoryUsage() * 5 / 2).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + Amount(1000)) / 4); // ... with a 1/2 halflife when mempool is < 1/2 its target size SetMockTime(42 + 2 * CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE / 2 + CTxMemPool::ROLLING_FEE_HALFLIFE / 4); BOOST_CHECK_EQUAL( pool.GetMinFee(pool.DynamicMemoryUsage() * 9 / 2).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + Amount(1000)) / 8); // ... with a 1/4 halflife when mempool is < 1/4 its target size SetMockTime(42 + 7 * CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE / 2 + CTxMemPool::ROLLING_FEE_HALFLIFE / 4); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), Amount(1000)); // ... but feerate should never drop below 1000 SetMockTime(42 + 8 * CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE / 2 + CTxMemPool::ROLLING_FEE_HALFLIFE / 4); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), Amount(0)); // ... unless it has gone all the way to 0 (after getting past 1000/2) SetMockTime(0); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 970429e1dd..66b077ff53 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -1,245 +1,245 @@ // Copyright (c) 2011-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "policy/fees.h" #include "policy/policy.h" #include "txmempool.h" #include "uint256.h" #include "util.h" #include "test/test_bitcoin.h" #include BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) { - CTxMemPool mpool(CFeeRate(Amount(1000))); + CTxMemPool mpool; TestMemPoolEntryHelper entry; Amount basefee(2000); Amount deltaFee(100); std::vector feeV; // Populate vectors of increasing fees for (int j = 0; j < 10; j++) { feeV.push_back((j + 1) * basefee); } // Store the hashes of transactions that have been added to the mempool by // their associate fee txHashes[j] is populated with transactions either of // fee = basefee * (j+1) std::vector txHashes[10]; // Create a transaction template CScript garbage; for (unsigned int i = 0; i < 128; i++) garbage.push_back('X'); CMutableTransaction tx; tx.vin.resize(1); tx.vin[0].scriptSig = garbage; tx.vout.resize(1); tx.vout[0].nValue = Amount(0); CFeeRate baseRate(basefee, CTransaction(tx).GetTotalSize()); // Create a fake block std::vector block; int blocknum = 0; // Loop through 200 blocks // At a decay .998 and 4 fee transactions per block // This makes the tx count about 1.33 per bucket, above the 1 threshold while (blocknum < 200) { // For each fee for (int j = 0; j < 10; j++) { // add 4 fee txs for (int k = 0; k < 4; k++) { // make transaction unique tx.vin[0].prevout.n = 10000 * blocknum + 100 * j + k; uint256 hash = tx.GetId(); mpool.addUnchecked(hash, entry.Fee(feeV[j]) .Time(GetTime()) .Priority(0) .Height(blocknum) .FromTx(tx, &mpool)); txHashes[j].push_back(hash); } } // Create blocks where higher fee txs are included more often for (int h = 0; h <= blocknum % 10; h++) { // 10/10 blocks add highest fee transactions // 9/10 blocks add 2nd highest and so on until ... // 1/10 blocks add lowest fee transactions while (txHashes[9 - h].size()) { CTransactionRef ptx = mpool.get(txHashes[9 - h].back()); if (ptx) block.push_back(ptx); txHashes[9 - h].pop_back(); } } mpool.removeForBlock(block, ++blocknum); block.clear(); if (blocknum == 30) { // At this point we should need to combine 5 buckets to get enough // data points. So estimateFee(1,2,3) should fail and estimateFee(4) // should return somewhere around 8*baserate. estimateFee(4) %'s // are 100,100,100,100,90 = average 98% BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(Amount(0))); BOOST_CHECK(mpool.estimateFee(2) == CFeeRate(Amount(0))); BOOST_CHECK(mpool.estimateFee(3) == CFeeRate(Amount(0))); BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() < 8 * baseRate.GetFeePerK() + deltaFee); BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() > 8 * baseRate.GetFeePerK() - deltaFee); int answerFound; BOOST_CHECK(mpool.estimateSmartFee(1, &answerFound) == mpool.estimateFee(4) && answerFound == 4); BOOST_CHECK(mpool.estimateSmartFee(3, &answerFound) == mpool.estimateFee(4) && answerFound == 4); BOOST_CHECK(mpool.estimateSmartFee(4, &answerFound) == mpool.estimateFee(4) && answerFound == 4); BOOST_CHECK(mpool.estimateSmartFee(8, &answerFound) == mpool.estimateFee(8) && answerFound == 8); } } std::vector origFeeEst; // Highest feerate is 10*baseRate and gets in all blocks, second highest // feerate is 9*baseRate and gets in 9/10 blocks = 90%, third highest // feerate is 8*base rate, and gets in 8/10 blocks = 80%, so estimateFee(1) // would return 10*baseRate but is hardcoded to return failure. Second // highest feerate has 100% chance of being included by 2 blocks, so // estimateFee(2) should return 9*baseRate etc... for (int i = 1; i < 10; i++) { origFeeEst.push_back(mpool.estimateFee(i).GetFeePerK()); // Fee estimates should be monotonically decreasing if (i > 2) { BOOST_CHECK(origFeeEst[i - 1] <= origFeeEst[i - 2]); } int mult = 11 - i; if (i > 1) { BOOST_CHECK(origFeeEst[i - 1] < mult * baseRate.GetFeePerK() + deltaFee); BOOST_CHECK(origFeeEst[i - 1] > mult * baseRate.GetFeePerK() - deltaFee); } else { BOOST_CHECK(origFeeEst[i - 1] == CFeeRate(Amount(0)).GetFeePerK()); } } // Mine 50 more blocks with no transactions happening, estimates shouldn't // change. We haven't decayed the moving average enough so we still have // enough data points in every bucket while (blocknum < 250) mpool.removeForBlock(block, ++blocknum); BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(Amount(0))); for (int i = 2; i < 10; i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i - 1] + deltaFee); BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i - 1] - deltaFee); } // Mine 15 more blocks with lots of transactions happening and not getting // mined. Estimates should go up while (blocknum < 265) { // For each fee multiple for (int j = 0; j < 10; j++) { // add 4 fee txs for (int k = 0; k < 4; k++) { tx.vin[0].prevout.n = 10000 * blocknum + 100 * j + k; uint256 txid = tx.GetId(); mpool.addUnchecked(txid, entry.Fee(feeV[j]) .Time(GetTime()) .Priority(0) .Height(blocknum) .FromTx(tx, &mpool)); txHashes[j].push_back(txid); } } mpool.removeForBlock(block, ++blocknum); } int answerFound; for (int i = 1; i < 10; i++) { BOOST_CHECK(mpool.estimateFee(i) == CFeeRate(Amount(0)) || mpool.estimateFee(i).GetFeePerK() > origFeeEst[i - 1] - deltaFee); Amount a1 = mpool.estimateSmartFee(i, &answerFound).GetFeePerK(); Amount a2 = origFeeEst[answerFound - 1] - deltaFee; BOOST_CHECK(a1 > a2); } // Mine all those transactions // Estimates should still not be below original for (int j = 0; j < 10; j++) { while (txHashes[j].size()) { CTransactionRef ptx = mpool.get(txHashes[j].back()); if (ptx) block.push_back(ptx); txHashes[j].pop_back(); } } mpool.removeForBlock(block, 265); block.clear(); BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(Amount(0))); for (int i = 2; i < 10; i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i - 1] - deltaFee); } // Mine 200 more blocks where everything is mined every block // Estimates should be below original estimates while (blocknum < 465) { // For each fee multiple for (int j = 0; j < 10; j++) { // add 4 fee txs for (int k = 0; k < 4; k++) { tx.vin[0].prevout.n = 10000 * blocknum + 100 * j + k; uint256 txid = tx.GetId(); mpool.addUnchecked(txid, entry.Fee(feeV[j]) .Time(GetTime()) .Priority(0) .Height(blocknum) .FromTx(tx, &mpool)); CTransactionRef ptx = mpool.get(txid); if (ptx) block.push_back(ptx); } } mpool.removeForBlock(block, ++blocknum); block.clear(); } BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(Amount(0))); for (int i = 2; i < 10; i++) { BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i - 1] - deltaFee); } // Test that if the mempool is limited, estimateSmartFee won't return a // value below the mempool min fee and that estimateSmartPriority returns // essentially an infinite value mpool.addUnchecked( tx.GetId(), entry.Fee(feeV[5]).Time(GetTime()).Priority(0).Height(blocknum).FromTx( tx, &mpool)); // evict that transaction which should set a mempool min fee of // minRelayTxFee + feeV[5] mpool.TrimToSize(1); BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[5]); for (int i = 1; i < 10; i++) { BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.estimateFee(i).GetFeePerK()); BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); BOOST_CHECK(mpool.estimateSmartPriority(i) == double(INF_PRIORITY.GetSatoshis())); } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 2eb236ba4c..1110c51f37 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1,1266 +1,1265 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "txmempool.h" #include "chainparams.h" // for GetConsensus. #include "clientversion.h" #include "consensus/consensus.h" #include "consensus/validation.h" #include "policy/fees.h" #include "policy/policy.h" #include "streams.h" #include "timedata.h" #include "util.h" #include "utilmoneystr.h" #include "utiltime.h" #include "validation.h" #include "version.h" #include CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef &_tx, const Amount _nFee, int64_t _nTime, double _entryPriority, unsigned int _entryHeight, Amount _inChainInputValue, bool _spendsCoinbase, int64_t _sigOpsCount, LockPoints lp) : tx(_tx), nFee(_nFee), nTime(_nTime), entryPriority(_entryPriority), entryHeight(_entryHeight), inChainInputValue(_inChainInputValue), spendsCoinbase(_spendsCoinbase), sigOpCount(_sigOpsCount), lockPoints(lp) { nTxSize = tx->GetTotalSize(); nModSize = tx->CalculateModifiedSize(GetTxSize()); nUsageSize = RecursiveDynamicUsage(tx); nCountWithDescendants = 1; nSizeWithDescendants = GetTxSize(); nModFeesWithDescendants = nFee; Amount nValueIn = tx->GetValueOut() + nFee; assert(inChainInputValue <= nValueIn); feeDelta = Amount(0); nCountWithAncestors = 1; nSizeWithAncestors = GetTxSize(); nModFeesWithAncestors = nFee; nSigOpCountWithAncestors = sigOpCount; } CTxMemPoolEntry::CTxMemPoolEntry(const CTxMemPoolEntry &other) { *this = other; } double CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const { double deltaPriority = double((currentHeight - entryHeight) * inChainInputValue.GetSatoshis()) / nModSize; double dResult = entryPriority + deltaPriority; // This should only happen if it was called with a height below entry height if (dResult < 0) { dResult = 0; } return dResult; } void CTxMemPoolEntry::UpdateFeeDelta(Amount newFeeDelta) { nModFeesWithDescendants += newFeeDelta - feeDelta; nModFeesWithAncestors += newFeeDelta - feeDelta; feeDelta = newFeeDelta; } void CTxMemPoolEntry::UpdateLockPoints(const LockPoints &lp) { lockPoints = lp; } // Update the given tx for any in-mempool descendants. // Assumes that setMemPoolChildren is correct for the given tx and all // descendants. void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap &cachedDescendants, const std::set &setExclude) { setEntries stageEntries, setAllDescendants; stageEntries = GetMemPoolChildren(updateIt); while (!stageEntries.empty()) { const txiter cit = *stageEntries.begin(); setAllDescendants.insert(cit); stageEntries.erase(cit); const setEntries &setChildren = GetMemPoolChildren(cit); for (const txiter childEntry : setChildren) { cacheMap::iterator cacheIt = cachedDescendants.find(childEntry); if (cacheIt != cachedDescendants.end()) { // We've already calculated this one, just add the entries for // this set but don't traverse again. for (const txiter cacheEntry : cacheIt->second) { setAllDescendants.insert(cacheEntry); } } else if (!setAllDescendants.count(childEntry)) { // Schedule for later processing stageEntries.insert(childEntry); } } } // setAllDescendants now contains all in-mempool descendants of updateIt. // Update and add to cached descendant map int64_t modifySize = 0; Amount modifyFee(0); int64_t modifyCount = 0; for (txiter cit : setAllDescendants) { if (!setExclude.count(cit->GetTx().GetId())) { modifySize += cit->GetTxSize(); modifyFee += cit->GetModifiedFee(); modifyCount++; cachedDescendants[updateIt].insert(cit); // Update ancestor state for each descendant mapTx.modify(cit, update_ancestor_state(updateIt->GetTxSize(), updateIt->GetModifiedFee(), 1, updateIt->GetSigOpCount())); } } mapTx.modify(updateIt, update_descendant_state(modifySize, modifyFee, modifyCount)); } // vHashesToUpdate is the set of transaction hashes from a disconnected block // which has been re-added to the mempool. For each entry, look for descendants // that are outside hashesToUpdate, and add fee/size information for such // descendants to the parent. For each such descendant, also update the ancestor // state to include the parent. void CTxMemPool::UpdateTransactionsFromBlock( const std::vector &vHashesToUpdate) { LOCK(cs); // For each entry in vHashesToUpdate, store the set of in-mempool, but not // in-vHashesToUpdate transactions, so that we don't have to recalculate // descendants when we come across a previously seen entry. cacheMap mapMemPoolDescendantsToUpdate; // Use a set for lookups into vHashesToUpdate (these entries are already // accounted for in the state of their ancestors) std::set setAlreadyIncluded(vHashesToUpdate.begin(), vHashesToUpdate.end()); // Iterate in reverse, so that whenever we are looking at at a transaction // we are sure that all in-mempool descendants have already been processed. // This maximizes the benefit of the descendant cache and guarantees that // setMemPoolChildren will be updated, an assumption made in // UpdateForDescendants. for (const uint256 &hash : boost::adaptors::reverse(vHashesToUpdate)) { // we cache the in-mempool children to avoid duplicate updates setEntries setChildren; // calculate children from mapNextTx txiter it = mapTx.find(hash); if (it == mapTx.end()) { continue; } auto iter = mapNextTx.lower_bound(COutPoint(hash, 0)); // First calculate the children, and update setMemPoolChildren to // include them, and update their setMemPoolParents to include this tx. for (; iter != mapNextTx.end() && iter->first->hash == hash; ++iter) { const uint256 &childHash = iter->second->GetId(); txiter childIter = mapTx.find(childHash); assert(childIter != mapTx.end()); // We can skip updating entries we've encountered before or that are // in the block (which are already accounted for). if (setChildren.insert(childIter).second && !setAlreadyIncluded.count(childHash)) { UpdateChild(it, childIter, true); UpdateParent(childIter, it, true); } } UpdateForDescendants(it, mapMemPoolDescendantsToUpdate, setAlreadyIncluded); } } bool CTxMemPool::CalculateMemPoolAncestors( const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents /* = true */) const { LOCK(cs); setEntries parentHashes; const CTransaction &tx = entry.GetTx(); if (fSearchForParents) { // Get parents of this transaction that are in the mempool // GetMemPoolParents() is only valid for entries in the mempool, so we // iterate mapTx to find parents. for (const CTxIn &in : tx.vin) { txiter piter = mapTx.find(in.prevout.hash); if (piter == mapTx.end()) { continue; } parentHashes.insert(piter); if (parentHashes.size() + 1 > limitAncestorCount) { errString = strprintf("too many unconfirmed parents [limit: %u]", limitAncestorCount); return false; } } } else { // If we're not searching for parents, we require this to be an entry in // the mempool already. txiter it = mapTx.iterator_to(entry); parentHashes = GetMemPoolParents(it); } size_t totalSizeWithAncestors = entry.GetTxSize(); while (!parentHashes.empty()) { txiter stageit = *parentHashes.begin(); setAncestors.insert(stageit); parentHashes.erase(stageit); totalSizeWithAncestors += stageit->GetTxSize(); if (stageit->GetSizeWithDescendants() + entry.GetTxSize() > limitDescendantSize) { errString = strprintf( "exceeds descendant size limit for tx %s [limit: %u]", stageit->GetTx().GetId().ToString(), limitDescendantSize); return false; } if (stageit->GetCountWithDescendants() + 1 > limitDescendantCount) { errString = strprintf("too many descendants for tx %s [limit: %u]", stageit->GetTx().GetId().ToString(), limitDescendantCount); return false; } if (totalSizeWithAncestors > limitAncestorSize) { errString = strprintf("exceeds ancestor size limit [limit: %u]", limitAncestorSize); return false; } const setEntries &setMemPoolParents = GetMemPoolParents(stageit); for (const txiter &phash : setMemPoolParents) { // If this is a new ancestor, add it. if (setAncestors.count(phash) == 0) { parentHashes.insert(phash); } if (parentHashes.size() + setAncestors.size() + 1 > limitAncestorCount) { errString = strprintf("too many unconfirmed ancestors [limit: %u]", limitAncestorCount); return false; } } } return true; } void CTxMemPool::UpdateAncestorsOf(bool add, txiter it, setEntries &setAncestors) { setEntries parentIters = GetMemPoolParents(it); // add or remove this tx as a child of each parent for (txiter piter : parentIters) { UpdateChild(piter, it, add); } const int64_t updateCount = (add ? 1 : -1); const int64_t updateSize = updateCount * it->GetTxSize(); const Amount updateFee = updateCount * it->GetModifiedFee(); for (txiter ancestorIt : setAncestors) { mapTx.modify(ancestorIt, update_descendant_state(updateSize, updateFee, updateCount)); } } void CTxMemPool::UpdateEntryForAncestors(txiter it, const setEntries &setAncestors) { int64_t updateCount = setAncestors.size(); int64_t updateSize = 0; Amount updateFee(0); int64_t updateSigOpsCount = 0; for (txiter ancestorIt : setAncestors) { updateSize += ancestorIt->GetTxSize(); updateFee += ancestorIt->GetModifiedFee(); updateSigOpsCount += ancestorIt->GetSigOpCount(); } mapTx.modify(it, update_ancestor_state(updateSize, updateFee, updateCount, updateSigOpsCount)); } void CTxMemPool::UpdateChildrenForRemoval(txiter it) { const setEntries &setMemPoolChildren = GetMemPoolChildren(it); for (txiter updateIt : setMemPoolChildren) { UpdateParent(updateIt, it, false); } } void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove, bool updateDescendants) { // For each entry, walk back all ancestors and decrement size associated // with this transaction. const uint64_t nNoLimit = std::numeric_limits::max(); if (updateDescendants) { // updateDescendants should be true whenever we're not recursively // removing a tx and all its descendants, eg when a transaction is // confirmed in a block. Here we only update statistics and not data in // mapLinks (which we need to preserve until we're finished with all // operations that need to traverse the mempool). for (txiter removeIt : entriesToRemove) { setEntries setDescendants; CalculateDescendants(removeIt, setDescendants); setDescendants.erase(removeIt); // don't update state for self int64_t modifySize = -((int64_t)removeIt->GetTxSize()); Amount modifyFee = -1 * removeIt->GetModifiedFee(); int modifySigOps = -removeIt->GetSigOpCount(); for (txiter dit : setDescendants) { mapTx.modify(dit, update_ancestor_state(modifySize, modifyFee, -1, modifySigOps)); } } } for (txiter removeIt : entriesToRemove) { setEntries setAncestors; const CTxMemPoolEntry &entry = *removeIt; std::string dummy; // Since this is a tx that is already in the mempool, we can call CMPA // with fSearchForParents = false. If the mempool is in a consistent // state, then using true or false should both be correct, though false // should be a bit faster. // However, if we happen to be in the middle of processing a reorg, then // the mempool can be in an inconsistent state. In this case, the set of // ancestors reachable via mapLinks will be the same as the set of // ancestors whose packages include this transaction, because when we // add a new transaction to the mempool in addUnchecked(), we assume it // has no children, and in the case of a reorg where that assumption is // false, the in-mempool children aren't linked to the in-block tx's // until UpdateTransactionsFromBlock() is called. So if we're being // called during a reorg, ie before UpdateTransactionsFromBlock() has // been called, then mapLinks[] will differ from the set of mempool // parents we'd calculate by searching, and it's important that we use // the mapLinks[] notion of ancestor transactions as the set of things // to update for removal. CalculateMemPoolAncestors(entry, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false); // Note that UpdateAncestorsOf severs the child links that point to // removeIt in the entries for the parents of removeIt. UpdateAncestorsOf(false, removeIt, setAncestors); } // After updating all the ancestor sizes, we can now sever the link between // each transaction being removed and any mempool children (ie, update // setMemPoolParents for each direct child of a transaction being removed). for (txiter removeIt : entriesToRemove) { UpdateChildrenForRemoval(removeIt); } } void CTxMemPoolEntry::UpdateDescendantState(int64_t modifySize, Amount modifyFee, int64_t modifyCount) { nSizeWithDescendants += modifySize; assert(int64_t(nSizeWithDescendants) > 0); nModFeesWithDescendants += modifyFee; nCountWithDescendants += modifyCount; assert(int64_t(nCountWithDescendants) > 0); } void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, Amount modifyFee, int64_t modifyCount, int modifySigOps) { nSizeWithAncestors += modifySize; assert(int64_t(nSizeWithAncestors) > 0); nModFeesWithAncestors += modifyFee; nCountWithAncestors += modifyCount; assert(int64_t(nCountWithAncestors) > 0); nSigOpCountWithAncestors += modifySigOps; assert(int(nSigOpCountWithAncestors) >= 0); } -CTxMemPool::CTxMemPool(const CFeeRate &_minReasonableRelayFee) - : nTransactionsUpdated(0) { +CTxMemPool::CTxMemPool() : nTransactionsUpdated(0) { // lock free clear _clear(); // Sanity checks off by default for performance, because otherwise accepting // transactions becomes O(N^2) where N is the number of transactions in the // pool nCheckFrequency = 0; - minerPolicyEstimator = new CBlockPolicyEstimator(_minReasonableRelayFee); + minerPolicyEstimator = new CBlockPolicyEstimator(); } CTxMemPool::~CTxMemPool() { delete minerPolicyEstimator; } bool CTxMemPool::isSpent(const COutPoint &outpoint) { LOCK(cs); return mapNextTx.count(outpoint); } unsigned int CTxMemPool::GetTransactionsUpdated() const { LOCK(cs); return nTransactionsUpdated; } void CTxMemPool::AddTransactionsUpdated(unsigned int n) { LOCK(cs); nTransactionsUpdated += n; } bool CTxMemPool::addUnchecked(const uint256 &hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool validFeeEstimate) { NotifyEntryAdded(entry.GetSharedTx()); // Add to memory pool without checking anything. // Used by AcceptToMemoryPool(), which DOES do all the appropriate checks. LOCK(cs); indexed_transaction_set::iterator newit = mapTx.insert(entry).first; mapLinks.insert(make_pair(newit, TxLinks())); // Update transaction for any feeDelta created by PrioritiseTransaction // TODO: refactor so that the fee delta is calculated before inserting into // mapTx. std::map>::const_iterator pos = mapDeltas.find(hash); if (pos != mapDeltas.end()) { const std::pair &deltas = pos->second; if (deltas.second != Amount(0)) { mapTx.modify(newit, update_fee_delta(deltas.second)); } } // Update cachedInnerUsage to include contained transaction's usage. // (When we update the entry for in-mempool parents, memory usage will be // further updated.) cachedInnerUsage += entry.DynamicMemoryUsage(); const CTransaction &tx = newit->GetTx(); std::set setParentTransactions; for (const CTxIn &in : tx.vin) { mapNextTx.insert(std::make_pair(&in.prevout, &tx)); setParentTransactions.insert(in.prevout.hash); } // Don't bother worrying about child transactions of this one. Normal case // of a new transaction arriving is that there can't be any children, // because such children would be orphans. An exception to that is if a // transaction enters that used to be in a block. In that case, our // disconnect block logic will call UpdateTransactionsFromBlock to clean up // the mess we're leaving here. // Update ancestors with information about this tx for (const uint256 &phash : setParentTransactions) { txiter pit = mapTx.find(phash); if (pit != mapTx.end()) { UpdateParent(newit, pit, true); } } UpdateAncestorsOf(true, newit, setAncestors); UpdateEntryForAncestors(newit, setAncestors); nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); minerPolicyEstimator->processTransaction(entry, validFeeEstimate); vTxHashes.emplace_back(tx.GetHash(), newit); newit->vTxHashesIdx = vTxHashes.size() - 1; return true; } void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) { NotifyEntryRemoved(it->GetSharedTx(), reason); const uint256 txid = it->GetTx().GetId(); for (const CTxIn &txin : it->GetTx().vin) { mapNextTx.erase(txin.prevout); } if (vTxHashes.size() > 1) { vTxHashes[it->vTxHashesIdx] = std::move(vTxHashes.back()); vTxHashes[it->vTxHashesIdx].second->vTxHashesIdx = it->vTxHashesIdx; vTxHashes.pop_back(); if (vTxHashes.size() * 2 < vTxHashes.capacity()) { vTxHashes.shrink_to_fit(); } } else { vTxHashes.clear(); } totalTxSize -= it->GetTxSize(); cachedInnerUsage -= it->DynamicMemoryUsage(); cachedInnerUsage -= memusage::DynamicUsage(mapLinks[it].parents) + memusage::DynamicUsage(mapLinks[it].children); mapLinks.erase(it); mapTx.erase(it); nTransactionsUpdated++; minerPolicyEstimator->removeTx(txid); } // Calculates descendants of entry that are not already in setDescendants, and // adds to setDescendants. Assumes entryit is already a tx in the mempool and // setMemPoolChildren is correct for tx and all descendants. Also assumes that // if an entry is in setDescendants already, then all in-mempool descendants of // it are already in setDescendants as well, so that we can save time by not // iterating over those entries. void CTxMemPool::CalculateDescendants(txiter entryit, setEntries &setDescendants) { setEntries stage; if (setDescendants.count(entryit) == 0) { stage.insert(entryit); } // Traverse down the children of entry, only adding children that are not // accounted for in setDescendants already (because those children have // either already been walked, or will be walked in this iteration). while (!stage.empty()) { txiter it = *stage.begin(); setDescendants.insert(it); stage.erase(it); const setEntries &setChildren = GetMemPoolChildren(it); for (const txiter &childiter : setChildren) { if (!setDescendants.count(childiter)) { stage.insert(childiter); } } } } void CTxMemPool::removeRecursive(const CTransaction &origTx, MemPoolRemovalReason reason) { // Remove transaction from memory pool. LOCK(cs); setEntries txToRemove; txiter origit = mapTx.find(origTx.GetId()); if (origit != mapTx.end()) { txToRemove.insert(origit); } else { // When recursively removing but origTx isn't in the mempool be sure to // remove any children that are in the pool. This can happen during // chain re-orgs if origTx isn't re-accepted into the mempool for any // reason. for (size_t i = 0; i < origTx.vout.size(); i++) { auto it = mapNextTx.find(COutPoint(origTx.GetId(), i)); if (it == mapNextTx.end()) { continue; } txiter nextit = mapTx.find(it->second->GetId()); assert(nextit != mapTx.end()); txToRemove.insert(nextit); } } setEntries setAllRemoves; for (txiter it : txToRemove) { CalculateDescendants(it, setAllRemoves); } RemoveStaged(setAllRemoves, false, reason); } void CTxMemPool::removeForReorg(const Config &config, const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags) { // Remove transactions spending a coinbase which are now immature and // no-longer-final transactions. LOCK(cs); setEntries txToRemove; for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { const CTransaction &tx = it->GetTx(); LockPoints lp = it->GetLockPoints(); bool validLP = TestLockPointValidity(&lp); CValidationState state; if (!ContextualCheckTransactionForCurrentBlock(config, tx, state, flags) || !CheckSequenceLocks(tx, flags, &lp, validLP)) { // Note if CheckSequenceLocks fails the LockPoints may still be // invalid. So it's critical that we remove the tx and not depend on // the LockPoints. txToRemove.insert(it); } else if (it->GetSpendsCoinbase()) { for (const CTxIn &txin : tx.vin) { indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); if (it2 != mapTx.end()) { continue; } const Coin &coin = pcoins->AccessCoin(txin.prevout); if (nCheckFrequency != 0) { assert(!coin.IsSpent()); } if (coin.IsSpent() || (coin.IsCoinBase() && int64_t(nMemPoolHeight) - coin.GetHeight() < COINBASE_MATURITY)) { txToRemove.insert(it); break; } } } if (!validLP) { mapTx.modify(it, update_lock_points(lp)); } } setEntries setAllRemoves; for (txiter it : txToRemove) { CalculateDescendants(it, setAllRemoves); } RemoveStaged(setAllRemoves, false, MemPoolRemovalReason::REORG); } void CTxMemPool::removeConflicts(const CTransaction &tx) { // Remove transactions which depend on inputs of tx, recursively LOCK(cs); for (const CTxIn &txin : tx.vin) { auto it = mapNextTx.find(txin.prevout); if (it != mapNextTx.end()) { const CTransaction &txConflict = *it->second; if (txConflict != tx) { ClearPrioritisation(txConflict.GetId()); removeRecursive(txConflict, MemPoolRemovalReason::CONFLICT); } } } } /** * Called when a block is connected. Removes from mempool and updates the miner * fee estimator. */ void CTxMemPool::removeForBlock(const std::vector &vtx, unsigned int nBlockHeight) { LOCK(cs); std::vector entries; for (const auto &tx : vtx) { uint256 txid = tx->GetId(); indexed_transaction_set::iterator i = mapTx.find(txid); if (i != mapTx.end()) { entries.push_back(&*i); } } // Before the txs in the new block have been removed from the mempool, // update policy estimates minerPolicyEstimator->processBlock(nBlockHeight, entries); for (const auto &tx : vtx) { txiter it = mapTx.find(tx->GetId()); if (it != mapTx.end()) { setEntries stage; stage.insert(it); RemoveStaged(stage, true, MemPoolRemovalReason::BLOCK); } removeConflicts(*tx); ClearPrioritisation(tx->GetId()); } lastRollingFeeUpdate = GetTime(); blockSinceLastRollingFeeBump = true; } void CTxMemPool::_clear() { mapLinks.clear(); mapTx.clear(); mapNextTx.clear(); vTxHashes.clear(); totalTxSize = 0; cachedInnerUsage = 0; lastRollingFeeUpdate = GetTime(); blockSinceLastRollingFeeBump = false; rollingMinimumFeeRate = 0; ++nTransactionsUpdated; } void CTxMemPool::clear() { LOCK(cs); _clear(); } void CTxMemPool::check(const CCoinsViewCache *pcoins) const { if (nCheckFrequency == 0) { return; } if (GetRand(std::numeric_limits::max()) >= nCheckFrequency) { return; } LogPrint(BCLog::MEMPOOL, "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); uint64_t checkTotal = 0; uint64_t innerUsage = 0; CCoinsViewCache mempoolDuplicate(const_cast(pcoins)); const int64_t nSpendHeight = GetSpendHeight(mempoolDuplicate); LOCK(cs); std::list waitingOnDependants; for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { unsigned int i = 0; checkTotal += it->GetTxSize(); innerUsage += it->DynamicMemoryUsage(); const CTransaction &tx = it->GetTx(); txlinksMap::const_iterator linksiter = mapLinks.find(it); assert(linksiter != mapLinks.end()); const TxLinks &links = linksiter->second; innerUsage += memusage::DynamicUsage(links.parents) + memusage::DynamicUsage(links.children); bool fDependsWait = false; setEntries setParentCheck; int64_t parentSizes = 0; int64_t parentSigOpCount = 0; for (const CTxIn &txin : tx.vin) { // Check that every mempool transaction's inputs refer to available // coins, or other mempool tx's. indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); if (it2 != mapTx.end()) { const CTransaction &tx2 = it2->GetTx(); assert(tx2.vout.size() > txin.prevout.n && !tx2.vout[txin.prevout.n].IsNull()); fDependsWait = true; if (setParentCheck.insert(it2).second) { parentSizes += it2->GetTxSize(); parentSigOpCount += it2->GetSigOpCount(); } } else { assert(pcoins->HaveCoin(txin.prevout)); } // Check whether its inputs are marked in mapNextTx. auto it3 = mapNextTx.find(txin.prevout); assert(it3 != mapNextTx.end()); assert(it3->first == &txin.prevout); assert(it3->second == &tx); i++; } assert(setParentCheck == GetMemPoolParents(it)); // Verify ancestor state is correct. setEntries setAncestors; uint64_t nNoLimit = std::numeric_limits::max(); std::string dummy; CalculateMemPoolAncestors(*it, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy); uint64_t nCountCheck = setAncestors.size() + 1; uint64_t nSizeCheck = it->GetTxSize(); Amount nFeesCheck = it->GetModifiedFee(); int64_t nSigOpCheck = it->GetSigOpCount(); for (txiter ancestorIt : setAncestors) { nSizeCheck += ancestorIt->GetTxSize(); nFeesCheck += ancestorIt->GetModifiedFee(); nSigOpCheck += ancestorIt->GetSigOpCount(); } assert(it->GetCountWithAncestors() == nCountCheck); assert(it->GetSizeWithAncestors() == nSizeCheck); assert(it->GetSigOpCountWithAncestors() == nSigOpCheck); assert(it->GetModFeesWithAncestors() == nFeesCheck); // Check children against mapNextTx CTxMemPool::setEntries setChildrenCheck; auto iter = mapNextTx.lower_bound(COutPoint(it->GetTx().GetId(), 0)); int64_t childSizes = 0; for (; iter != mapNextTx.end() && iter->first->hash == it->GetTx().GetId(); ++iter) { txiter childit = mapTx.find(iter->second->GetId()); // mapNextTx points to in-mempool transactions assert(childit != mapTx.end()); if (setChildrenCheck.insert(childit).second) { childSizes += childit->GetTxSize(); } } assert(setChildrenCheck == GetMemPoolChildren(it)); // Also check to make sure size is greater than sum with immediate // children. Just a sanity check, not definitive that this calc is // correct... assert(it->GetSizeWithDescendants() >= childSizes + it->GetTxSize()); if (fDependsWait) { waitingOnDependants.push_back(&(*it)); } else { CValidationState state; bool fCheckResult = tx.IsCoinBase() || Consensus::CheckTxInputs( tx, state, mempoolDuplicate, nSpendHeight); assert(fCheckResult); UpdateCoins(tx, mempoolDuplicate, 1000000); } } unsigned int stepsSinceLastRemove = 0; while (!waitingOnDependants.empty()) { const CTxMemPoolEntry *entry = waitingOnDependants.front(); waitingOnDependants.pop_front(); CValidationState state; if (!mempoolDuplicate.HaveInputs(entry->GetTx())) { waitingOnDependants.push_back(entry); stepsSinceLastRemove++; assert(stepsSinceLastRemove < waitingOnDependants.size()); } else { bool fCheckResult = entry->GetTx().IsCoinBase() || Consensus::CheckTxInputs(entry->GetTx(), state, mempoolDuplicate, nSpendHeight); assert(fCheckResult); UpdateCoins(entry->GetTx(), mempoolDuplicate, 1000000); stepsSinceLastRemove = 0; } } for (auto it = mapNextTx.cbegin(); it != mapNextTx.cend(); it++) { uint256 txid = it->second->GetId(); indexed_transaction_set::const_iterator it2 = mapTx.find(txid); const CTransaction &tx = it2->GetTx(); assert(it2 != mapTx.end()); assert(&tx == it->second); } assert(totalTxSize == checkTotal); assert(innerUsage == cachedInnerUsage); } bool CTxMemPool::CompareDepthAndScore(const uint256 &hasha, const uint256 &hashb) { LOCK(cs); indexed_transaction_set::const_iterator i = mapTx.find(hasha); if (i == mapTx.end()) { return false; } indexed_transaction_set::const_iterator j = mapTx.find(hashb); if (j == mapTx.end()) { return true; } uint64_t counta = i->GetCountWithAncestors(); uint64_t countb = j->GetCountWithAncestors(); if (counta == countb) { return CompareTxMemPoolEntryByScore()(*i, *j); } return counta < countb; } namespace { class DepthAndScoreComparator { public: bool operator()(const CTxMemPool::indexed_transaction_set::const_iterator &a, const CTxMemPool::indexed_transaction_set::const_iterator &b) { uint64_t counta = a->GetCountWithAncestors(); uint64_t countb = b->GetCountWithAncestors(); if (counta == countb) { return CompareTxMemPoolEntryByScore()(*a, *b); } return counta < countb; } }; } // namespace std::vector CTxMemPool::GetSortedDepthAndScore() const { std::vector iters; AssertLockHeld(cs); iters.reserve(mapTx.size()); for (indexed_transaction_set::iterator mi = mapTx.begin(); mi != mapTx.end(); ++mi) { iters.push_back(mi); } std::sort(iters.begin(), iters.end(), DepthAndScoreComparator()); return iters; } void CTxMemPool::queryHashes(std::vector &vtxid) { LOCK(cs); auto iters = GetSortedDepthAndScore(); vtxid.clear(); vtxid.reserve(mapTx.size()); for (auto it : iters) { vtxid.push_back(it->GetTx().GetId()); } } static TxMempoolInfo GetInfo(CTxMemPool::indexed_transaction_set::const_iterator it) { return TxMempoolInfo{it->GetSharedTx(), it->GetTime(), CFeeRate(it->GetFee(), it->GetTxSize()), it->GetModifiedFee() - it->GetFee()}; } std::vector CTxMemPool::infoAll() const { LOCK(cs); auto iters = GetSortedDepthAndScore(); std::vector ret; ret.reserve(mapTx.size()); for (auto it : iters) { ret.push_back(GetInfo(it)); } return ret; } CTransactionRef CTxMemPool::get(const uint256 &txid) const { LOCK(cs); indexed_transaction_set::const_iterator i = mapTx.find(txid); if (i == mapTx.end()) { return nullptr; } return i->GetSharedTx(); } TxMempoolInfo CTxMemPool::info(const uint256 &txid) const { LOCK(cs); indexed_transaction_set::const_iterator i = mapTx.find(txid); if (i == mapTx.end()) { return TxMempoolInfo(); } return GetInfo(i); } CFeeRate CTxMemPool::estimateFee(int nBlocks) const { LOCK(cs); return minerPolicyEstimator->estimateFee(nBlocks); } CFeeRate CTxMemPool::estimateSmartFee(int nBlocks, int *answerFoundAtBlocks) const { LOCK(cs); return minerPolicyEstimator->estimateSmartFee(nBlocks, answerFoundAtBlocks, *this); } double CTxMemPool::estimateSmartPriority(int nBlocks, int *answerFoundAtBlocks) const { LOCK(cs); return minerPolicyEstimator->estimateSmartPriority( nBlocks, answerFoundAtBlocks, *this); } bool CTxMemPool::WriteFeeEstimates(CAutoFile &fileout) const { try { LOCK(cs); // version required to read: 0.13.99 or later fileout << 139900; // version that wrote the file fileout << CLIENT_VERSION; minerPolicyEstimator->Write(fileout); } catch (const std::exception &) { LogPrintf("CTxMemPool::WriteFeeEstimates(): unable to write policy " "estimator data (non-fatal)\n"); return false; } return true; } bool CTxMemPool::ReadFeeEstimates(CAutoFile &filein) { try { int nVersionRequired, nVersionThatWrote; filein >> nVersionRequired >> nVersionThatWrote; if (nVersionRequired > CLIENT_VERSION) { return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee " "estimate file", nVersionRequired); } LOCK(cs); minerPolicyEstimator->Read(filein, nVersionThatWrote); } catch (const std::exception &) { LogPrintf("CTxMemPool::ReadFeeEstimates(): unable to read policy " "estimator data (non-fatal)\n"); return false; } return true; } void CTxMemPool::PrioritiseTransaction(const uint256 hash, const std::string strHash, double dPriorityDelta, const Amount nFeeDelta) { { LOCK(cs); std::pair &deltas = mapDeltas[hash]; deltas.first += dPriorityDelta; deltas.second += nFeeDelta; txiter it = mapTx.find(hash); if (it != mapTx.end()) { mapTx.modify(it, update_fee_delta(deltas.second)); // Now update all ancestors' modified fees with descendants setEntries setAncestors; uint64_t nNoLimit = std::numeric_limits::max(); std::string dummy; CalculateMemPoolAncestors(*it, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false); for (txiter ancestorIt : setAncestors) { mapTx.modify(ancestorIt, update_descendant_state(0, nFeeDelta, 0)); } // Now update all descendants' modified fees with ancestors setEntries setDescendants; CalculateDescendants(it, setDescendants); setDescendants.erase(it); for (txiter descendantIt : setDescendants) { mapTx.modify(descendantIt, update_ancestor_state(0, nFeeDelta, 0, 0)); } } } LogPrintf("PrioritiseTransaction: %s priority += %f, fee += %d\n", strHash, dPriorityDelta, FormatMoney(nFeeDelta)); } void CTxMemPool::ApplyDeltas(const uint256 hash, double &dPriorityDelta, Amount &nFeeDelta) const { LOCK(cs); std::map>::const_iterator pos = mapDeltas.find(hash); if (pos == mapDeltas.end()) { return; } const std::pair &deltas = pos->second; dPriorityDelta += deltas.first; nFeeDelta += deltas.second; } void CTxMemPool::ClearPrioritisation(const uint256 hash) { LOCK(cs); mapDeltas.erase(hash); } bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const { for (const CTxIn &in : tx.vin) { if (exists(in.prevout.hash)) { return false; } } return true; } CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView *baseIn, const CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) {} bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { // If an entry in the mempool exists, always return that one, as it's // guaranteed to never conflict with the underlying cache, and it cannot // have pruned entries (as it contains full) transactions. First checking // the underlying cache risks returning a pruned entry instead. CTransactionRef ptx = mempool.get(outpoint.hash); if (ptx) { if (outpoint.n < ptx->vout.size()) { coin = Coin(ptx->vout[outpoint.n], MEMPOOL_HEIGHT, false); return true; } return false; } return base->GetCoin(outpoint, coin) && !coin.IsSpent(); } bool CCoinsViewMemPool::HaveCoin(const COutPoint &outpoint) const { return mempool.exists(outpoint) || base->HaveCoin(outpoint); } size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); // Estimate the overhead of mapTx to be 15 pointers + an allocation, as no // exact formula for boost::multi_index_contained is implemented. return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void *)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; } void CTxMemPool::RemoveStaged(setEntries &stage, bool updateDescendants, MemPoolRemovalReason reason) { AssertLockHeld(cs); UpdateForRemoveFromMempool(stage, updateDescendants); for (const txiter &it : stage) { removeUnchecked(it, reason); } } int CTxMemPool::Expire(int64_t time) { LOCK(cs); indexed_transaction_set::index::type::iterator it = mapTx.get().begin(); setEntries toremove; while (it != mapTx.get().end() && it->GetTime() < time) { toremove.insert(mapTx.project<0>(it)); it++; } setEntries stage; for (txiter removeit : toremove) { CalculateDescendants(removeit, stage); } RemoveStaged(stage, false, MemPoolRemovalReason::EXPIRY); return stage.size(); } bool CTxMemPool::addUnchecked(const uint256 &hash, const CTxMemPoolEntry &entry, bool validFeeEstimate) { LOCK(cs); setEntries setAncestors; uint64_t nNoLimit = std::numeric_limits::max(); std::string dummy; CalculateMemPoolAncestors(entry, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy); return addUnchecked(hash, entry, setAncestors, validFeeEstimate); } void CTxMemPool::UpdateChild(txiter entry, txiter child, bool add) { setEntries s; if (add && mapLinks[entry].children.insert(child).second) { cachedInnerUsage += memusage::IncrementalDynamicUsage(s); } else if (!add && mapLinks[entry].children.erase(child)) { cachedInnerUsage -= memusage::IncrementalDynamicUsage(s); } } void CTxMemPool::UpdateParent(txiter entry, txiter parent, bool add) { setEntries s; if (add && mapLinks[entry].parents.insert(parent).second) { cachedInnerUsage += memusage::IncrementalDynamicUsage(s); } else if (!add && mapLinks[entry].parents.erase(parent)) { cachedInnerUsage -= memusage::IncrementalDynamicUsage(s); } } const CTxMemPool::setEntries & CTxMemPool::GetMemPoolParents(txiter entry) const { assert(entry != mapTx.end()); txlinksMap::const_iterator it = mapLinks.find(entry); assert(it != mapLinks.end()); return it->second.parents; } const CTxMemPool::setEntries & CTxMemPool::GetMemPoolChildren(txiter entry) const { assert(entry != mapTx.end()); txlinksMap::const_iterator it = mapLinks.find(entry); assert(it != mapLinks.end()); return it->second.children; } CFeeRate CTxMemPool::GetMinFee(size_t sizelimit) const { LOCK(cs); if (!blockSinceLastRollingFeeBump || rollingMinimumFeeRate == 0) { return CFeeRate(Amount(int64_t(rollingMinimumFeeRate))); } int64_t time = GetTime(); if (time > lastRollingFeeUpdate + 10) { double halflife = ROLLING_FEE_HALFLIFE; if (DynamicMemoryUsage() < sizelimit / 4) { halflife /= 4; } else if (DynamicMemoryUsage() < sizelimit / 2) { halflife /= 2; } rollingMinimumFeeRate = rollingMinimumFeeRate / pow(2.0, (time - lastRollingFeeUpdate) / halflife); lastRollingFeeUpdate = time; if (rollingMinimumFeeRate < (double)incrementalRelayFee.GetFeePerK().GetSatoshis() / 2) { rollingMinimumFeeRate = 0; return CFeeRate(Amount(0)); } } return std::max(CFeeRate(Amount(int64_t(rollingMinimumFeeRate))), incrementalRelayFee); } void CTxMemPool::trackPackageRemoved(const CFeeRate &rate) { AssertLockHeld(cs); if (rate.GetFeePerK().GetSatoshis() > rollingMinimumFeeRate) { rollingMinimumFeeRate = rate.GetFeePerK().GetSatoshis(); blockSinceLastRollingFeeBump = false; } } void CTxMemPool::TrimToSize(size_t sizelimit, std::vector *pvNoSpendsRemaining) { LOCK(cs); unsigned nTxnRemoved = 0; CFeeRate maxFeeRateRemoved(Amount(0)); while (!mapTx.empty() && DynamicMemoryUsage() > sizelimit) { indexed_transaction_set::index::type::iterator it = mapTx.get().begin(); // We set the new mempool min fee to the feerate of the removed set, // plus the "minimum reasonable fee rate" (ie some value under which we // consider txn to have 0 fee). This way, we don't allow txn to enter // mempool with feerate equal to txn which were removed with no block in // between. CFeeRate removed(it->GetModFeesWithDescendants(), it->GetSizeWithDescendants()); removed += incrementalRelayFee; trackPackageRemoved(removed); maxFeeRateRemoved = std::max(maxFeeRateRemoved, removed); setEntries stage; CalculateDescendants(mapTx.project<0>(it), stage); nTxnRemoved += stage.size(); std::vector txn; if (pvNoSpendsRemaining) { txn.reserve(stage.size()); for (txiter iter : stage) { txn.push_back(iter->GetTx()); } } RemoveStaged(stage, false, MemPoolRemovalReason::SIZELIMIT); if (pvNoSpendsRemaining) { for (const CTransaction &tx : txn) { for (const CTxIn &txin : tx.vin) { if (exists(txin.prevout.hash)) { continue; } if (!mapNextTx.count(txin.prevout)) { pvNoSpendsRemaining->push_back(txin.prevout); } } } } } if (maxFeeRateRemoved > CFeeRate(Amount(0))) { LogPrint(BCLog::MEMPOOL, "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); } } bool CTxMemPool::TransactionWithinChainLimit(const uint256 &txid, size_t chainLimit) const { LOCK(cs); auto it = mapTx.find(txid); return it == mapTx.end() || (it->GetCountWithAncestors() < chainLimit && it->GetCountWithDescendants() < chainLimit); } SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} diff --git a/src/txmempool.h b/src/txmempool.h index 9d780c1ac9..0ff3fb07fd 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -1,905 +1,905 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_TXMEMPOOL_H #define BITCOIN_TXMEMPOOL_H #include "amount.h" #include "coins.h" #include "indirectmap.h" #include "primitives/transaction.h" #include "random.h" #include "sync.h" #include #include #include #include #include #include #include #include #include #include #include class CAutoFile; class CBlockIndex; class Config; inline double AllowFreeThreshold() { return COIN.GetSatoshis() * 144 / 250; } inline bool AllowFree(double dPriority) { // Large (in bytes) low-priority (new, small-coin) transactions need a fee. return dPriority > AllowFreeThreshold(); } /** * Fake height value used in Coins to signify they are only in the memory * pool(since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; struct LockPoints { // Will be set to the blockchain height and median time past values that // would be necessary to satisfy all relative locktime constraints (BIP68) // of this tx given our view of block chain history int height; int64_t time; // As long as the current chain descends from the highest height block // containing one of the inputs used in the calculation, then the cached // values are still valid even after a reorg. CBlockIndex *maxInputBlock; LockPoints() : height(0), time(0), maxInputBlock(nullptr) {} }; class CTxMemPool; /** \class CTxMemPoolEntry * * CTxMemPoolEntry stores data about the corresponding transaction, as well as * data about all in-mempool transactions that depend on the transaction * ("descendant" transactions). * * When a new entry is added to the mempool, we update the descendant state * (nCountWithDescendants, nSizeWithDescendants, and nModFeesWithDescendants) * for all ancestors of the newly added transaction. * * If updating the descendant state is skipped, we can mark the entry as * "dirty", and set nSizeWithDescendants/nModFeesWithDescendants to equal * nTxSize/nFee+feeDelta. (This can potentially happen during a reorg, where we * limit the amount of work we're willing to do to avoid consuming too much * CPU.) */ class CTxMemPoolEntry { private: CTransactionRef tx; //!< Cached to avoid expensive parent-transaction lookups Amount nFee; //!< ... and avoid recomputing tx size size_t nTxSize; //!< ... and modified size for priority size_t nModSize; //!< ... and total memory usage size_t nUsageSize; //!< Local time when entering the mempool int64_t nTime; //!< Priority when entering the mempool double entryPriority; //!< Chain height when entering the mempool unsigned int entryHeight; //!< Sum of all txin values that are already in blockchain Amount inChainInputValue; //!< keep track of transactions that spend a coinbase bool spendsCoinbase; //!< Total sigop plus P2SH sigops count int64_t sigOpCount; //!< Used for determining the priority of the transaction for mining in a //! block Amount feeDelta; //!< Track the height and time at which tx was final LockPoints lockPoints; // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these // descendants as well. if nCountWithDescendants is 0, treat this entry as // dirty, and nSizeWithDescendants and nModFeesWithDescendants will not be // correct. //!< number of descendant transactions uint64_t nCountWithDescendants; //!< ... and size uint64_t nSizeWithDescendants; //!< ... and total fees (all including us) Amount nModFeesWithDescendants; // Analogous statistics for ancestor transactions uint64_t nCountWithAncestors; uint64_t nSizeWithAncestors; Amount nModFeesWithAncestors; int64_t nSigOpCountWithAncestors; public: CTxMemPoolEntry(const CTransactionRef &_tx, const Amount _nFee, int64_t _nTime, double _entryPriority, unsigned int _entryHeight, Amount _inChainInputValue, bool spendsCoinbase, int64_t nSigOpsCost, LockPoints lp); CTxMemPoolEntry(const CTxMemPoolEntry &other); const CTransaction &GetTx() const { return *this->tx; } CTransactionRef GetSharedTx() const { return this->tx; } /** * Fast calculation of lower bound of current priority as update from entry * priority. Only inputs that were originally in-chain will age. */ double GetPriority(unsigned int currentHeight) const; const Amount GetFee() const { return nFee; } size_t GetTxSize() const { return nTxSize; } int64_t GetTime() const { return nTime; } unsigned int GetHeight() const { return entryHeight; } int64_t GetSigOpCount() const { return sigOpCount; } Amount GetModifiedFee() const { return nFee + feeDelta; } size_t DynamicMemoryUsage() const { return nUsageSize; } const LockPoints &GetLockPoints() const { return lockPoints; } // Adjusts the descendant state, if this entry is not dirty. void UpdateDescendantState(int64_t modifySize, Amount modifyFee, int64_t modifyCount); // Adjusts the ancestor state void UpdateAncestorState(int64_t modifySize, Amount modifyFee, int64_t modifyCount, int modifySigOps); // Updates the fee delta used for mining priority score, and the // modified fees with descendants. void UpdateFeeDelta(Amount feeDelta); // Update the LockPoints after a reorg void UpdateLockPoints(const LockPoints &lp); uint64_t GetCountWithDescendants() const { return nCountWithDescendants; } uint64_t GetSizeWithDescendants() const { return nSizeWithDescendants; } Amount GetModFeesWithDescendants() const { return nModFeesWithDescendants; } bool GetSpendsCoinbase() const { return spendsCoinbase; } uint64_t GetCountWithAncestors() const { return nCountWithAncestors; } uint64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } Amount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } int64_t GetSigOpCountWithAncestors() const { return nSigOpCountWithAncestors; } //!< Index in mempool's vTxHashes mutable size_t vTxHashesIdx; }; // Helpers for modifying CTxMemPool::mapTx, which is a boost multi_index. struct update_descendant_state { update_descendant_state(int64_t _modifySize, Amount _modifyFee, int64_t _modifyCount) : modifySize(_modifySize), modifyFee(_modifyFee), modifyCount(_modifyCount) {} void operator()(CTxMemPoolEntry &e) { e.UpdateDescendantState(modifySize, modifyFee, modifyCount); } private: int64_t modifySize; Amount modifyFee; int64_t modifyCount; }; struct update_ancestor_state { update_ancestor_state(int64_t _modifySize, Amount _modifyFee, int64_t _modifyCount, int64_t _modifySigOpsCost) : modifySize(_modifySize), modifyFee(_modifyFee), modifyCount(_modifyCount), modifySigOpsCost(_modifySigOpsCost) {} void operator()(CTxMemPoolEntry &e) { e.UpdateAncestorState(modifySize, modifyFee, modifyCount, modifySigOpsCost); } private: int64_t modifySize; Amount modifyFee; int64_t modifyCount; int64_t modifySigOpsCost; }; struct update_fee_delta { update_fee_delta(Amount _feeDelta) : feeDelta(_feeDelta) {} void operator()(CTxMemPoolEntry &e) { e.UpdateFeeDelta(feeDelta); } private: Amount feeDelta; }; struct update_lock_points { update_lock_points(const LockPoints &_lp) : lp(_lp) {} void operator()(CTxMemPoolEntry &e) { e.UpdateLockPoints(lp); } private: const LockPoints &lp; }; // extracts a transaction hash from CTxMempoolEntry or CTransactionRef struct mempoolentry_txid { typedef uint256 result_type; result_type operator()(const CTxMemPoolEntry &entry) const { return entry.GetTx().GetId(); } result_type operator()(const CTransactionRef &tx) const { return tx->GetId(); } }; /** \class CompareTxMemPoolEntryByDescendantScore * * Sort an entry by max(score/size of entry's tx, score/size with all * descendants). */ class CompareTxMemPoolEntryByDescendantScore { public: bool operator()(const CTxMemPoolEntry &a, const CTxMemPoolEntry &b) const { bool fUseADescendants = UseDescendantScore(a); bool fUseBDescendants = UseDescendantScore(b); double aModFee = (fUseADescendants ? a.GetModFeesWithDescendants() : a.GetModifiedFee()) .GetSatoshis(); double aSize = fUseADescendants ? a.GetSizeWithDescendants() : a.GetTxSize(); double bModFee = (fUseBDescendants ? b.GetModFeesWithDescendants() : b.GetModifiedFee()) .GetSatoshis(); double bSize = fUseBDescendants ? b.GetSizeWithDescendants() : b.GetTxSize(); // Avoid division by rewriting (a/b > c/d) as (a*d > c*b). double f1 = aModFee * bSize; double f2 = aSize * bModFee; if (f1 == f2) { return a.GetTime() >= b.GetTime(); } return f1 < f2; } // Calculate which score to use for an entry (avoiding division). bool UseDescendantScore(const CTxMemPoolEntry &a) const { double f1 = double(a.GetSizeWithDescendants() * a.GetModifiedFee().GetSatoshis()); double f2 = double(a.GetTxSize() * a.GetModFeesWithDescendants().GetSatoshis()); return f2 > f1; } }; /** \class CompareTxMemPoolEntryByScore * * Sort by score of entry ((fee+delta)/size) in descending order */ class CompareTxMemPoolEntryByScore { public: bool operator()(const CTxMemPoolEntry &a, const CTxMemPoolEntry &b) const { double f1 = double(b.GetTxSize() * a.GetModifiedFee().GetSatoshis()); double f2 = double(a.GetTxSize() * b.GetModifiedFee().GetSatoshis()); if (f1 == f2) { return b.GetTx().GetId() < a.GetTx().GetId(); } return f1 > f2; } }; class CompareTxMemPoolEntryByEntryTime { public: bool operator()(const CTxMemPoolEntry &a, const CTxMemPoolEntry &b) const { return a.GetTime() < b.GetTime(); } }; class CompareTxMemPoolEntryByAncestorFee { public: bool operator()(const CTxMemPoolEntry &a, const CTxMemPoolEntry &b) const { double aFees = double(a.GetModFeesWithAncestors().GetSatoshis()); double aSize = a.GetSizeWithAncestors(); double bFees = double(b.GetModFeesWithAncestors().GetSatoshis()); double bSize = b.GetSizeWithAncestors(); // Avoid division by rewriting (a/b > c/d) as (a*d > c*b). double f1 = aFees * bSize; double f2 = aSize * bFees; if (f1 == f2) { return a.GetTx().GetId() < b.GetTx().GetId(); } return f1 > f2; } }; // Multi_index tag names struct descendant_score {}; struct entry_time {}; struct mining_score {}; struct ancestor_score {}; class CBlockPolicyEstimator; /** * Information about a mempool transaction. */ struct TxMempoolInfo { /** The transaction itself */ CTransactionRef tx; /** Time the transaction entered the mempool. */ int64_t nTime; /** Feerate of the transaction. */ CFeeRate feeRate; /** The fee delta. */ Amount nFeeDelta; }; /** * Reason why a transaction was removed from the mempool, this is passed to the * notification signal. */ enum class MemPoolRemovalReason { //! Manually removed or unknown reason UNKNOWN = 0, //! Expired from mempool EXPIRY, //! Removed in size limiting SIZELIMIT, //! Removed for reorganization REORG, //! Removed for block BLOCK, //! Removed for conflict with in-block transaction CONFLICT, //! Removed for replacement REPLACED }; class SaltedTxidHasher { private: /** Salt */ const uint64_t k0, k1; public: SaltedTxidHasher(); size_t operator()(const uint256 &txid) const { return SipHashUint256(k0, k1, txid); } }; /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions that * may be included in the next block. * * Transactions are added when they are seen on the network (or created by the * local node), but not all transactions seen are added to the pool. For * example, the following new transactions will not be added to the mempool: * - a transaction which doesn't meet the minimum fee requirements. * - a new transaction that double-spends an input of a transaction already in * the pool where the new transaction does not meet the Replace-By-Fee * requirements as defined in BIP 125. * - a non-standard transaction. * * CTxMemPool::mapTx, and CTxMemPoolEntry bookkeeping: * * mapTx is a boost::multi_index that sorts the mempool on 4 criteria: * - transaction hash * - feerate [we use max(feerate of tx, feerate of tx with all descendants)] * - time in mempool * - mining score (feerate modified by any fee deltas from * PrioritiseTransaction) * * Note: the term "descendant" refers to in-mempool transactions that depend on * this one, while "ancestor" refers to in-mempool transactions that a given * transaction depends on. * * In order for the feerate sort to remain correct, we must update transactions * in the mempool when new descendants arrive. To facilitate this, we track the * set of in-mempool direct parents and direct children in mapLinks. Within each * CTxMemPoolEntry, we track the size and fees of all descendants. * * Usually when a new transaction is added to the mempool, it has no in-mempool * children (because any such children would be an orphan). So in * addUnchecked(), we: * - update a new entry's setMemPoolParents to include all in-mempool parents * - update the new entry's direct parents to include the new tx as a child * - update all ancestors of the transaction to include the new tx's size/fee * * When a transaction is removed from the mempool, we must: * - update all in-mempool parents to not track the tx in setMemPoolChildren * - update all ancestors to not include the tx's size/fees in descendant state * - update all in-mempool children to not include it as a parent * * These happen in UpdateForRemoveFromMempool(). (Note that when removing a * transaction along with its descendants, we must calculate that set of * transactions to be removed before doing the removal, or else the mempool can * be in an inconsistent state where it's impossible to walk the ancestors of a * transaction.) * * In the event of a reorg, the assumption that a newly added tx has no * in-mempool children is false. In particular, the mempool is in an * inconsistent state while new transactions are being added, because there may * be descendant transactions of a tx coming from a disconnected block that are * unreachable from just looking at transactions in the mempool (the linking * transactions may also be in the disconnected block, waiting to be added). * Because of this, there's not much benefit in trying to search for in-mempool * children in addUnchecked(). Instead, in the special case of transactions * being added from a disconnected block, we require the caller to clean up the * state, to account for in-mempool, out-of-block descendants for all the * in-block transactions by calling UpdateTransactionsFromBlock(). Note that * until this is called, the mempool state is not consistent, and in particular * mapLinks may not be correct (and therefore functions like * CalculateMemPoolAncestors() and CalculateDescendants() that rely on them to * walk the mempool are not generally safe to use). * * Computational limits: * * Updating all in-mempool ancestors of a newly added transaction can be slow, * if no bound exists on how many in-mempool ancestors there may be. * CalculateMemPoolAncestors() takes configurable limits that are designed to * prevent these calculations from being too CPU intensive. * * Adding transactions from a disconnected block can be very time consuming, * because we don't have a way to limit the number of in-mempool descendants. To * bound CPU processing, we limit the amount of work we're willing to do to * properly update the descendant information for a tx being added from a * disconnected block. If we would exceed the limit, then we instead mark the * entry as "dirty", and set the feerate for sorting purposes to be equal the * feerate of the transaction without any descendants. */ class CTxMemPool { private: //!< Value n means that n times in 2^32 we check. uint32_t nCheckFrequency; unsigned int nTransactionsUpdated; CBlockPolicyEstimator *minerPolicyEstimator; //!< sum of all mempool tx's virtual sizes. uint64_t totalTxSize; //!< sum of dynamic memory usage of all the map elements (NOT the maps //! themselves) uint64_t cachedInnerUsage; mutable int64_t lastRollingFeeUpdate; mutable bool blockSinceLastRollingFeeBump; //!< minimum fee to get into the pool, decreases exponentially mutable double rollingMinimumFeeRate; void trackPackageRemoved(const CFeeRate &rate); public: // public only for testing static const int ROLLING_FEE_HALFLIFE = 60 * 60 * 12; typedef boost::multi_index_container< CTxMemPoolEntry, boost::multi_index::indexed_by< // sorted by txid boost::multi_index::hashed_unique< mempoolentry_txid, SaltedTxidHasher>, // sorted by fee rate boost::multi_index::ordered_non_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByDescendantScore>, // sorted by entry time boost::multi_index::ordered_non_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByEntryTime>, // sorted by score (for mining prioritization) boost::multi_index::ordered_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByScore>, // sorted by fee rate with ancestors boost::multi_index::ordered_non_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByAncestorFee>>> indexed_transaction_set; mutable CCriticalSection cs; indexed_transaction_set mapTx; typedef indexed_transaction_set::nth_index<0>::type::iterator txiter; //!< All tx hashes/entries in mapTx, in random order std::vector> vTxHashes; struct CompareIteratorByHash { bool operator()(const txiter &a, const txiter &b) const { return a->GetTx().GetId() < b->GetTx().GetId(); } }; typedef std::set setEntries; const setEntries &GetMemPoolParents(txiter entry) const; const setEntries &GetMemPoolChildren(txiter entry) const; private: typedef std::map cacheMap; struct TxLinks { setEntries parents; setEntries children; }; typedef std::map txlinksMap; txlinksMap mapLinks; void UpdateParent(txiter entry, txiter parent, bool add); void UpdateChild(txiter entry, txiter child, bool add); std::vector GetSortedDepthAndScore() const; public: indirectmap mapNextTx; std::map> mapDeltas; /** Create a new CTxMemPool. */ - CTxMemPool(const CFeeRate &_minReasonableRelayFee); + CTxMemPool(); ~CTxMemPool(); /** * If sanity-checking is turned on, check makes sure the pool is consistent * (does not contain two transactions that spend the same inputs, all inputs * are in the mapNextTx array). If sanity-checking is turned off, check does * nothing. */ void check(const CCoinsViewCache *pcoins) const; void setSanityCheck(double dFrequency = 1.0) { nCheckFrequency = dFrequency * 4294967295.0; } // addUnchecked must updated state for all ancestors of a given transaction, // to track size/count of descendant transactions. First version of // addUnchecked can be used to have it call CalculateMemPoolAncestors(), and // then invoke the second version. bool addUnchecked(const uint256 &hash, const CTxMemPoolEntry &entry, bool validFeeEstimate = true); bool addUnchecked(const uint256 &hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool validFeeEstimate = true); void removeRecursive( const CTransaction &tx, MemPoolRemovalReason reason = MemPoolRemovalReason::UNKNOWN); void removeForReorg(const Config &config, const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags); void removeConflicts(const CTransaction &tx); void removeForBlock(const std::vector &vtx, unsigned int nBlockHeight); void clear(); // lock free void _clear(); bool CompareDepthAndScore(const uint256 &hasha, const uint256 &hashb); void queryHashes(std::vector &vtxid); bool isSpent(const COutPoint &outpoint); unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); /** * Check that none of this transactions inputs are in the mempool, and thus * the tx is not dependent on other mempool transactions to be included in a * block. */ bool HasNoInputsOf(const CTransaction &tx) const; /** Affect CreateNewBlock prioritisation of transactions */ void PrioritiseTransaction(const uint256 hash, const std::string strHash, double dPriorityDelta, const Amount nFeeDelta); void ApplyDeltas(const uint256 hash, double &dPriorityDelta, Amount &nFeeDelta) const; void ClearPrioritisation(const uint256 hash); public: /** * Remove a set of transactions from the mempool. If a transaction is in * this set, then all in-mempool descendants must also be in the set, unless * this transaction is being removed for being in a block. Set * updateDescendants to true when removing a tx that was in a block, so that * any in-mempool descendants have their ancestor state updated. */ void RemoveStaged(setEntries &stage, bool updateDescendants, MemPoolRemovalReason reason = MemPoolRemovalReason::UNKNOWN); /** * When adding transactions from a disconnected block back to the mempool, * new mempool entries may have children in the mempool (which is generally * not the case when otherwise adding transactions). * UpdateTransactionsFromBlock() will find child transactions and update the * descendant state for each transaction in hashesToUpdate (excluding any * child transactions present in hashesToUpdate, which are already accounted * for). Note: hashesToUpdate should be the set of transactions from the * disconnected block that have been accepted back into the mempool. */ void UpdateTransactionsFromBlock(const std::vector &hashesToUpdate); /** * Try to calculate all in-mempool ancestors of entry. * (these are all calculated including the tx itself) * limitAncestorCount = max number of ancestors * limitAncestorSize = max size of ancestors * limitDescendantCount = max number of descendants any ancestor can have * limitDescendantSize = max size of descendants any ancestor can have * errString = populated with error reason if any limits are hit * fSearchForParents = whether to search a tx's vin for in-mempool parents, * or look up parents from mapLinks. Must be true for entries not in the * mempool */ bool CalculateMemPoolAncestors( const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents = true) const; /** * Populate setDescendants with all in-mempool descendants of hash. * Assumes that setDescendants includes all in-mempool descendants of * anything already in it. */ void CalculateDescendants(txiter it, setEntries &setDescendants); /** * The minimum fee to get into the mempool, which may itself not be enough * for larger-sized transactions. The incrementalRelayFee policy variable is * used to bound the time it takes the fee rate to go back down all the way * to 0. When the feerate would otherwise be half of this, it is set to 0 * instead. */ CFeeRate GetMinFee(size_t sizelimit) const; /** * Remove transactions from the mempool until its dynamic size is <= * sizelimit. pvNoSpendsRemaining, if set, will be populated with the list * of outpoints which are not in mempool which no longer have any spends in * this mempool. */ void TrimToSize(size_t sizelimit, std::vector *pvNoSpendsRemaining = nullptr); /** Expire all transaction (and their dependencies) in the mempool older * than time. Return the number of removed transactions. */ int Expire(int64_t time); /** Returns false if the transaction is in the mempool and not within the * chain limit specified. */ bool TransactionWithinChainLimit(const uint256 &txid, size_t chainLimit) const; unsigned long size() { LOCK(cs); return mapTx.size(); } uint64_t GetTotalTxSize() { LOCK(cs); return totalTxSize; } bool exists(uint256 hash) const { LOCK(cs); return mapTx.count(hash) != 0; } bool exists(const COutPoint &outpoint) const { LOCK(cs); auto it = mapTx.find(outpoint.hash); return it != mapTx.end() && outpoint.n < it->GetTx().vout.size(); } CTransactionRef get(const uint256 &hash) const; TxMempoolInfo info(const uint256 &hash) const; std::vector infoAll() const; /** * Estimate fee rate needed to get into the next nBlocks. If no answer can * be given at nBlocks, return an estimate at the lowest number of blocks * where one can be given. */ CFeeRate estimateSmartFee(int nBlocks, int *answerFoundAtBlocks = nullptr) const; /** Estimate fee rate needed to get into the next nBlocks */ CFeeRate estimateFee(int nBlocks) const; /** * Estimate priority needed to get into the next nBlocks. If no answer can * be given at nBlocks, return an estimate at the lowest number of blocks * where one can be given. */ double estimateSmartPriority(int nBlocks, int *answerFoundAtBlocks = nullptr) const; /** Write/Read estimates to disk */ bool WriteFeeEstimates(CAutoFile &fileout) const; bool ReadFeeEstimates(CAutoFile &filein); size_t DynamicMemoryUsage() const; boost::signals2::signal NotifyEntryAdded; boost::signals2::signal NotifyEntryRemoved; private: /** * UpdateForDescendants is used by UpdateTransactionsFromBlock to update the * descendants for a single transaction that has been added to the mempool * but may have child transactions in the mempool, eg during a chain reorg. * setExclude is the set of descendant transactions in the mempool that must * not be accounted for (because any descendants in setExclude were added to * the mempool after the transaction being updated and hence their state is * already reflected in the parent state). * * cachedDescendants will be updated with the descendants of the transaction * being updated, so that future invocations don't need to walk the same * transaction again, if encountered in another transaction chain. */ void UpdateForDescendants(txiter updateIt, cacheMap &cachedDescendants, const std::set &setExclude); /** Update ancestors of hash to add/remove it as a descendant transaction. */ void UpdateAncestorsOf(bool add, txiter hash, setEntries &setAncestors); /** Set ancestor state for an entry */ void UpdateEntryForAncestors(txiter it, const setEntries &setAncestors); /** * For each transaction being removed, update ancestors and any direct * children. If updateDescendants is true, then also update in-mempool * descendants' ancestor state. */ void UpdateForRemoveFromMempool(const setEntries &entriesToRemove, bool updateDescendants); /** Sever link between specified transaction and direct children. */ void UpdateChildrenForRemoval(txiter entry); /** * Before calling removeUnchecked for a given transaction, * UpdateForRemoveFromMempool must be called on the entire (dependent) set * of transactions being removed at the same time. We use each * CTxMemPoolEntry's setMemPoolParents in order to walk ancestors of a given * transaction that is removed, so we can't remove intermediate transactions * in a chain before we've updated all the state for the removal. */ void removeUnchecked( txiter entry, MemPoolRemovalReason reason = MemPoolRemovalReason::UNKNOWN); }; /** * CCoinsView that brings transactions from a memorypool into view. * It does not check for spendings by memory pool transactions. */ class CCoinsViewMemPool : public CCoinsViewBacked { protected: const CTxMemPool &mempool; public: CCoinsViewMemPool(CCoinsView *baseIn, const CTxMemPool &mempoolIn); bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; bool HaveCoin(const COutPoint &outpoint) const override; }; // We want to sort transactions by coin age priority typedef std::pair TxCoinAgePriority; struct TxCoinAgePriorityCompare { bool operator()(const TxCoinAgePriority &a, const TxCoinAgePriority &b) { if (a.first == b.first) { // Reverse order to make sort less than return CompareTxMemPoolEntryByScore()(*(b.second), *(a.second)); } return a.first < b.first; } }; /** * DisconnectedBlockTransactions * During the reorg, it's desirable to re-add previously confirmed transactions * to the mempool, so that anything not re-confirmed in the new chain is * available to be mined. However, it's more efficient to wait until the reorg * is complete and process all still-unconfirmed transactions at that time, * since we expect most confirmed transactions to (typically) still be * confirmed in the new chain, and re-accepting to the memory pool is expensive * (and therefore better to not do in the middle of reorg-processing). * Instead, store the disconnected transactions (in order!) as we go, remove any * that are included in blocks in the new chain, and then process the remaining * still-unconfirmed transactions at the end. */ // multi_index tag names struct txid_index {}; struct insertion_order {}; struct DisconnectedBlockTransactions { typedef boost::multi_index_container< CTransactionRef, boost::multi_index::indexed_by< // sorted by txid boost::multi_index::hashed_unique< boost::multi_index::tag, mempoolentry_txid, SaltedTxidHasher>, // sorted by order in the blockchain boost::multi_index::sequenced< boost::multi_index::tag>>> indexed_disconnected_transactions; // It's almost certainly a logic bug if we don't clear out queuedTx before // destruction, as we add to it while disconnecting blocks, and then we // need to re-process remaining transactions to ensure mempool consistency. // For now, assert() that we've emptied out this object on destruction. // This assert() can always be removed if the reorg-processing code were // to be refactored such that this assumption is no longer true (for // instance if there was some other way we cleaned up the mempool after a // reorg, besides draining this object). ~DisconnectedBlockTransactions() { assert(queuedTx.empty()); } indexed_disconnected_transactions queuedTx; uint64_t cachedInnerUsage = 0; // Estimate the overhead of queuedTx to be 6 pointers + an allocation, as // no exact formula for boost::multi_index_contained is implemented. size_t DynamicMemoryUsage() const { return memusage::MallocUsage(sizeof(CTransactionRef) + 6 * sizeof(void *)) * queuedTx.size() + cachedInnerUsage; } void addTransaction(const CTransactionRef &tx) { queuedTx.insert(tx); cachedInnerUsage += RecursiveDynamicUsage(tx); } // Remove entries based on txid_index, and update memory usage. void removeForBlock(const std::vector &vtx) { // Short-circuit in the common case of a block being added to the tip if (queuedTx.empty()) { return; } for (auto const &tx : vtx) { auto it = queuedTx.find(tx->GetHash()); if (it != queuedTx.end()) { cachedInnerUsage -= RecursiveDynamicUsage(*it); queuedTx.erase(it); } } } // Remove an entry by insertion_order index, and update memory usage. void removeEntry(indexed_disconnected_transactions::index< insertion_order>::type::iterator entry) { cachedInnerUsage -= RecursiveDynamicUsage(*entry); queuedTx.get().erase(entry); } void clear() { cachedInnerUsage = 0; queuedTx.clear(); } }; #endif // BITCOIN_TXMEMPOOL_H diff --git a/src/validation.cpp b/src/validation.cpp index c746e8cabc..0b54db396b 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1,5462 +1,5462 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Copyright (c) 2017-2018 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "validation.h" #include "arith_uint256.h" #include "chainparams.h" #include "checkpoints.h" #include "checkqueue.h" #include "config.h" #include "consensus/consensus.h" #include "consensus/merkle.h" #include "consensus/validation.h" #include "fs.h" #include "hash.h" #include "init.h" #include "policy/fees.h" #include "policy/policy.h" #include "pow.h" #include "primitives/block.h" #include "primitives/transaction.h" #include "random.h" #include "script/script.h" #include "script/scriptcache.h" #include "script/sigcache.h" #include "script/standard.h" #include "timedata.h" #include "tinyformat.h" #include "txdb.h" #include "txmempool.h" #include "ui_interface.h" #include "undo.h" #include "util.h" #include "utilmoneystr.h" #include "utilstrencodings.h" #include "validationinterface.h" #include "versionbits.h" #include "warnings.h" #include #include #include #include #include #include #include #include #if defined(NDEBUG) #error "Bitcoin cannot be compiled without assertions." #endif /** * Global state */ CCriticalSection cs_main; BlockMap mapBlockIndex; CChain chainActive; CBlockIndex *pindexBestHeader = nullptr; CWaitableCriticalSection csBestBlock; CConditionVariable cvBlockChange; int nScriptCheckThreads = 0; std::atomic_bool fImporting(false); bool fReindex = false; bool fTxIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; bool fRequireStandard = true; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; size_t nCoinCacheUsage = 5000 * 300; uint64_t nPruneTarget = 0; int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE; uint256 hashAssumeValid; arith_uint256 nMinimumChainWork; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); Amount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; -CTxMemPool mempool(::minRelayTxFee); +CTxMemPool mempool; static void CheckBlockIndex(const Consensus::Params &consensusParams); /** Constant stuff for coinbase transactions we create: */ CScript COINBASE_FLAGS; const std::string strMessageMagic = "Bitcoin Signed Message:\n"; // Internal stuff namespace { struct CBlockIndexWorkComparator { bool operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { // First sort by most total work, ... if (pa->nChainWork > pb->nChainWork) return false; if (pa->nChainWork < pb->nChainWork) return true; // ... then by earliest time received, ... if (pa->nSequenceId < pb->nSequenceId) return false; if (pa->nSequenceId > pb->nSequenceId) return true; // Use pointer address as tie breaker (should only happen with blocks // loaded from disk, as those all have id 0). if (pa < pb) return false; if (pa > pb) return true; // Identical blocks. return false; } }; CBlockIndex *pindexBestInvalid; /** * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself * and all ancestors) and as good as our current tip or better. Entries may be * failed, though, and pruning nodes may be missing the data for the block. */ std::set setBlockIndexCandidates; /** * All pairs A->B, where A (or one of its ancestors) misses transactions, but B * has transactions. Pruned nodes may have entries where B is missing data. */ std::multimap mapBlocksUnlinked; CCriticalSection cs_LastBlockFile; std::vector vinfoBlockFile; int nLastBlockFile = 0; /** * Global flag to indicate we should check to see if there are block/undo files * that should be deleted. Set on startup or if we allocate more file space when * we're in prune mode. */ bool fCheckForPruning = false; /** * Every received block is assigned a unique and increasing identifier, so we * know which one to give priority in case of a fork. */ CCriticalSection cs_nBlockSequenceId; /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ int32_t nBlockSequenceId = 1; /** Decreasing counter (used by subsequent preciousblock calls). */ int32_t nBlockReverseSequenceId = -1; /** chainwork for the last block that preciousblock has been applied to. */ arith_uint256 nLastPreciousChainwork = 0; /** Dirty block index entries. */ std::set setDirtyBlockIndex; /** Dirty block file entries. */ std::set setDirtyFileInfo; } // namespace CBlockIndex *FindForkInGlobalIndex(const CChain &chain, const CBlockLocator &locator) { // Find the first block the caller has in the main chain for (const uint256 &hash : locator.vHave) { BlockMap::iterator mi = mapBlockIndex.find(hash); if (mi != mapBlockIndex.end()) { CBlockIndex *pindex = (*mi).second; if (chain.Contains(pindex)) return pindex; if (pindex->GetAncestor(chain.Height()) == chain.Tip()) { return chain.Tip(); } } } return chain.Genesis(); } CCoinsViewCache *pcoinsTip = nullptr; CBlockTreeDB *pblocktree = nullptr; enum FlushStateMode { FLUSH_STATE_NONE, FLUSH_STATE_IF_NEEDED, FLUSH_STATE_PERIODIC, FLUSH_STATE_ALWAYS }; // See definition for documentation static bool FlushStateToDisk(const CChainParams &chainParams, CValidationState &state, FlushStateMode mode, int nManualPruneHeight = 0); static void FindFilesToPruneManual(std::set &setFilesToPrune, int nManualPruneHeight); static void FindFilesToPrune(std::set &setFilesToPrune, uint64_t nPruneAfterHeight); static FILE *OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false); static uint32_t GetBlockScriptFlags(const Config &config, const CBlockIndex *pChainTip); static bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) { if (tx.nLockTime == 0) { return true; } int64_t lockTime = tx.nLockTime; int64_t lockTimeLimit = (lockTime < LOCKTIME_THRESHOLD) ? nBlockHeight : nBlockTime; if (lockTime < lockTimeLimit) { return true; } for (const auto &txin : tx.vin) { if (txin.nSequence != CTxIn::SEQUENCE_FINAL) { return false; } } return true; } /** * Calculates the block height and previous block's median time past at * which the transaction will be considered final in the context of BIP 68. * Also removes from the vector of input heights any entries which did not * correspond to sequence locked inputs as they do not affect the calculation. */ static std::pair CalculateSequenceLocks(const CTransaction &tx, int flags, std::vector *prevHeights, const CBlockIndex &block) { assert(prevHeights->size() == tx.vin.size()); // Will be set to the equivalent height- and time-based nLockTime // values that would be necessary to satisfy all relative lock- // time constraints given our view of block chain history. // The semantics of nLockTime are the last invalid height/time, so // use -1 to have the effect of any height or time being valid. int nMinHeight = -1; int64_t nMinTime = -1; // tx.nVersion is signed integer so requires cast to unsigned otherwise // we would be doing a signed comparison and half the range of nVersion // wouldn't support BIP 68. bool fEnforceBIP68 = static_cast(tx.nVersion) >= 2 && flags & LOCKTIME_VERIFY_SEQUENCE; // Do not enforce sequence numbers as a relative lock time // unless we have been instructed to if (!fEnforceBIP68) { return std::make_pair(nMinHeight, nMinTime); } for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { const CTxIn &txin = tx.vin[txinIndex]; // Sequence numbers with the most significant bit set are not // treated as relative lock-times, nor are they given any // consensus-enforced meaning at this point. if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_DISABLE_FLAG) { // The height of this input is not relevant for sequence locks (*prevHeights)[txinIndex] = 0; continue; } int nCoinHeight = (*prevHeights)[txinIndex]; if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG) { int64_t nCoinTime = block.GetAncestor(std::max(nCoinHeight - 1, 0)) ->GetMedianTimePast(); // NOTE: Subtract 1 to maintain nLockTime semantics. // BIP 68 relative lock times have the semantics of calculating the // first block or time at which the transaction would be valid. When // calculating the effective block time or height for the entire // transaction, we switch to using the semantics of nLockTime which // is the last invalid block time or height. Thus we subtract 1 from // the calculated time or height. // Time-based relative lock-times are measured from the smallest // allowed timestamp of the block containing the txout being spent, // which is the median time past of the block prior. nMinTime = std::max( nMinTime, nCoinTime + (int64_t)((txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) << CTxIn::SEQUENCE_LOCKTIME_GRANULARITY) - 1); } else { nMinHeight = std::max( nMinHeight, nCoinHeight + (int)(txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) - 1); } } return std::make_pair(nMinHeight, nMinTime); } static bool EvaluateSequenceLocks(const CBlockIndex &block, std::pair lockPair) { assert(block.pprev); int64_t nBlockTime = block.pprev->GetMedianTimePast(); if (lockPair.first >= block.nHeight || lockPair.second >= nBlockTime) return false; return true; } bool SequenceLocks(const CTransaction &tx, int flags, std::vector *prevHeights, const CBlockIndex &block) { return EvaluateSequenceLocks( block, CalculateSequenceLocks(tx, flags, prevHeights, block)); } bool TestLockPointValidity(const LockPoints *lp) { AssertLockHeld(cs_main); assert(lp); // If there are relative lock times then the maxInputBlock will be set // If there are no relative lock times, the LockPoints don't depend on the // chain if (lp->maxInputBlock) { // Check whether chainActive is an extension of the block at which the // LockPoints // calculation was valid. If not LockPoints are no longer valid if (!chainActive.Contains(lp->maxInputBlock)) { return false; } } // LockPoints still valid return true; } bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints *lp, bool useExistingLockPoints) { AssertLockHeld(cs_main); AssertLockHeld(mempool.cs); CBlockIndex *tip = chainActive.Tip(); CBlockIndex index; index.pprev = tip; // CheckSequenceLocks() uses chainActive.Height()+1 to evaluate height based // locks because when SequenceLocks() is called within ConnectBlock(), the // height of the block *being* evaluated is what is used. Thus if we want to // know if a transaction can be part of the *next* block, we need to use one // more than chainActive.Height() index.nHeight = tip->nHeight + 1; std::pair lockPair; if (useExistingLockPoints) { assert(lp); lockPair.first = lp->height; lockPair.second = lp->time; } else { // pcoinsTip contains the UTXO set for chainActive.Tip() CCoinsViewMemPool viewMemPool(pcoinsTip, mempool); std::vector prevheights; prevheights.resize(tx.vin.size()); for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { const CTxIn &txin = tx.vin[txinIndex]; Coin coin; if (!viewMemPool.GetCoin(txin.prevout, coin)) { return error("%s: Missing input", __func__); } if (coin.GetHeight() == MEMPOOL_HEIGHT) { // Assume all mempool transaction confirm in the next block prevheights[txinIndex] = tip->nHeight + 1; } else { prevheights[txinIndex] = coin.GetHeight(); } } lockPair = CalculateSequenceLocks(tx, flags, &prevheights, index); if (lp) { lp->height = lockPair.first; lp->time = lockPair.second; // Also store the hash of the block with the highest height of all // the blocks which have sequence locked prevouts. This hash needs // to still be on the chain for these LockPoint calculations to be // valid. // Note: It is impossible to correctly calculate a maxInputBlock if // any of the sequence locked inputs depend on unconfirmed txs, // except in the special case where the relative lock time/height is // 0, which is equivalent to no sequence lock. Since we assume input // height of tip+1 for mempool txs and test the resulting lockPair // from CalculateSequenceLocks against tip+1. We know // EvaluateSequenceLocks will fail if there was a non-zero sequence // lock on a mempool input, so we can use the return value of // CheckSequenceLocks to indicate the LockPoints validity int maxInputHeight = 0; for (int height : prevheights) { // Can ignore mempool inputs since we'll fail if they had // non-zero locks if (height != tip->nHeight + 1) { maxInputHeight = std::max(maxInputHeight, height); } } lp->maxInputBlock = tip->GetAncestor(maxInputHeight); } } return EvaluateSequenceLocks(index, lockPair); } uint64_t GetSigOpCountWithoutP2SH(const CTransaction &tx) { uint64_t nSigOps = 0; for (const auto &txin : tx.vin) { nSigOps += txin.scriptSig.GetSigOpCount(false); } for (const auto &txout : tx.vout) { nSigOps += txout.scriptPubKey.GetSigOpCount(false); } return nSigOps; } uint64_t GetP2SHSigOpCount(const CTransaction &tx, const CCoinsViewCache &inputs) { if (tx.IsCoinBase()) { return 0; } uint64_t nSigOps = 0; for (auto &i : tx.vin) { const CTxOut &prevout = inputs.GetOutputFor(i); if (prevout.scriptPubKey.IsPayToScriptHash()) { nSigOps += prevout.scriptPubKey.GetSigOpCount(i.scriptSig); } } return nSigOps; } uint64_t GetTransactionSigOpCount(const CTransaction &tx, const CCoinsViewCache &inputs, int flags) { uint64_t nSigOps = GetSigOpCountWithoutP2SH(tx); if (tx.IsCoinBase()) { return nSigOps; } if (flags & SCRIPT_VERIFY_P2SH) { nSigOps += GetP2SHSigOpCount(tx, inputs); } return nSigOps; } static bool CheckTransactionCommon(const CTransaction &tx, CValidationState &state, bool fCheckDuplicateInputs) { // Basic checks that don't depend on any context if (tx.vin.empty()) { return state.DoS(10, false, REJECT_INVALID, "bad-txns-vin-empty"); } if (tx.vout.empty()) { return state.DoS(10, false, REJECT_INVALID, "bad-txns-vout-empty"); } // Size limit if (::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION) > MAX_TX_SIZE) { return state.DoS(100, false, REJECT_INVALID, "bad-txns-oversize"); } // Check for negative or overflow output values Amount nValueOut(0); for (const auto &txout : tx.vout) { if (txout.nValue < Amount(0)) { return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-negative"); } if (txout.nValue > MAX_MONEY) { return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-toolarge"); } nValueOut += txout.nValue; if (!MoneyRange(nValueOut)) { return state.DoS(100, false, REJECT_INVALID, "bad-txns-txouttotal-toolarge"); } } if (GetSigOpCountWithoutP2SH(tx) > MAX_TX_SIGOPS_COUNT) { return state.DoS(100, false, REJECT_INVALID, "bad-txn-sigops"); } // Check for duplicate inputs - note that this check is slow so we skip it // in CheckBlock if (fCheckDuplicateInputs) { std::set vInOutPoints; for (const auto &txin : tx.vin) { if (!vInOutPoints.insert(txin.prevout).second) { return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); } } } return true; } bool CheckCoinbase(const CTransaction &tx, CValidationState &state, bool fCheckDuplicateInputs) { if (!tx.IsCoinBase()) { return state.DoS(100, false, REJECT_INVALID, "bad-cb-missing", false, "first tx is not coinbase"); } if (!CheckTransactionCommon(tx, state, fCheckDuplicateInputs)) { // CheckTransactionCommon fill in the state. return false; } if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) { return state.DoS(100, false, REJECT_INVALID, "bad-cb-length"); } return true; } bool CheckRegularTransaction(const CTransaction &tx, CValidationState &state, bool fCheckDuplicateInputs) { if (tx.IsCoinBase()) { return state.DoS(100, false, REJECT_INVALID, "bad-tx-coinbase"); } if (!CheckTransactionCommon(tx, state, fCheckDuplicateInputs)) { // CheckTransactionCommon fill in the state. return false; } for (const auto &txin : tx.vin) { if (txin.prevout.IsNull()) { return state.DoS(10, false, REJECT_INVALID, "bad-txns-prevout-null"); } } return true; } static void LimitMempoolSize(CTxMemPool &pool, size_t limit, unsigned long age) { int expired = pool.Expire(GetTime() - age); if (expired != 0) { LogPrint(BCLog::MEMPOOL, "Expired %i transactions from the memory pool\n", expired); } std::vector vNoSpendsRemaining; pool.TrimToSize(limit, &vNoSpendsRemaining); for (const COutPoint &removed : vNoSpendsRemaining) { pcoinsTip->Uncache(removed); } } /** Convert CValidationState to a human-readable message for logging */ std::string FormatStateMessage(const CValidationState &state) { return strprintf( "%s%s (code %i)", state.GetRejectReason(), state.GetDebugMessage().empty() ? "" : ", " + state.GetDebugMessage(), state.GetRejectCode()); } static bool IsCurrentForFeeEstimation() { AssertLockHeld(cs_main); if (IsInitialBlockDownload()) { return false; } if (chainActive.Tip()->GetBlockTime() < (GetTime() - MAX_FEE_ESTIMATION_TIP_AGE)) { return false; } if (chainActive.Height() < pindexBestHeader->nHeight - 1) { return false; } return true; } static bool IsUAHFenabled(const Config &config, int nHeight) { return nHeight >= config.GetChainParams().GetConsensus().uahfHeight; } bool IsUAHFenabled(const Config &config, const CBlockIndex *pindexPrev) { if (pindexPrev == nullptr) { return false; } return IsUAHFenabled(config, pindexPrev->nHeight); } static bool IsDAAEnabled(const Config &config, int nHeight) { return nHeight >= config.GetChainParams().GetConsensus().daaHeight; } bool IsDAAEnabled(const Config &config, const CBlockIndex *pindexPrev) { if (pindexPrev == nullptr) { return false; } return IsDAAEnabled(config, pindexPrev->nHeight); } static bool IsMonolithEnabled(const Config &config, int64_t nMedianTimePast) { return nMedianTimePast >= gArgs.GetArg( "-monolithactivationtime", config.GetChainParams().GetConsensus().monolithActivationTime); } bool IsMonolithEnabled(const Config &config, const CBlockIndex *pindexPrev) { if (pindexPrev == nullptr) { return false; } return IsMonolithEnabled(config, pindexPrev->GetMedianTimePast()); } static bool IsReplayProtectionEnabled(const Config &config, int64_t nMedianTimePast) { return nMedianTimePast >= gArgs.GetArg("-replayprotectionactivationtime", config.GetChainParams() .GetConsensus() .magneticAnomalyActivationTime); } static bool IsReplayProtectionEnabled(const Config &config, const CBlockIndex *pindexPrev) { if (pindexPrev == nullptr) { return false; } return IsReplayProtectionEnabled(config, pindexPrev->GetMedianTimePast()); } static bool IsReplayProtectionEnabledForCurrentBlock(const Config &config) { AssertLockHeld(cs_main); return IsReplayProtectionEnabled(config, chainActive.Tip()); } /** * Make mempool consistent after a reorg, by re-adding or recursively erasing * disconnected block transactions from the mempool, and also removing any other * transactions from the mempool that are no longer valid given the new * tip/height. * * Note: we assume that disconnectpool only contains transactions that are NOT * confirmed in the current chain nor already in the mempool (otherwise, * in-mempool descendants of such transactions would be removed). * * Passing fAddToMempool=false will skip trying to add the transactions back, * and instead just erase from the mempool as needed. */ void UpdateMempoolForReorg(const Config &config, DisconnectedBlockTransactions &disconnectpool, bool fAddToMempool) { AssertLockHeld(cs_main); std::vector vHashUpdate; // disconnectpool's insertion_order index sorts the entries from oldest to // newest, but the oldest entry will be the last tx from the latest mined // block that was disconnected. // Iterate disconnectpool in reverse, so that we add transactions back to // the mempool starting with the earliest transaction that had been // previously seen in a block. auto it = disconnectpool.queuedTx.get().rbegin(); while (it != disconnectpool.queuedTx.get().rend()) { // ignore validation errors in resurrected transactions CValidationState stateDummy; if (!fAddToMempool || (*it)->IsCoinBase() || !AcceptToMemoryPool(config, mempool, stateDummy, *it, false, nullptr, true)) { // If the transaction doesn't make it in to the mempool, remove any // transactions that depend on it (which would now be orphans). mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); } else if (mempool.exists((*it)->GetId())) { vHashUpdate.push_back((*it)->GetId()); } ++it; } disconnectpool.queuedTx.clear(); // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have // no in-mempool children, which is generally not true when adding // previously-confirmed transactions back to the mempool. // UpdateTransactionsFromBlock finds descendants of any transactions in the // disconnectpool that were added back and cleans up the mempool state. mempool.UpdateTransactionsFromBlock(vHashUpdate); // We also need to remove any now-immature transactions mempool.removeForReorg(config, pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); // Re-limit mempool size, in case we added any transactions LimitMempoolSize( mempool, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); } // Used to avoid mempool polluting consensus critical paths if CCoinsViewMempool // were somehow broken and returning the wrong scriptPubKeys static bool CheckInputsFromMempoolAndCache(const CTransaction &tx, CValidationState &state, const CCoinsViewCache &view, CTxMemPool &pool, const uint32_t flags, bool cacheSigStore, PrecomputedTransactionData &txdata) { AssertLockHeld(cs_main); // pool.cs should be locked already, but go ahead and re-take the lock here // to enforce that mempool doesn't change between when we check the view and // when we actually call through to CheckInputs LOCK(pool.cs); assert(!tx.IsCoinBase()); for (const CTxIn &txin : tx.vin) { const Coin &coin = view.AccessCoin(txin.prevout); // At this point we haven't actually checked if the coins are all // available (or shouldn't assume we have, since CheckInputs does). So // we just return failure if the inputs are not available here, and then // only have to check equivalence for available inputs. if (coin.IsSpent()) { return false; } const CTransactionRef &txFrom = pool.get(txin.prevout.hash); if (txFrom) { assert(txFrom->GetHash() == txin.prevout.hash); assert(txFrom->vout.size() > txin.prevout.n); assert(txFrom->vout[txin.prevout.n] == coin.GetTxOut()); } else { const Coin &coinFromDisk = pcoinsTip->AccessCoin(txin.prevout); assert(!coinFromDisk.IsSpent()); assert(coinFromDisk.GetTxOut() == coin.GetTxOut()); } } return CheckInputs(tx, state, view, true, flags, cacheSigStore, true, txdata); } static bool AcceptToMemoryPoolWorker( const Config &config, CTxMemPool &pool, CValidationState &state, const CTransactionRef &ptx, bool fLimitFree, bool *pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const Amount nAbsurdFee, std::vector &coins_to_uncache) { AssertLockHeld(cs_main); const CTransaction &tx = *ptx; const uint256 txid = tx.GetId(); if (pfMissingInputs) { *pfMissingInputs = false; } // Coinbase is only valid in a block, not as a loose transaction. if (!CheckRegularTransaction(tx, state, true)) { // state filled in by CheckRegularTransaction. return false; } // After the May, 15 hard fork, we start accepting larger op_return. const bool hasMonolith = IsMonolithEnabled(config, chainActive.Tip()); // Rather not work on nonstandard transactions (unless -testnet/-regtest) std::string reason; if (fRequireStandard && !IsStandardTx(tx, reason, hasMonolith)) { return state.DoS(0, false, REJECT_NONSTANDARD, reason); } // Only accept nLockTime-using transactions that can be mined in the next // block; we don't want our mempool filled up with transactions that can't // be mined yet. CValidationState ctxState; if (!ContextualCheckTransactionForCurrentBlock( config, tx, ctxState, STANDARD_LOCKTIME_VERIFY_FLAGS)) { // We copy the state from a dummy to ensure we don't increase the // ban score of peer for transaction that could be valid in the future. return state.DoS( 0, false, REJECT_NONSTANDARD, ctxState.GetRejectReason(), ctxState.CorruptionPossible(), ctxState.GetDebugMessage()); } // Is it already in the memory pool? if (pool.exists(txid)) { return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-in-mempool"); } // Check for conflicts with in-memory transactions { // Protect pool.mapNextTx LOCK(pool.cs); for (const CTxIn &txin : tx.vin) { auto itConflicting = pool.mapNextTx.find(txin.prevout); if (itConflicting != pool.mapNextTx.end()) { // Disable replacement feature for good return state.Invalid(false, REJECT_CONFLICT, "txn-mempool-conflict"); } } } { CCoinsView dummy; CCoinsViewCache view(&dummy); Amount nValueIn(0); LockPoints lp; { LOCK(pool.cs); CCoinsViewMemPool viewMemPool(pcoinsTip, pool); view.SetBackend(viewMemPool); // Do we already have it? for (size_t out = 0; out < tx.vout.size(); out++) { COutPoint outpoint(txid, out); bool had_coin_in_cache = pcoinsTip->HaveCoinInCache(outpoint); if (view.HaveCoin(outpoint)) { if (!had_coin_in_cache) { coins_to_uncache.push_back(outpoint); } return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); } } // Do all inputs exist? for (const CTxIn txin : tx.vin) { if (!pcoinsTip->HaveCoinInCache(txin.prevout)) { coins_to_uncache.push_back(txin.prevout); } if (!view.HaveCoin(txin.prevout)) { if (pfMissingInputs) { *pfMissingInputs = true; } // fMissingInputs and !state.IsInvalid() is used to detect // this condition, don't set state.Invalid() return false; } } // Are the actual inputs available? if (!view.HaveInputs(tx)) { return state.Invalid(false, REJECT_DUPLICATE, "bad-txns-inputs-spent"); } // Bring the best block into scope. view.GetBestBlock(); nValueIn = view.GetValueIn(tx); // We have all inputs cached now, so switch back to dummy, so we // don't need to keep lock on mempool. view.SetBackend(dummy); // Only accept BIP68 sequence locked transactions that can be mined // in the next block; we don't want our mempool filled up with // transactions that can't be mined yet. Must keep pool.cs for this // unless we change CheckSequenceLocks to take a CoinsViewCache // instead of create its own. if (!CheckSequenceLocks(tx, STANDARD_LOCKTIME_VERIFY_FLAGS, &lp)) { return state.DoS(0, false, REJECT_NONSTANDARD, "non-BIP68-final"); } } // Check for non-standard pay-to-script-hash in inputs if (fRequireStandard && !AreInputsStandard(tx, view)) { return state.Invalid(false, REJECT_NONSTANDARD, "bad-txns-nonstandard-inputs"); } int64_t nSigOpsCount = GetTransactionSigOpCount(tx, view, STANDARD_SCRIPT_VERIFY_FLAGS); Amount nValueOut = tx.GetValueOut(); Amount nFees = nValueIn - nValueOut; // nModifiedFees includes any fee deltas from PrioritiseTransaction Amount nModifiedFees = nFees; double nPriorityDummy = 0; pool.ApplyDeltas(txid, nPriorityDummy, nModifiedFees); Amount inChainInputValue; double dPriority = view.GetPriority(tx, chainActive.Height(), inChainInputValue); // Keep track of transactions that spend a coinbase, which we re-scan // during reorgs to ensure COINBASE_MATURITY is still met. bool fSpendsCoinbase = false; for (const CTxIn &txin : tx.vin) { const Coin &coin = view.AccessCoin(txin.prevout); if (coin.IsCoinBase()) { fSpendsCoinbase = true; break; } } CTxMemPoolEntry entry(ptx, nFees, nAcceptTime, dPriority, chainActive.Height(), inChainInputValue, fSpendsCoinbase, nSigOpsCount, lp); unsigned int nSize = entry.GetTxSize(); // Check that the transaction doesn't have an excessive number of // sigops, making it impossible to mine. Since the coinbase transaction // itself can contain sigops MAX_STANDARD_TX_SIGOPS is less than // MAX_BLOCK_SIGOPS_PER_MB; we still consider this an invalid rather // than merely non-standard transaction. if (nSigOpsCount > MAX_STANDARD_TX_SIGOPS) { return state.DoS(0, false, REJECT_NONSTANDARD, "bad-txns-too-many-sigops", false, strprintf("%d", nSigOpsCount)); } Amount mempoolRejectFee = pool.GetMinFee( gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000) .GetFee(nSize); if (mempoolRejectFee > Amount(0) && nModifiedFees < mempoolRejectFee) { return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool min fee not met", false, strprintf("%d < %d", nFees, mempoolRejectFee)); } if (gArgs.GetBoolArg("-relaypriority", DEFAULT_RELAYPRIORITY) && nModifiedFees < ::minRelayTxFee.GetFee(nSize) && !AllowFree(entry.GetPriority(chainActive.Height() + 1))) { // Require that free transactions have sufficient priority to be // mined in the next block. return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "insufficient priority"); } // Continuously rate-limit free (really, very-low-fee) transactions. // This mitigates 'penny-flooding' -- sending thousands of free // transactions just to be annoying or make others' transactions take // longer to confirm. if (fLimitFree && nModifiedFees < ::minRelayTxFee.GetFee(nSize)) { static CCriticalSection csFreeLimiter; static double dFreeCount; static int64_t nLastTime; int64_t nNow = GetTime(); LOCK(csFreeLimiter); // Use an exponentially decaying ~10-minute window: dFreeCount *= pow(1.0 - 1.0 / 600.0, double(nNow - nLastTime)); nLastTime = nNow; // -limitfreerelay unit is thousand-bytes-per-minute // At default rate it would take over a month to fill 1GB if (dFreeCount + nSize >= gArgs.GetArg("-limitfreerelay", DEFAULT_LIMITFREERELAY) * 10 * 1000) { return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "rate limited free transaction"); } LogPrint(BCLog::MEMPOOL, "Rate limit dFreeCount: %g => %g\n", dFreeCount, dFreeCount + nSize); dFreeCount += nSize; } if (nAbsurdFee != Amount(0) && nFees > nAbsurdFee) { return state.Invalid(false, REJECT_HIGHFEE, "absurdly-high-fee", strprintf("%d > %d", nFees, nAbsurdFee)); } // Calculate in-mempool ancestors, up to a limit. CTxMemPool::setEntries setAncestors; size_t nLimitAncestors = gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); size_t nLimitAncestorSize = gArgs.GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT) * 1000; size_t nLimitDescendants = gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); size_t nLimitDescendantSize = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000; std::string errString; if (!pool.CalculateMemPoolAncestors( entry, setAncestors, nLimitAncestors, nLimitAncestorSize, nLimitDescendants, nLimitDescendantSize, errString)) { return state.DoS(0, false, REJECT_NONSTANDARD, "too-long-mempool-chain", false, errString); } // Set extraFlags as a set of flags that needs to be activated. uint32_t extraFlags = SCRIPT_VERIFY_NONE; if (hasMonolith) { extraFlags |= SCRIPT_ENABLE_MONOLITH_OPCODES; } if (IsReplayProtectionEnabledForCurrentBlock(config)) { extraFlags |= SCRIPT_ENABLE_REPLAY_PROTECTION; } // Check inputs based on the set of flags we activate. uint32_t scriptVerifyFlags = STANDARD_SCRIPT_VERIFY_FLAGS; if (!config.GetChainParams().RequireStandard()) { scriptVerifyFlags = SCRIPT_ENABLE_SIGHASH_FORKID | gArgs.GetArg("-promiscuousmempoolflags", scriptVerifyFlags); } // Make sure whatever we need to activate is actually activated. scriptVerifyFlags |= extraFlags; // Check against previous transactions. This is done last to help // prevent CPU exhaustion denial-of-service attacks. PrecomputedTransactionData txdata(tx); if (!CheckInputs(tx, state, view, true, scriptVerifyFlags, true, false, txdata)) { // State filled in by CheckInputs. return false; } // Check again against the current block tip's script verification flags // to cache our script execution flags. This is, of course, useless if // the next block has different script flags from the previous one, but // because the cache tracks script flags for us it will auto-invalidate // and we'll just have a few blocks of extra misses on soft-fork // activation. // // This is also useful in case of bugs in the standard flags that cause // transactions to pass as valid when they're actually invalid. For // instance the STRICTENC flag was incorrectly allowing certain CHECKSIG // NOT scripts to pass, even though they were invalid. // // There is a similar check in CreateNewBlock() to prevent creating // invalid blocks (using TestBlockValidity), however allowing such // transactions into the mempool can be exploited as a DoS attack. uint32_t currentBlockScriptVerifyFlags = GetBlockScriptFlags(config, chainActive.Tip()); if (!CheckInputsFromMempoolAndCache(tx, state, view, pool, currentBlockScriptVerifyFlags, true, txdata)) { // If we're using promiscuousmempoolflags, we may hit this normally. // Check if current block has some flags that scriptVerifyFlags does // not before printing an ominous warning. if (!(~scriptVerifyFlags & currentBlockScriptVerifyFlags)) { return error( "%s: BUG! PLEASE REPORT THIS! ConnectInputs failed against " "MANDATORY but not STANDARD flags %s, %s", __func__, txid.ToString(), FormatStateMessage(state)); } if (!CheckInputs(tx, state, view, true, MANDATORY_SCRIPT_VERIFY_FLAGS | extraFlags, true, false, txdata)) { return error( "%s: ConnectInputs failed against MANDATORY but not " "STANDARD flags due to promiscuous mempool %s, %s", __func__, txid.ToString(), FormatStateMessage(state)); } LogPrintf("Warning: -promiscuousmempool flags set to not include " "currently enforced soft forks, this may break mining or " "otherwise cause instability!\n"); } // This transaction should only count for fee estimation if // the node is not behind and it is not dependent on any other // transactions in the mempool. bool validForFeeEstimation = IsCurrentForFeeEstimation() && pool.HasNoInputsOf(tx); // Store transaction in memory. pool.addUnchecked(txid, entry, setAncestors, validForFeeEstimation); // Trim mempool and check if tx was trimmed. if (!fOverrideMempoolLimit) { LimitMempoolSize( pool, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); if (!pool.exists(txid)) { return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool full"); } } } GetMainSignals().TransactionAddedToMempool(ptx); return true; } /** * (try to) add transaction to memory pool with a specified acceptance time. */ static bool AcceptToMemoryPoolWithTime(const Config &config, CTxMemPool &pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree, bool *pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit = false, const Amount nAbsurdFee = Amount(0)) { std::vector coins_to_uncache; bool res = AcceptToMemoryPoolWorker( config, pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, fOverrideMempoolLimit, nAbsurdFee, coins_to_uncache); if (!res) { for (const COutPoint &outpoint : coins_to_uncache) { pcoinsTip->Uncache(outpoint); } } // After we've (potentially) uncached entries, ensure our coins cache is // still within its size limits CValidationState stateDummy; FlushStateToDisk(config.GetChainParams(), stateDummy, FLUSH_STATE_PERIODIC); return res; } bool AcceptToMemoryPool(const Config &config, CTxMemPool &pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree, bool *pfMissingInputs, bool fOverrideMempoolLimit, const Amount nAbsurdFee) { return AcceptToMemoryPoolWithTime(config, pool, state, tx, fLimitFree, pfMissingInputs, GetTime(), fOverrideMempoolLimit, nAbsurdFee); } /** Return transaction in txOut, and if it was found inside a block, its hash is * placed in hashBlock */ bool GetTransaction(const Config &config, const uint256 &txid, CTransactionRef &txOut, uint256 &hashBlock, bool fAllowSlow) { CBlockIndex *pindexSlow = nullptr; LOCK(cs_main); CTransactionRef ptx = mempool.get(txid); if (ptx) { txOut = ptx; return true; } if (fTxIndex) { CDiskTxPos postx; if (pblocktree->ReadTxIndex(txid, postx)) { CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); if (file.IsNull()) return error("%s: OpenBlockFile failed", __func__); CBlockHeader header; try { file >> header; fseek(file.Get(), postx.nTxOffset, SEEK_CUR); file >> txOut; } catch (const std::exception &e) { return error("%s: Deserialize or I/O error - %s", __func__, e.what()); } hashBlock = header.GetHash(); if (txOut->GetId() != txid) return error("%s: txid mismatch", __func__); return true; } } // use coin database to locate block that contains transaction, and scan it if (fAllowSlow) { const Coin &coin = AccessByTxid(*pcoinsTip, txid); if (!coin.IsSpent()) { pindexSlow = chainActive[coin.GetHeight()]; } } if (pindexSlow) { CBlock block; if (ReadBlockFromDisk(block, pindexSlow, config)) { for (const auto &tx : block.vtx) { if (tx->GetId() == txid) { txOut = tx; hashBlock = pindexSlow->GetBlockHash(); return true; } } } } return false; } ////////////////////////////////////////////////////////////////////////////// // // CBlock and CBlockIndex // static bool WriteBlockToDisk(const CBlock &block, CDiskBlockPos &pos, const CMessageHeader::MessageMagic &messageStart) { // Open history file to append CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); if (fileout.IsNull()) { return error("WriteBlockToDisk: OpenBlockFile failed"); } // Write index header unsigned int nSize = GetSerializeSize(fileout, block); fileout << FLATDATA(messageStart) << nSize; // Write block long fileOutPos = ftell(fileout.Get()); if (fileOutPos < 0) { return error("WriteBlockToDisk: ftell failed"); } pos.nPos = (unsigned int)fileOutPos; fileout << block; return true; } bool ReadBlockFromDisk(CBlock &block, const CDiskBlockPos &pos, const Config &config) { block.SetNull(); // Open history file to read CAutoFile filein(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION); if (filein.IsNull()) { return error("ReadBlockFromDisk: OpenBlockFile failed for %s", pos.ToString()); } // Read block try { filein >> block; } catch (const std::exception &e) { return error("%s: Deserialize or I/O error - %s at %s", __func__, e.what(), pos.ToString()); } // Check the header if (!CheckProofOfWork(block.GetHash(), block.nBits, config)) { return error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString()); } return true; } bool ReadBlockFromDisk(CBlock &block, const CBlockIndex *pindex, const Config &config) { if (!ReadBlockFromDisk(block, pindex->GetBlockPos(), config)) { return false; } if (block.GetHash() != pindex->GetBlockHash()) { return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() " "doesn't match index for %s at %s", pindex->ToString(), pindex->GetBlockPos().ToString()); } return true; } Amount GetBlockSubsidy(int nHeight, const Consensus::Params &consensusParams) { int halvings = nHeight / consensusParams.nSubsidyHalvingInterval; // Force block reward to zero when right shift is undefined. if (halvings >= 64) return Amount(0); Amount nSubsidy = 50 * COIN; // Subsidy is cut in half every 210,000 blocks which will occur // approximately every 4 years. return Amount(nSubsidy.GetSatoshis() >> halvings); } bool IsInitialBlockDownload() { // Once this function has returned false, it must remain false. static std::atomic latchToFalse{false}; // Optimization: pre-test latch before taking the lock. if (latchToFalse.load(std::memory_order_relaxed)) return false; LOCK(cs_main); if (latchToFalse.load(std::memory_order_relaxed)) return false; if (fImporting || fReindex) return true; if (chainActive.Tip() == nullptr) return true; if (chainActive.Tip()->nChainWork < nMinimumChainWork) return true; if (chainActive.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) return true; LogPrintf("Leaving InitialBlockDownload (latching to false)\n"); latchToFalse.store(true, std::memory_order_relaxed); return false; } CBlockIndex *pindexBestForkTip = nullptr, *pindexBestForkBase = nullptr; static void AlertNotify(const std::string &strMessage) { uiInterface.NotifyAlertChanged(); std::string strCmd = gArgs.GetArg("-alertnotify", ""); if (strCmd.empty()) return; // Alert text should be plain ascii coming from a trusted source, but to be // safe we first strip anything not in safeChars, then add single quotes // around the whole string before passing it to the shell: std::string singleQuote("'"); std::string safeStatus = SanitizeString(strMessage); safeStatus = singleQuote + safeStatus + singleQuote; boost::replace_all(strCmd, "%s", safeStatus); boost::thread t(runCommand, strCmd); // thread runs free } static void CheckForkWarningConditions() { AssertLockHeld(cs_main); // Before we get past initial download, we cannot reliably alert about forks // (we assume we don't get stuck on a fork before finishing our initial // sync) if (IsInitialBlockDownload()) return; // If our best fork is no longer within 72 blocks (+/- 12 hours if no one // mines it) of our head, drop it if (pindexBestForkTip && chainActive.Height() - pindexBestForkTip->nHeight >= 72) pindexBestForkTip = nullptr; if (pindexBestForkTip || (pindexBestInvalid && pindexBestInvalid->nChainWork > chainActive.Tip()->nChainWork + (GetBlockProof(*chainActive.Tip()) * 6))) { if (!GetfLargeWorkForkFound() && pindexBestForkBase) { std::string warning = std::string("'Warning: Large-work fork detected, forking after " "block ") + pindexBestForkBase->phashBlock->ToString() + std::string("'"); AlertNotify(warning); } if (pindexBestForkTip && pindexBestForkBase) { LogPrintf("%s: Warning: Large valid fork found\n forking the " "chain at height %d (%s)\n lasting to height %d " "(%s).\nChain state database corruption likely.\n", __func__, pindexBestForkBase->nHeight, pindexBestForkBase->phashBlock->ToString(), pindexBestForkTip->nHeight, pindexBestForkTip->phashBlock->ToString()); SetfLargeWorkForkFound(true); } else { LogPrintf("%s: Warning: Found invalid chain at least ~6 blocks " "longer than our best chain.\nChain state database " "corruption likely.\n", __func__); SetfLargeWorkInvalidChainFound(true); } } else { SetfLargeWorkForkFound(false); SetfLargeWorkInvalidChainFound(false); } } static void CheckForkWarningConditionsOnNewFork(CBlockIndex *pindexNewForkTip) { AssertLockHeld(cs_main); // If we are on a fork that is sufficiently large, set a warning flag CBlockIndex *pfork = pindexNewForkTip; CBlockIndex *plonger = chainActive.Tip(); while (pfork && pfork != plonger) { while (plonger && plonger->nHeight > pfork->nHeight) plonger = plonger->pprev; if (pfork == plonger) break; pfork = pfork->pprev; } // We define a condition where we should warn the user about as a fork of at // least 7 blocks with a tip within 72 blocks (+/- 12 hours if no one mines // it) of ours. We use 7 blocks rather arbitrarily as it represents just // under 10% of sustained network hash rate operating on the fork, or a // chain that is entirely longer than ours and invalid (note that this // should be detected by both). We define it this way because it allows us // to only store the highest fork tip (+ base) which meets the 7-block // condition and from this always have the most-likely-to-cause-warning fork if (pfork && (!pindexBestForkTip || (pindexBestForkTip && pindexNewForkTip->nHeight > pindexBestForkTip->nHeight)) && pindexNewForkTip->nChainWork - pfork->nChainWork > (GetBlockProof(*pfork) * 7) && chainActive.Height() - pindexNewForkTip->nHeight < 72) { pindexBestForkTip = pindexNewForkTip; pindexBestForkBase = pfork; } CheckForkWarningConditions(); } static void InvalidChainFound(CBlockIndex *pindexNew) { if (!pindexBestInvalid || pindexNew->nChainWork > pindexBestInvalid->nChainWork) { pindexBestInvalid = pindexNew; } LogPrintf( "%s: invalid block=%s height=%d log2_work=%.8g date=%s\n", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, log(pindexNew->nChainWork.getdouble()) / log(2.0), DateTimeStrFormat("%Y-%m-%d %H:%M:%S", pindexNew->GetBlockTime())); CBlockIndex *tip = chainActive.Tip(); assert(tip); LogPrintf("%s: current best=%s height=%d log2_work=%.8g date=%s\n", __func__, tip->GetBlockHash().ToString(), chainActive.Height(), log(tip->nChainWork.getdouble()) / log(2.0), DateTimeStrFormat("%Y-%m-%d %H:%M:%S", tip->GetBlockTime())); CheckForkWarningConditions(); } static void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) { if (!state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(pindex); setBlockIndexCandidates.erase(pindex); InvalidChainFound(pindex); } } void UpdateCoins(const CTransaction &tx, CCoinsViewCache &inputs, CTxUndo &txundo, int nHeight) { // Mark inputs spent. if (!tx.IsCoinBase()) { txundo.vprevout.reserve(tx.vin.size()); for (const CTxIn &txin : tx.vin) { txundo.vprevout.emplace_back(); bool is_spent = inputs.SpendCoin(txin.prevout, &txundo.vprevout.back()); assert(is_spent); } } // Add outputs. AddCoins(inputs, tx, nHeight); } void UpdateCoins(const CTransaction &tx, CCoinsViewCache &inputs, int nHeight) { CTxUndo txundo; UpdateCoins(tx, inputs, txundo, nHeight); } bool CScriptCheck::operator()() { const CScript &scriptSig = ptxTo->vin[nIn].scriptSig; return VerifyScript(scriptSig, scriptPubKey, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, amount, cacheStore, txdata), &error); } int GetSpendHeight(const CCoinsViewCache &inputs) { LOCK(cs_main); CBlockIndex *pindexPrev = mapBlockIndex.find(inputs.GetBestBlock())->second; return pindexPrev->nHeight + 1; } namespace Consensus { bool CheckTxInputs(const CTransaction &tx, CValidationState &state, const CCoinsViewCache &inputs, int nSpendHeight) { // This doesn't trigger the DoS code on purpose; if it did, it would make it // easier for an attacker to attempt to split the network. if (!inputs.HaveInputs(tx)) { return state.Invalid(false, 0, "", "Inputs unavailable"); } Amount nValueIn(0); Amount nFees(0); for (size_t i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; const Coin &coin = inputs.AccessCoin(prevout); assert(!coin.IsSpent()); // If prev is coinbase, check that it's matured if (coin.IsCoinBase()) { if (nSpendHeight - coin.GetHeight() < COINBASE_MATURITY) { return state.Invalid( false, REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", strprintf("tried to spend coinbase at depth %d", nSpendHeight - coin.GetHeight())); } } // Check for negative or overflow input values nValueIn += coin.GetTxOut().nValue; if (!MoneyRange(coin.GetTxOut().nValue) || !MoneyRange(nValueIn)) { return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); } } if (nValueIn < tx.GetValueOut()) { return state.DoS(100, false, REJECT_INVALID, "bad-txns-in-belowout", false, strprintf("value in (%s) < value out (%s)", FormatMoney(nValueIn), FormatMoney(tx.GetValueOut()))); } // Tally transaction fees Amount nTxFee = nValueIn - tx.GetValueOut(); if (nTxFee < Amount(0)) { return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-negative"); } nFees += nTxFee; if (!MoneyRange(nFees)) { return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-outofrange"); } return true; } } // namespace Consensus bool CheckInputs(const CTransaction &tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, const uint32_t flags, bool sigCacheStore, bool scriptCacheStore, const PrecomputedTransactionData &txdata, std::vector *pvChecks) { assert(!tx.IsCoinBase()); if (!Consensus::CheckTxInputs(tx, state, inputs, GetSpendHeight(inputs))) { return false; } if (pvChecks) { pvChecks->reserve(tx.vin.size()); } // The first loop above does all the inexpensive checks. Only if ALL inputs // pass do we perform expensive ECDSA signature checks. Helps prevent CPU // exhaustion attacks. // Skip script verification when connecting blocks under the assumedvalid // block. Assuming the assumedvalid block is valid this is safe because // block merkle hashes are still computed and checked, of course, if an // assumed valid block is invalid due to false scriptSigs this optimization // would allow an invalid chain to be accepted. if (!fScriptChecks) { return true; } // First check if script executions have been cached with the same flags. // Note that this assumes that the inputs provided are correct (ie that the // transaction hash which is in tx's prevouts properly commits to the // scriptPubKey in the inputs view of that transaction). uint256 hashCacheEntry = GetScriptCacheKey(tx, flags); if (IsKeyInScriptCache(hashCacheEntry, !scriptCacheStore)) { return true; } for (size_t i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; const Coin &coin = inputs.AccessCoin(prevout); assert(!coin.IsSpent()); // We very carefully only pass in things to CScriptCheck which are // clearly committed to by tx' witness hash. This provides a sanity // check that our caching is not introducing consensus failures through // additional data in, eg, the coins being spent being checked as a part // of CScriptCheck. const CScript &scriptPubKey = coin.GetTxOut().scriptPubKey; const Amount amount = coin.GetTxOut().nValue; // Verify signature CScriptCheck check(scriptPubKey, amount, tx, i, flags, sigCacheStore, txdata); if (pvChecks) { pvChecks->push_back(std::move(check)); } else if (!check()) { const bool hasNonMandatoryFlags = (flags & STANDARD_NOT_MANDATORY_VERIFY_FLAGS) != 0; const bool doesNotHaveMonolith = (flags & SCRIPT_ENABLE_MONOLITH_OPCODES) == 0; if (hasNonMandatoryFlags || doesNotHaveMonolith) { // Check whether the failure was caused by a non-mandatory // script verification check, such as non-standard DER encodings // or non-null dummy arguments; if so, don't trigger DoS // protection to avoid splitting the network between upgraded // and non-upgraded nodes. // // We also check activating the monolith opcodes as it is a // strictly additive change and we would not like to ban some of // our peer that are ahead of us and are considering the fork // as activated. CScriptCheck check2( scriptPubKey, amount, tx, i, (flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS) | SCRIPT_ENABLE_MONOLITH_OPCODES, sigCacheStore, txdata); if (check2()) { return state.Invalid( false, REJECT_NONSTANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); } } // Failures of other flags indicate a transaction that is invalid in // new blocks, e.g. a invalid P2SH. We DoS ban such nodes as they // are not following the protocol. That said during an upgrade // careful thought should be taken as to the correct behavior - we // may want to continue peering with non-upgraded nodes even after // soft-fork super-majority signaling has occurred. return state.DoS( 100, false, REJECT_INVALID, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); } } if (scriptCacheStore && !pvChecks) { // We executed all of the provided scripts, and were told to cache the // result. Do so now. AddKeyInScriptCache(hashCacheEntry); } return true; } namespace { bool UndoWriteToDisk(const CBlockUndo &blockundo, CDiskBlockPos &pos, const uint256 &hashBlock, const CMessageHeader::MessageMagic &messageStart) { // Open history file to append CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); if (fileout.IsNull()) return error("%s: OpenUndoFile failed", __func__); // Write index header unsigned int nSize = GetSerializeSize(fileout, blockundo); fileout << FLATDATA(messageStart) << nSize; // Write undo data long fileOutPos = ftell(fileout.Get()); if (fileOutPos < 0) return error("%s: ftell failed", __func__); pos.nPos = (unsigned int)fileOutPos; fileout << blockundo; // calculate & write checksum CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); hasher << hashBlock; hasher << blockundo; fileout << hasher.GetHash(); return true; } bool UndoReadFromDisk(CBlockUndo &blockundo, const CDiskBlockPos &pos, const uint256 &hashBlock) { // Open history file to read CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); if (filein.IsNull()) { return error("%s: OpenUndoFile failed", __func__); } // Read block uint256 hashChecksum; // We need a CHashVerifier as reserializing may lose data CHashVerifier verifier(&filein); try { verifier << hashBlock; verifier >> blockundo; filein >> hashChecksum; } catch (const std::exception &e) { return error("%s: Deserialize or I/O error - %s", __func__, e.what()); } // Verify checksum if (hashChecksum != verifier.GetHash()) { return error("%s: Checksum mismatch", __func__); } return true; } /** Abort with a message */ bool AbortNode(const std::string &strMessage, const std::string &userMessage = "") { SetMiscWarning(strMessage); LogPrintf("*** %s\n", strMessage); uiInterface.ThreadSafeMessageBox( userMessage.empty() ? _("Error: A fatal internal error occurred, see " "debug.log for details") : userMessage, "", CClientUIInterface::MSG_ERROR); StartShutdown(); return false; } bool AbortNode(CValidationState &state, const std::string &strMessage, const std::string &userMessage = "") { AbortNode(strMessage, userMessage); return state.Error(strMessage); } } // namespace /** Restore the UTXO in a Coin at a given COutPoint. */ DisconnectResult UndoCoinSpend(const Coin &undo, CCoinsViewCache &view, const COutPoint &out) { bool fClean = true; if (view.HaveCoin(out)) { // Overwriting transaction output. fClean = false; } if (undo.GetHeight() == 0) { // Missing undo metadata (height and coinbase). Older versions included // this information only in undo records for the last spend of a // transactions' outputs. This implies that it must be present for some // other output of the same tx. const Coin &alternate = AccessByTxid(view, out.hash); if (alternate.IsSpent()) { // Adding output for transaction without known metadata return DISCONNECT_FAILED; } // This is somewhat ugly, but hopefully utility is limited. This is only // useful when working from legacy on disck data. In any case, putting // the correct information in there doesn't hurt. const_cast(undo) = Coin(undo.GetTxOut(), alternate.GetHeight(), alternate.IsCoinBase()); } // The potential_overwrite parameter to AddCoin is only allowed to be false // if we know for sure that the coin did not already exist in the cache. As // we have queried for that above using HaveCoin, we don't need to guess. // When fClean is false, a coin already existed and it is an overwrite. view.AddCoin(out, std::move(undo), !fClean); return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } /** * Undo the effects of this block (with given index) on the UTXO set represented * by coins. When FAILED is returned, view is left in an indeterminate state. */ static DisconnectResult DisconnectBlock(const CBlock &block, const CBlockIndex *pindex, CCoinsViewCache &view) { CBlockUndo blockUndo; CDiskBlockPos pos = pindex->GetUndoPos(); if (pos.IsNull()) { error("DisconnectBlock(): no undo data available"); return DISCONNECT_FAILED; } if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) { error("DisconnectBlock(): failure reading undo data"); return DISCONNECT_FAILED; } return ApplyBlockUndo(blockUndo, block, pindex, view); } DisconnectResult ApplyBlockUndo(const CBlockUndo &blockUndo, const CBlock &block, const CBlockIndex *pindex, CCoinsViewCache &view) { bool fClean = true; if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) { error("DisconnectBlock(): block and undo data inconsistent"); return DISCONNECT_FAILED; } // Undo transactions in reverse order. size_t i = block.vtx.size(); while (i-- > 0) { const CTransaction &tx = *(block.vtx[i]); uint256 txid = tx.GetId(); // Check that all outputs are available and match the outputs in the // block itself exactly. for (size_t o = 0; o < tx.vout.size(); o++) { if (tx.vout[o].scriptPubKey.IsUnspendable()) { continue; } COutPoint out(txid, o); Coin coin; bool is_spent = view.SpendCoin(out, &coin); if (!is_spent || tx.vout[o] != coin.GetTxOut()) { // transaction output mismatch fClean = false; } } // Restore inputs. if (i < 1) { // Skip the coinbase. continue; } const CTxUndo &txundo = blockUndo.vtxundo[i - 1]; if (txundo.vprevout.size() != tx.vin.size()) { error("DisconnectBlock(): transaction and undo data inconsistent"); return DISCONNECT_FAILED; } for (size_t j = tx.vin.size(); j-- > 0;) { const COutPoint &out = tx.vin[j].prevout; const Coin &undo = txundo.vprevout[j]; DisconnectResult res = UndoCoinSpend(undo, view, out); if (res == DISCONNECT_FAILED) { return DISCONNECT_FAILED; } fClean = fClean && res != DISCONNECT_UNCLEAN; } } // Move best block pointer to previous block. view.SetBestBlock(block.hashPrevBlock); return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } static void FlushBlockFile(bool fFinalize = false) { LOCK(cs_LastBlockFile); CDiskBlockPos posOld(nLastBlockFile, 0); FILE *fileOld = OpenBlockFile(posOld); if (fileOld) { if (fFinalize) TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nSize); FileCommit(fileOld); fclose(fileOld); } fileOld = OpenUndoFile(posOld); if (fileOld) { if (fFinalize) TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nUndoSize); FileCommit(fileOld); fclose(fileOld); } } static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); static CCheckQueue scriptcheckqueue(128); void ThreadScriptCheck() { RenameThread("bitcoin-scriptch"); scriptcheckqueue.Thread(); } // Protected by cs_main VersionBitsCache versionbitscache; int32_t ComputeBlockVersion(const CBlockIndex *pindexPrev, const Consensus::Params ¶ms) { LOCK(cs_main); int32_t nVersion = VERSIONBITS_TOP_BITS; for (int i = 0; i < (int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; i++) { ThresholdState state = VersionBitsState( pindexPrev, params, (Consensus::DeploymentPos)i, versionbitscache); if (state == THRESHOLD_LOCKED_IN || state == THRESHOLD_STARTED) { nVersion |= VersionBitsMask(params, (Consensus::DeploymentPos)i); } } return nVersion; } /** * Threshold condition checker that triggers when unknown versionbits are seen * on the network. */ class WarningBitsConditionChecker : public AbstractThresholdConditionChecker { private: int bit; public: WarningBitsConditionChecker(int bitIn) : bit(bitIn) {} int64_t BeginTime(const Consensus::Params ¶ms) const override { return 0; } int64_t EndTime(const Consensus::Params ¶ms) const override { return std::numeric_limits::max(); } int Period(const Consensus::Params ¶ms) const override { return params.nMinerConfirmationWindow; } int Threshold(const Consensus::Params ¶ms) const override { return params.nRuleChangeActivationThreshold; } bool Condition(const CBlockIndex *pindex, const Consensus::Params ¶ms) const override { return ((pindex->nVersion & VERSIONBITS_TOP_MASK) == VERSIONBITS_TOP_BITS) && ((pindex->nVersion >> bit) & 1) != 0 && ((ComputeBlockVersion(pindex->pprev, params) >> bit) & 1) == 0; } }; // Protected by cs_main static ThresholdConditionCache warningcache[VERSIONBITS_NUM_BITS]; // Returns the script flags which should be checked for a given block static uint32_t GetBlockScriptFlags(const Config &config, const CBlockIndex *pChainTip) { AssertLockHeld(cs_main); const Consensus::Params &consensusparams = config.GetChainParams().GetConsensus(); uint32_t flags = SCRIPT_VERIFY_NONE; // P2SH didn't become active until Apr 1 2012 if (pChainTip->GetMedianTimePast() >= P2SH_ACTIVATION_TIME) { flags |= SCRIPT_VERIFY_P2SH; } // Start enforcing the DERSIG (BIP66) rule if ((pChainTip->nHeight + 1) >= consensusparams.BIP66Height) { flags |= SCRIPT_VERIFY_DERSIG; } // Start enforcing CHECKLOCKTIMEVERIFY (BIP65) rule if ((pChainTip->nHeight + 1) >= consensusparams.BIP65Height) { flags |= SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY; } // Start enforcing BIP112 (CHECKSEQUENCEVERIFY) using versionbits logic. if (VersionBitsState(pChainTip, consensusparams, Consensus::DEPLOYMENT_CSV, versionbitscache) == THRESHOLD_ACTIVE) { flags |= SCRIPT_VERIFY_CHECKSEQUENCEVERIFY; } // If the UAHF is enabled, we start accepting replay protected txns if (IsUAHFenabled(config, pChainTip)) { flags |= SCRIPT_VERIFY_STRICTENC; flags |= SCRIPT_ENABLE_SIGHASH_FORKID; } // If the DAA HF is enabled, we start rejecting transaction that use a high // s in their signature. We also make sure that signature that are supposed // to fail (for instance in multisig or other forms of smart contracts) are // null. if (IsDAAEnabled(config, pChainTip)) { flags |= SCRIPT_VERIFY_LOW_S; flags |= SCRIPT_VERIFY_NULLFAIL; } // The monolith HF enable a set of opcodes. if (IsMonolithEnabled(config, pChainTip)) { flags |= SCRIPT_ENABLE_MONOLITH_OPCODES; } // We make sure this node will have replay protection during the next hard // fork. if (IsReplayProtectionEnabled(config, pChainTip)) { flags |= SCRIPT_ENABLE_REPLAY_PROTECTION; } return flags; } static int64_t nTimeCheck = 0; static int64_t nTimeForks = 0; static int64_t nTimeVerify = 0; static int64_t nTimeConnect = 0; static int64_t nTimeIndex = 0; static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; /** * Apply the effects of this block (with given index) on the UTXO set * represented by coins. Validity checks that depend on the UTXO set are also * done; ConnectBlock() can fail if those validity checks fail (among other * reasons). */ static bool ConnectBlock(const Config &config, const CBlock &block, CValidationState &state, CBlockIndex *pindex, CCoinsViewCache &view, bool fJustCheck = false) { AssertLockHeld(cs_main); int64_t nTimeStart = GetTimeMicros(); // Check it again in case a previous version let a bad block in BlockValidationOptions validationOptions = BlockValidationOptions(!fJustCheck, !fJustCheck); if (!CheckBlock(config, block, state, validationOptions)) { return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); } // Verify that the view's current state corresponds to the previous block uint256 hashPrevBlock = pindex->pprev == nullptr ? uint256() : pindex->pprev->GetBlockHash(); assert(hashPrevBlock == view.GetBestBlock()); // Special case for the genesis block, skipping connection of its // transactions (its coinbase is unspendable) const Consensus::Params &consensusParams = config.GetChainParams().GetConsensus(); if (block.GetHash() == consensusParams.hashGenesisBlock) { if (!fJustCheck) { view.SetBestBlock(pindex->GetBlockHash()); } return true; } bool fScriptChecks = true; if (!hashAssumeValid.IsNull()) { // We've been configured with the hash of a block which has been // externally verified to have a valid history. A suitable default value // is included with the software and updated from time to time. Because // validity relative to a piece of software is an objective fact these // defaults can be easily reviewed. This setting doesn't force the // selection of any particular chain but makes validating some faster by // effectively caching the result of part of the verification. BlockMap::const_iterator it = mapBlockIndex.find(hashAssumeValid); if (it != mapBlockIndex.end()) { if (it->second->GetAncestor(pindex->nHeight) == pindex && pindexBestHeader->GetAncestor(pindex->nHeight) == pindex && pindexBestHeader->nChainWork >= nMinimumChainWork) { // This block is a member of the assumed verified chain and an // ancestor of the best header. The equivalent time check // discourages hashpower from extorting the network via DOS // attack into accepting an invalid block through telling users // they must manually set assumevalid. Requiring a software // change or burying the invalid block, regardless of the // setting, makes it hard to hide the implication of the demand. // This also avoids having release candidates that are hardly // doing any signature verification at all in testing without // having to artificially set the default assumed verified block // further back. The test against nMinimumChainWork prevents the // skipping when denied access to any chain at least as good as // the expected chain. fScriptChecks = (GetBlockProofEquivalentTime( *pindexBestHeader, *pindex, *pindexBestHeader, consensusParams) <= 60 * 60 * 24 * 7 * 2); } } } int64_t nTime1 = GetTimeMicros(); nTimeCheck += nTime1 - nTimeStart; LogPrint(BCLog::BENCH, " - Sanity checks: %.2fms [%.2fs]\n", 0.001 * (nTime1 - nTimeStart), nTimeCheck * 0.000001); // Do not allow blocks that contain transactions which 'overwrite' older // transactions, unless those are already completely spent. If such // overwrites are allowed, coinbases and transactions depending upon those // can be duplicated to remove the ability to spend the first instance -- // even after being sent to another address. See BIP30 and // http://r6.ca/blog/20120206T005236Z.html for more information. This logic // is not necessary for memory pool transactions, as AcceptToMemoryPool // already refuses previously-known transaction ids entirely. This rule was // originally applied to all blocks with a timestamp after March 15, 2012, // 0:00 UTC. Now that the whole chain is irreversibly beyond that time it is // applied to all blocks except the two in the chain that violate it. This // prevents exploiting the issue against nodes during their initial block // download. bool fEnforceBIP30 = (!pindex->phashBlock) || // Enforce on CreateNewBlock // invocations which don't // have a hash. !((pindex->nHeight == 91842 && pindex->GetBlockHash() == uint256S("0x00000000000a4d0a398161ffc163c503763" "b1f4360639393e0e4c8e300e0caec")) || (pindex->nHeight == 91880 && pindex->GetBlockHash() == uint256S("0x00000000000743f190a18c5577a3c2d2a1f" "610ae9601ac046a38084ccb7cd721"))); // Once BIP34 activated it was not possible to create new duplicate // coinbases and thus other than starting with the 2 existing duplicate // coinbase pairs, not possible to create overwriting txs. But by the time // BIP34 activated, in each of the existing pairs the duplicate coinbase had // overwritten the first before the first had been spent. Since those // coinbases are sufficiently buried its no longer possible to create // further duplicate transactions descending from the known pairs either. If // we're on the known chain at height greater than where BIP34 activated, we // can save the db accesses needed for the BIP30 check. CBlockIndex *pindexBIP34height = pindex->pprev->GetAncestor(consensusParams.BIP34Height); // Only continue to enforce if we're below BIP34 activation height or the // block hash at that height doesn't correspond. fEnforceBIP30 = fEnforceBIP30 && (!pindexBIP34height || !(pindexBIP34height->GetBlockHash() == consensusParams.BIP34Hash)); if (fEnforceBIP30) { for (const auto &tx : block.vtx) { for (size_t o = 0; o < tx->vout.size(); o++) { if (view.HaveCoin(COutPoint(tx->GetId(), o))) { return state.DoS( 100, error("ConnectBlock(): tried to overwrite transaction"), REJECT_INVALID, "bad-txns-BIP30"); } } } } // Start enforcing BIP68 (sequence locks) using versionbits logic. int nLockTimeFlags = 0; if (VersionBitsState(pindex->pprev, consensusParams, Consensus::DEPLOYMENT_CSV, versionbitscache) == THRESHOLD_ACTIVE) { nLockTimeFlags |= LOCKTIME_VERIFY_SEQUENCE; } const uint32_t flags = GetBlockScriptFlags(config, pindex->pprev); int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1; LogPrint(BCLog::BENCH, " - Fork checks: %.2fms [%.2fs]\n", 0.001 * (nTime2 - nTime1), nTimeForks * 0.000001); CBlockUndo blockundo; CCheckQueueControl control(fScriptChecks ? &scriptcheckqueue : nullptr); std::vector prevheights; Amount nFees(0); int nInputs = 0; // Sigops counting. We need to do it again because of P2SH. uint64_t nSigOpsCount = 0; const uint64_t currentBlockSize = ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION); const uint64_t nMaxSigOpsCount = GetMaxBlockSigOpsCount(currentBlockSize); CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); std::vector> vPos; vPos.reserve(block.vtx.size()); blockundo.vtxundo.reserve(block.vtx.size() - 1); for (size_t i = 0; i < block.vtx.size(); i++) { const CTransaction &tx = *(block.vtx[i]); nInputs += tx.vin.size(); if (!tx.IsCoinBase()) { if (!view.HaveInputs(tx)) { return state.DoS( 100, error("ConnectBlock(): inputs missing/spent"), REJECT_INVALID, "bad-txns-inputs-missingorspent"); } // Check that transaction is BIP68 final BIP68 lock checks (as // opposed to nLockTime checks) must be in ConnectBlock because they // require the UTXO set. prevheights.resize(tx.vin.size()); for (size_t j = 0; j < tx.vin.size(); j++) { prevheights[j] = view.AccessCoin(tx.vin[j].prevout).GetHeight(); } if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { return state.DoS( 100, error("%s: contains a non-BIP68-final transaction", __func__), REJECT_INVALID, "bad-txns-nonfinal"); } } // GetTransactionSigOpCount counts 2 types of sigops: // * legacy (always) // * p2sh (when P2SH enabled in flags and excludes coinbase) auto txSigOpsCount = GetTransactionSigOpCount(tx, view, flags); if (txSigOpsCount > MAX_TX_SIGOPS_COUNT) { return state.DoS(100, false, REJECT_INVALID, "bad-txn-sigops"); } nSigOpsCount += txSigOpsCount; if (nSigOpsCount > nMaxSigOpsCount) { return state.DoS(100, error("ConnectBlock(): too many sigops"), REJECT_INVALID, "bad-blk-sigops"); } if (!tx.IsCoinBase()) { Amount fee = view.GetValueIn(tx) - tx.GetValueOut(); nFees += fee; // Don't cache results if we're actually connecting blocks (still // consult the cache, though). bool fCacheResults = fJustCheck; std::vector vChecks; if (!CheckInputs(tx, state, view, fScriptChecks, flags, fCacheResults, fCacheResults, PrecomputedTransactionData(tx), &vChecks)) { return error("ConnectBlock(): CheckInputs on %s failed with %s", tx.GetId().ToString(), FormatStateMessage(state)); } control.Add(vChecks); } CTxUndo undoDummy; if (i > 0) { blockundo.vtxundo.push_back(CTxUndo()); } UpdateCoins(tx, view, i == 0 ? undoDummy : blockundo.vtxundo.back(), pindex->nHeight); vPos.push_back(std::make_pair(tx.GetId(), pos)); pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION); } int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2; LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, " "%.3fms/txin) [%.2fs]\n", (unsigned)block.vtx.size(), 0.001 * (nTime3 - nTime2), 0.001 * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * (nTime3 - nTime2) / (nInputs - 1), nTimeConnect * 0.000001); Amount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, consensusParams); if (block.vtx[0]->GetValueOut() > blockReward) { return state.DoS(100, error("ConnectBlock(): coinbase pays too much " "(actual=%d vs limit=%d)", block.vtx[0]->GetValueOut(), blockReward), REJECT_INVALID, "bad-cb-amount"); } if (!control.Wait()) { return state.DoS(100, false, REJECT_INVALID, "blk-bad-inputs", false, "parallel script check failed"); } int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2; LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs]\n", nInputs - 1, 0.001 * (nTime4 - nTime2), nInputs <= 1 ? 0 : 0.001 * (nTime4 - nTime2) / (nInputs - 1), nTimeVerify * 0.000001); if (fJustCheck) { return true; } // Write undo information to disk if (pindex->GetUndoPos().IsNull() || !pindex->IsValid(BLOCK_VALID_SCRIPTS)) { if (pindex->GetUndoPos().IsNull()) { CDiskBlockPos _pos; if (!FindUndoPos( state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 40)) { return error("ConnectBlock(): FindUndoPos failed"); } if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), config.GetChainParams().DiskMagic())) { return AbortNode(state, "Failed to write undo data"); } // update nUndoPos in block index pindex->nUndoPos = _pos.nPos; pindex->nStatus |= BLOCK_HAVE_UNDO; } pindex->RaiseValidity(BLOCK_VALID_SCRIPTS); setDirtyBlockIndex.insert(pindex); } if (fTxIndex && !pblocktree->WriteTxIndex(vPos)) { return AbortNode(state, "Failed to write transaction index"); } // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs]\n", 0.001 * (nTime5 - nTime4), nTimeIndex * 0.000001); int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5; LogPrint(BCLog::BENCH, " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime6 - nTime5), nTimeCallbacks * 0.000001); // If we just activated the replay protection with that block, it means // transaction in the mempool are now invalid. As a result, we need to clear // the mempool. if (IsReplayProtectionEnabled(config, pindex) && !IsReplayProtectionEnabled(config, pindex->pprev)) { mempool.clear(); } return true; } /** * Update the on-disk chain state. * The caches and indexes are flushed depending on the mode we're called with if * they're too large, if it's been a while since the last write, or always and * in all cases if we're in prune mode and are deleting files. */ static bool FlushStateToDisk(const CChainParams &chainparams, CValidationState &state, FlushStateMode mode, int nManualPruneHeight) { int64_t nMempoolUsage = mempool.DynamicMemoryUsage(); LOCK(cs_main); static int64_t nLastWrite = 0; static int64_t nLastFlush = 0; static int64_t nLastSetChain = 0; std::set setFilesToPrune; bool fFlushForPrune = false; bool fDoFullFlush = false; int64_t nNow = 0; try { { LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { if (nManualPruneHeight > 0) { FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight); } else { FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight()); fCheckForPruning = false; } if (!setFilesToPrune.empty()) { fFlushForPrune = true; if (!fHavePruned) { pblocktree->WriteFlag("prunedblockfiles", true); fHavePruned = true; } } } nNow = GetTimeMicros(); // Avoid writing/flushing immediately after startup. if (nLastWrite == 0) { nLastWrite = nNow; } if (nLastFlush == 0) { nLastFlush = nNow; } if (nLastSetChain == 0) { nLastSetChain = nNow; } int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t cacheSize = pcoinsTip->DynamicMemoryUsage(); int64_t nTotalSpace = nCoinCacheUsage + std::max(nMempoolSizeMax - nMempoolUsage, 0); // The cache is large and we're within 10% and 10 MiB of the limit, // but we have time now (not in the middle of a block processing). bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024); // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nTotalSpace; // It's been a while since we wrote the block index to disk. Do this // frequently, so we don't need to redownload after a crash. bool fPeriodicWrite = mode == FLUSH_STATE_PERIODIC && nNow > nLastWrite + (int64_t)DATABASE_WRITE_INTERVAL * 1000000; // It's been very long since we flushed the cache. Do this // infrequently, to optimize cache usage. bool fPeriodicFlush = mode == FLUSH_STATE_PERIODIC && nNow > nLastFlush + (int64_t)DATABASE_FLUSH_INTERVAL * 1000000; // Combine all conditions that result in a full cache flush. fDoFullFlush = (mode == FLUSH_STATE_ALWAYS) || fCacheLarge || fCacheCritical || fPeriodicFlush || fFlushForPrune; // Write blocks and block index to disk. if (fDoFullFlush || fPeriodicWrite) { // Depend on nMinDiskSpace to ensure we can write block index if (!CheckDiskSpace(0)) { return state.Error("out of disk space"); } // First make sure all block and undo data is flushed to disk. FlushBlockFile(); // Then update all block file information (which may refer to // block and undo files). { std::vector> vFiles; vFiles.reserve(setDirtyFileInfo.size()); for (std::set::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end();) { vFiles.push_back( std::make_pair(*it, &vinfoBlockFile[*it])); setDirtyFileInfo.erase(it++); } std::vector vBlocks; vBlocks.reserve(setDirtyBlockIndex.size()); for (std::set::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end();) { vBlocks.push_back(*it); setDirtyBlockIndex.erase(it++); } if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { return AbortNode( state, "Failed to write to block index database"); } } // Finally remove any pruned files if (fFlushForPrune) UnlinkPrunedFiles(setFilesToPrune); nLastWrite = nNow; } // Flush best chain related state. This can only be done if the // blocks / block index write was also done. if (fDoFullFlush) { // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is // already an overestimation, as most will delete an existing // entry or overwrite one. Still, use a conservative safety // factor of 2. if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) { return state.Error("out of disk space"); } // Flush the chainstate (which may refer to block index // entries). if (!pcoinsTip->Flush()) { return AbortNode(state, "Failed to write to coin database"); } nLastFlush = nNow; } } if (fDoFullFlush || ((mode == FLUSH_STATE_ALWAYS || mode == FLUSH_STATE_PERIODIC) && nNow > nLastSetChain + (int64_t)DATABASE_WRITE_INTERVAL * 1000000)) { // Update best block in wallet (so we can detect restored wallets). GetMainSignals().SetBestChain(chainActive.GetLocator()); nLastSetChain = nNow; } } catch (const std::runtime_error &e) { return AbortNode( state, std::string("System error while flushing: ") + e.what()); } return true; } void FlushStateToDisk() { CValidationState state; const CChainParams &chainparams = Params(); FlushStateToDisk(chainparams, state, FLUSH_STATE_ALWAYS); } void PruneAndFlush() { CValidationState state; fCheckForPruning = true; const CChainParams &chainparams = Params(); FlushStateToDisk(chainparams, state, FLUSH_STATE_NONE); } /** * Update chainActive and related internal data structures when adding a new * block to the chain tip. */ static void UpdateTip(const Config &config, CBlockIndex *pindexNew) { const Consensus::Params &consensusParams = config.GetChainParams().GetConsensus(); chainActive.SetTip(pindexNew); // New best block mempool.AddTransactionsUpdated(1); cvBlockChange.notify_all(); static bool fWarned = false; std::vector warningMessages; if (!IsInitialBlockDownload()) { int nUpgraded = 0; const CBlockIndex *pindex = chainActive.Tip(); for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(bit); ThresholdState state = checker.GetStateFor(pindex, consensusParams, warningcache[bit]); if (state == THRESHOLD_ACTIVE || state == THRESHOLD_LOCKED_IN) { if (state == THRESHOLD_ACTIVE) { std::string strWarning = strprintf(_("Warning: unknown new rules activated " "(versionbit %i)"), bit); SetMiscWarning(strWarning); if (!fWarned) { AlertNotify(strWarning); fWarned = true; } } else { warningMessages.push_back( strprintf("unknown new rules are about to activate " "(versionbit %i)", bit)); } } } // Check the version of the last 100 blocks to see if we need to // upgrade: for (int i = 0; i < 100 && pindex != nullptr; i++) { int32_t nExpectedVersion = ComputeBlockVersion(pindex->pprev, consensusParams); if (pindex->nVersion > VERSIONBITS_LAST_OLD_BLOCK_VERSION && (pindex->nVersion & ~nExpectedVersion) != 0) ++nUpgraded; pindex = pindex->pprev; } if (nUpgraded > 0) warningMessages.push_back(strprintf( "%d of last 100 blocks have unexpected version", nUpgraded)); if (nUpgraded > 100 / 2) { std::string strWarning = _("Warning: Unknown block versions being mined! It's possible " "unknown rules are in effect"); // notify GetWarnings(), called by Qt and the JSON-RPC code to warn // the user: SetMiscWarning(strWarning); if (!fWarned) { AlertNotify(strWarning); fWarned = true; } } } LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu " "date='%s' progress=%f cache=%.1fMiB(%utxo)", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, log(chainActive.Tip()->nChainWork.getdouble()) / log(2.0), (unsigned long)chainActive.Tip()->nChainTx, DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), GuessVerificationProgress(config.GetChainParams().TxData(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1 << 20)), pcoinsTip->GetCacheSize()); if (!warningMessages.empty()) LogPrintf(" warning='%s'", boost::algorithm::join(warningMessages, ", ")); LogPrintf("\n"); } /** * Disconnect chainActive's tip. * After calling, the mempool will be in an inconsistent state, with * transactions from disconnected blocks being added to disconnectpool. You * should make the mempool consistent again by calling UpdateMempoolForReorg. * with cs_main held. * * If disconnectpool is nullptr, then no disconnected transactions are added to * disconnectpool (note that the caller is responsible for mempool consistency * in any case). */ static bool DisconnectTip(const Config &config, CValidationState &state, DisconnectedBlockTransactions *disconnectpool) { CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); // Read block from disk. std::shared_ptr pblock = std::make_shared(); CBlock &block = *pblock; if (!ReadBlockFromDisk(block, pindexDelete, config)) { return AbortNode(state, "Failed to read block"); } // Apply the block atomically to the chain state. int64_t nStart = GetTimeMicros(); { CCoinsViewCache view(pcoinsTip); assert(view.GetBestBlock() == pindexDelete->GetBlockHash()); if (DisconnectBlock(block, pindexDelete, view) != DISCONNECT_OK) { return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); } bool flushed = view.Flush(); assert(flushed); } LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); // Write the chain state to disk, if necessary. if (!FlushStateToDisk(config.GetChainParams(), state, FLUSH_STATE_IF_NEEDED)) { return false; } // If this block was deactivating the replay protection, then we need to // remove transactions that are replay protected from the mempool. There is // no easy way to do this so we'll just discard the whole mempool and then // add the transaction of the block we just disconnected back. // // Samewise, if this block enabled the monolith opcodes, then we need to // clear the mempool of any transaction using them. if ((IsReplayProtectionEnabled(config, pindexDelete) && !IsReplayProtectionEnabled(config, pindexDelete->pprev)) || (IsMonolithEnabled(config, pindexDelete) && !IsMonolithEnabled(config, pindexDelete->pprev))) { mempool.clear(); // While not strictly necessary, clearing the disconnect pool is also // beneficial so we don't try to reuse its content at the end of the // reorg, which we know will fail. if (disconnectpool) { disconnectpool->clear(); } } if (disconnectpool) { // Save transactions to re-add to mempool at end of reorg for (const auto &tx : boost::adaptors::reverse(block.vtx)) { disconnectpool->addTransaction(tx); } while (disconnectpool->DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE) { // Drop the earliest entry, and remove its children from the // mempool. auto it = disconnectpool->queuedTx.get().begin(); mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); disconnectpool->removeEntry(it); } } // Update chainActive and related variables. UpdateTip(config, pindexDelete->pprev); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: GetMainSignals().BlockDisconnected(pblock); return true; } static int64_t nTimeReadFromDisk = 0; static int64_t nTimeConnectTotal = 0; static int64_t nTimeFlush = 0; static int64_t nTimeChainState = 0; static int64_t nTimePostConnect = 0; struct PerBlockConnectTrace { CBlockIndex *pindex = nullptr; std::shared_ptr pblock; std::shared_ptr> conflictedTxs; PerBlockConnectTrace() : conflictedTxs(std::make_shared>()) {} }; /** * Used to track blocks whose transactions were applied to the UTXO state as a * part of a single ActivateBestChainStep call. * * This class also tracks transactions that are removed from the mempool as * conflicts (per block) and can be used to pass all those transactions through * SyncTransaction. * * This class assumes (and asserts) that the conflicted transactions for a given * block are added via mempool callbacks prior to the BlockConnected() * associated with those transactions. If any transactions are marked * conflicted, it is assumed that an associated block will always be added. * * This class is single-use, once you call GetBlocksConnected() you have to * throw it away and make a new one. */ class ConnectTrace { private: std::vector blocksConnected; CTxMemPool &pool; public: ConnectTrace(CTxMemPool &_pool) : blocksConnected(1), pool(_pool) { pool.NotifyEntryRemoved.connect( boost::bind(&ConnectTrace::NotifyEntryRemoved, this, _1, _2)); } ~ConnectTrace() { pool.NotifyEntryRemoved.disconnect( boost::bind(&ConnectTrace::NotifyEntryRemoved, this, _1, _2)); } void BlockConnected(CBlockIndex *pindex, std::shared_ptr pblock) { assert(!blocksConnected.back().pindex); assert(pindex); assert(pblock); blocksConnected.back().pindex = pindex; blocksConnected.back().pblock = std::move(pblock); blocksConnected.emplace_back(); } std::vector &GetBlocksConnected() { // We always keep one extra block at the end of our list because blocks // are added after all the conflicted transactions have been filled in. // Thus, the last entry should always be an empty one waiting for the // transactions from the next block. We pop the last entry here to make // sure the list we return is sane. assert(!blocksConnected.back().pindex); assert(blocksConnected.back().conflictedTxs->empty()); blocksConnected.pop_back(); return blocksConnected; } void NotifyEntryRemoved(CTransactionRef txRemoved, MemPoolRemovalReason reason) { assert(!blocksConnected.back().pindex); if (reason == MemPoolRemovalReason::CONFLICT) { blocksConnected.back().conflictedTxs->emplace_back( std::move(txRemoved)); } } }; /** * Connect a new block to chainActive. pblock is either nullptr or a pointer to * a CBlock corresponding to pindexNew, to bypass loading it again from disk. * * The block is always added to connectTrace (either after loading from disk or * by copying pblock) - if that is not intended, care must be taken to remove * the last entry in blocksConnected in case of failure. */ static bool ConnectTip(const Config &config, CValidationState &state, CBlockIndex *pindexNew, const std::shared_ptr &pblock, ConnectTrace &connectTrace, DisconnectedBlockTransactions &disconnectpool) { assert(pindexNew->pprev == chainActive.Tip()); // Read block from disk. int64_t nTime1 = GetTimeMicros(); std::shared_ptr pthisBlock; if (!pblock) { std::shared_ptr pblockNew = std::make_shared(); if (!ReadBlockFromDisk(*pblockNew, pindexNew, config)) { return AbortNode(state, "Failed to read block"); } pthisBlock = pblockNew; } else { pthisBlock = pblock; } const CBlock &blockConnecting = *pthisBlock; // Apply the block atomically to the chain state. int64_t nTime2 = GetTimeMicros(); nTimeReadFromDisk += nTime2 - nTime1; int64_t nTime3; LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); { CCoinsViewCache view(pcoinsTip); bool rv = ConnectBlock(config, blockConnecting, state, pindexNew, view); GetMainSignals().BlockChecked(blockConnecting, state); if (!rv) { if (state.IsInvalid()) { InvalidBlockFound(pindexNew, state); } return error("ConnectTip(): ConnectBlock %s failed (%s)", pindexNew->GetBlockHash().ToString(), FormatStateMessage(state)); } nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); bool flushed = view.Flush(); assert(flushed); } int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); // Write the chain state to disk, if necessary. if (!FlushStateToDisk(config.GetChainParams(), state, FLUSH_STATE_IF_NEEDED)) { return false; } int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool.; mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); disconnectpool.removeForBlock(blockConnecting.vtx); // Update chainActive & related variables. UpdateTip(config, pindexNew); int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; LogPrint(BCLog::BENCH, " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); LogPrint(BCLog::BENCH, "- Connect block: %.2fms [%.2fs]\n", (nTime6 - nTime1) * 0.001, nTimeTotal * 0.000001); connectTrace.BlockConnected(pindexNew, std::move(pthisBlock)); return true; } /** * Return the tip of the chain with the most work in it, that isn't known to be * invalid (it's however far from certain to be valid). */ static CBlockIndex *FindMostWorkChain() { do { CBlockIndex *pindexNew = nullptr; // Find the best candidate header. { std::set::reverse_iterator it = setBlockIndexCandidates.rbegin(); if (it == setBlockIndexCandidates.rend()) return nullptr; pindexNew = *it; } // Check whether all blocks on the path between the currently active // chain and the candidate are valid. Just going until the active chain // is an optimization, as we know all blocks in it are valid already. CBlockIndex *pindexTest = pindexNew; bool fInvalidAncestor = false; while (pindexTest && !chainActive.Contains(pindexTest)) { assert(pindexTest->nChainTx || pindexTest->nHeight == 0); // Pruned nodes may have entries in setBlockIndexCandidates for // which block files have been deleted. Remove those as candidates // for the most work chain if we come across them; we can't switch // to a chain unless we have all the non-active-chain parent blocks. bool fFailedChain = pindexTest->nStatus & BLOCK_FAILED_MASK; bool fMissingData = !(pindexTest->nStatus & BLOCK_HAVE_DATA); if (fFailedChain || fMissingData) { // Candidate chain is not usable (either invalid or missing // data) if (fFailedChain && (pindexBestInvalid == nullptr || pindexNew->nChainWork > pindexBestInvalid->nChainWork)) pindexBestInvalid = pindexNew; CBlockIndex *pindexFailed = pindexNew; // Remove the entire chain from the set. while (pindexTest != pindexFailed) { if (fFailedChain) { pindexFailed->nStatus |= BLOCK_FAILED_CHILD; } else if (fMissingData) { // If we're missing data, then add back to // mapBlocksUnlinked, so that if the block arrives in // the future we can try adding to // setBlockIndexCandidates again. mapBlocksUnlinked.insert( std::make_pair(pindexFailed->pprev, pindexFailed)); } setBlockIndexCandidates.erase(pindexFailed); pindexFailed = pindexFailed->pprev; } setBlockIndexCandidates.erase(pindexTest); fInvalidAncestor = true; break; } pindexTest = pindexTest->pprev; } if (!fInvalidAncestor) return pindexNew; } while (true); } /** Delete all entries in setBlockIndexCandidates that are worse than the * current tip. */ static void PruneBlockIndexCandidates() { // Note that we can't delete the current block itself, as we may need to // return to it later in case a reorganization to a better block fails. std::set::iterator it = setBlockIndexCandidates.begin(); while (it != setBlockIndexCandidates.end() && setBlockIndexCandidates.value_comp()(*it, chainActive.Tip())) { setBlockIndexCandidates.erase(it++); } // Either the current tip or a successor of it we're working towards is left // in setBlockIndexCandidates. assert(!setBlockIndexCandidates.empty()); } /** * Try to make some progress towards making pindexMostWork the active block. * pblock is either nullptr or a pointer to a CBlock corresponding to * pindexMostWork. */ static bool ActivateBestChainStep(const Config &config, CValidationState &state, CBlockIndex *pindexMostWork, const std::shared_ptr &pblock, bool &fInvalidFound, ConnectTrace &connectTrace) { AssertLockHeld(cs_main); const CBlockIndex *pindexOldTip = chainActive.Tip(); const CBlockIndex *pindexFork = chainActive.FindFork(pindexMostWork); // Disconnect active blocks which are no longer in the best chain. bool fBlocksDisconnected = false; DisconnectedBlockTransactions disconnectpool; while (chainActive.Tip() && chainActive.Tip() != pindexFork) { if (!DisconnectTip(config, state, &disconnectpool)) { // This is likely a fatal error, but keep the mempool consistent, // just in case. Only remove from the mempool in this case. UpdateMempoolForReorg(config, disconnectpool, false); return false; } fBlocksDisconnected = true; } // Build list of new blocks to connect. std::vector vpindexToConnect; bool fContinue = true; int nHeight = pindexFork ? pindexFork->nHeight : -1; while (fContinue && nHeight != pindexMostWork->nHeight) { // Don't iterate the entire list of potential improvements toward the // best tip, as we likely only need a few blocks along the way. int nTargetHeight = std::min(nHeight + 32, pindexMostWork->nHeight); vpindexToConnect.clear(); vpindexToConnect.reserve(nTargetHeight - nHeight); CBlockIndex *pindexIter = pindexMostWork->GetAncestor(nTargetHeight); while (pindexIter && pindexIter->nHeight != nHeight) { vpindexToConnect.push_back(pindexIter); pindexIter = pindexIter->pprev; } nHeight = nTargetHeight; // Connect new blocks. for (CBlockIndex *pindexConnect : boost::adaptors::reverse(vpindexToConnect)) { if (!ConnectTip(config, state, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr(), connectTrace, disconnectpool)) { if (state.IsInvalid()) { // The block violates a consensus rule. if (!state.CorruptionPossible()) InvalidChainFound(vpindexToConnect.back()); state = CValidationState(); fInvalidFound = true; fContinue = false; break; } else { // A system error occurred (disk space, database error, // ...). // Make the mempool consistent with the current tip, just in // case any observers try to use it before shutdown. UpdateMempoolForReorg(config, disconnectpool, false); return false; } } else { PruneBlockIndexCandidates(); if (!pindexOldTip || chainActive.Tip()->nChainWork > pindexOldTip->nChainWork) { // We're in a better position than we were. Return // temporarily to release the lock. fContinue = false; break; } } } } if (fBlocksDisconnected) { // If any blocks were disconnected, disconnectpool may be non empty. Add // any disconnected transactions back to the mempool. UpdateMempoolForReorg(config, disconnectpool, true); } mempool.check(pcoinsTip); // Callbacks/notifications for a new best chain. if (fInvalidFound) CheckForkWarningConditionsOnNewFork(vpindexToConnect.back()); else CheckForkWarningConditions(); return true; } static void NotifyHeaderTip() { bool fNotify = false; bool fInitialBlockDownload = false; static CBlockIndex *pindexHeaderOld = nullptr; CBlockIndex *pindexHeader = nullptr; { LOCK(cs_main); pindexHeader = pindexBestHeader; if (pindexHeader != pindexHeaderOld) { fNotify = true; fInitialBlockDownload = IsInitialBlockDownload(); pindexHeaderOld = pindexHeader; } } // Send block tip changed notifications without cs_main if (fNotify) { uiInterface.NotifyHeaderTip(fInitialBlockDownload, pindexHeader); } } bool ActivateBestChain(const Config &config, CValidationState &state, std::shared_ptr pblock) { // Note that while we're often called here from ProcessNewBlock, this is // far from a guarantee. Things in the P2P/RPC will often end up calling // us in the middle of ProcessNewBlock - do not assume pblock is set // sanely for performance or correctness! CBlockIndex *pindexMostWork = nullptr; CBlockIndex *pindexNewTip = nullptr; do { boost::this_thread::interruption_point(); if (ShutdownRequested()) break; const CBlockIndex *pindexFork; bool fInitialDownload; { LOCK(cs_main); // Destructed before cs_main is unlocked. ConnectTrace connectTrace(mempool); CBlockIndex *pindexOldTip = chainActive.Tip(); if (pindexMostWork == nullptr) { pindexMostWork = FindMostWorkChain(); } // Whether we have anything to do at all. if (pindexMostWork == nullptr || pindexMostWork == chainActive.Tip()) { return true; } bool fInvalidFound = false; std::shared_ptr nullBlockPtr; if (!ActivateBestChainStep( config, state, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : nullBlockPtr, fInvalidFound, connectTrace)) { return false; } if (fInvalidFound) { // Wipe cache, we may need another branch now. pindexMostWork = nullptr; } pindexNewTip = chainActive.Tip(); pindexFork = chainActive.FindFork(pindexOldTip); fInitialDownload = IsInitialBlockDownload(); for (const PerBlockConnectTrace &trace : connectTrace.GetBlocksConnected()) { assert(trace.pblock && trace.pindex); GetMainSignals().BlockConnected(trace.pblock, trace.pindex, *trace.conflictedTxs); } } // When we reach this point, we switched to a new tip (stored in // pindexNewTip). // Notifications/callbacks that can run without cs_main // Notify external listeners about the new tip. GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload); // Always notify the UI if a new block tip was connected if (pindexFork != pindexNewTip) { uiInterface.NotifyBlockTip(fInitialDownload, pindexNewTip); } } while (pindexNewTip != pindexMostWork); const CChainParams ¶ms = config.GetChainParams(); CheckBlockIndex(params.GetConsensus()); // Write changes periodically to disk, after relay. if (!FlushStateToDisk(params, state, FLUSH_STATE_PERIODIC)) { return false; } int nStopAtHeight = gArgs.GetArg("-stopatheight", DEFAULT_STOPATHEIGHT); if (nStopAtHeight && pindexNewTip && pindexNewTip->nHeight >= nStopAtHeight) { StartShutdown(); } return true; } bool PreciousBlock(const Config &config, CValidationState &state, CBlockIndex *pindex) { { LOCK(cs_main); if (pindex->nChainWork < chainActive.Tip()->nChainWork) { // Nothing to do, this block is not at the tip. return true; } if (chainActive.Tip()->nChainWork > nLastPreciousChainwork) { // The chain has been extended since the last call, reset the // counter. nBlockReverseSequenceId = -1; } nLastPreciousChainwork = chainActive.Tip()->nChainWork; setBlockIndexCandidates.erase(pindex); pindex->nSequenceId = nBlockReverseSequenceId; if (nBlockReverseSequenceId > std::numeric_limits::min()) { // We can't keep reducing the counter if somebody really wants to // call preciousblock 2**31-1 times on the same set of tips... nBlockReverseSequenceId--; } if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && pindex->nChainTx) { setBlockIndexCandidates.insert(pindex); PruneBlockIndexCandidates(); } } return ActivateBestChain(config, state); } bool InvalidateBlock(const Config &config, CValidationState &state, CBlockIndex *pindex) { AssertLockHeld(cs_main); // Mark the block itself as invalid. pindex->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(pindex); setBlockIndexCandidates.erase(pindex); DisconnectedBlockTransactions disconnectpool; while (chainActive.Contains(pindex)) { CBlockIndex *pindexWalk = chainActive.Tip(); pindexWalk->nStatus |= BLOCK_FAILED_CHILD; setDirtyBlockIndex.insert(pindexWalk); setBlockIndexCandidates.erase(pindexWalk); // ActivateBestChain considers blocks already in chainActive // unconditionally valid already, so force disconnect away from it. if (!DisconnectTip(config, state, &disconnectpool)) { // It's probably hopeless to try to make the mempool consistent // here if DisconnectTip failed, but we can try. UpdateMempoolForReorg(config, disconnectpool, false); return false; } } // DisconnectTip will add transactions to disconnectpool; try to add these // back to the mempool. UpdateMempoolForReorg(config, disconnectpool, true); // The resulting new best tip may not be in setBlockIndexCandidates anymore, // so add it again. BlockMap::iterator it = mapBlockIndex.begin(); while (it != mapBlockIndex.end()) { if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->nChainTx && !setBlockIndexCandidates.value_comp()(it->second, chainActive.Tip())) { setBlockIndexCandidates.insert(it->second); } it++; } InvalidChainFound(pindex); uiInterface.NotifyBlockTip(IsInitialBlockDownload(), pindex->pprev); return true; } bool ResetBlockFailureFlags(CBlockIndex *pindex) { AssertLockHeld(cs_main); int nHeight = pindex->nHeight; // Remove the invalidity flag from this block and all its descendants. BlockMap::iterator it = mapBlockIndex.begin(); while (it != mapBlockIndex.end()) { if (!it->second->IsValid() && it->second->GetAncestor(nHeight) == pindex) { it->second->nStatus &= ~BLOCK_FAILED_MASK; setDirtyBlockIndex.insert(it->second); if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->nChainTx && setBlockIndexCandidates.value_comp()(chainActive.Tip(), it->second)) { setBlockIndexCandidates.insert(it->second); } if (it->second == pindexBestInvalid) { // Reset invalid block marker if it was pointing to one of // those. pindexBestInvalid = nullptr; } } it++; } // Remove the invalidity flag from all ancestors too. while (pindex != nullptr) { if (pindex->nStatus & BLOCK_FAILED_MASK) { pindex->nStatus &= ~BLOCK_FAILED_MASK; setDirtyBlockIndex.insert(pindex); } pindex = pindex->pprev; } return true; } static CBlockIndex *AddToBlockIndex(const CBlockHeader &block) { // Check for duplicate uint256 hash = block.GetHash(); BlockMap::iterator it = mapBlockIndex.find(hash); if (it != mapBlockIndex.end()) return it->second; // Construct new block index object CBlockIndex *pindexNew = new CBlockIndex(block); assert(pindexNew); // We assign the sequence id to blocks only when the full data is available, // to avoid miners withholding blocks but broadcasting headers, to get a // competitive advantage. pindexNew->nSequenceId = 0; BlockMap::iterator mi = mapBlockIndex.insert(std::make_pair(hash, pindexNew)).first; pindexNew->phashBlock = &((*mi).first); BlockMap::iterator miPrev = mapBlockIndex.find(block.hashPrevBlock); if (miPrev != mapBlockIndex.end()) { pindexNew->pprev = (*miPrev).second; pindexNew->nHeight = pindexNew->pprev->nHeight + 1; pindexNew->BuildSkip(); } pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime); pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew); pindexNew->RaiseValidity(BLOCK_VALID_TREE); if (pindexBestHeader == nullptr || pindexBestHeader->nChainWork < pindexNew->nChainWork) { pindexBestHeader = pindexNew; } setDirtyBlockIndex.insert(pindexNew); return pindexNew; } /** * Mark a block as having its data received and checked (up to * BLOCK_VALID_TRANSACTIONS). */ bool ReceivedBlockTransactions(const CBlock &block, CValidationState &state, CBlockIndex *pindexNew, const CDiskBlockPos &pos) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; pindexNew->nFile = pos.nFile; pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); setDirtyBlockIndex.insert(pindexNew); if (pindexNew->pprev == nullptr || pindexNew->pprev->nChainTx) { // If pindexNew is the genesis block or all parents are // BLOCK_VALID_TRANSACTIONS. std::deque queue; queue.push_back(pindexNew); // Recursively process any descendant blocks that now may be eligible to // be connected. while (!queue.empty()) { CBlockIndex *pindex = queue.front(); queue.pop_front(); pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; { LOCK(cs_nBlockSequenceId); pindex->nSequenceId = nBlockSequenceId++; } if (chainActive.Tip() == nullptr || !setBlockIndexCandidates.value_comp()(pindex, chainActive.Tip())) { setBlockIndexCandidates.insert(pindex); } std::pair::iterator, std::multimap::iterator> range = mapBlocksUnlinked.equal_range(pindex); while (range.first != range.second) { std::multimap::iterator it = range.first; queue.push_back(it->second); range.first++; mapBlocksUnlinked.erase(it); } } } else { if (pindexNew->pprev && pindexNew->pprev->IsValid(BLOCK_VALID_TREE)) { mapBlocksUnlinked.insert( std::make_pair(pindexNew->pprev, pindexNew)); } } return true; } static bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) { LOCK(cs_LastBlockFile); unsigned int nFile = fKnown ? pos.nFile : nLastBlockFile; if (vinfoBlockFile.size() <= nFile) { vinfoBlockFile.resize(nFile + 1); } if (!fKnown) { while (vinfoBlockFile[nFile].nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { nFile++; if (vinfoBlockFile.size() <= nFile) { vinfoBlockFile.resize(nFile + 1); } } pos.nFile = nFile; pos.nPos = vinfoBlockFile[nFile].nSize; } if ((int)nFile != nLastBlockFile) { if (!fKnown) { LogPrintf("Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); } FlushBlockFile(!fKnown); nLastBlockFile = nFile; } vinfoBlockFile[nFile].AddBlock(nHeight, nTime); if (fKnown) vinfoBlockFile[nFile].nSize = std::max(pos.nPos + nAddSize, vinfoBlockFile[nFile].nSize); else vinfoBlockFile[nFile].nSize += nAddSize; if (!fKnown) { unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; if (nNewChunks > nOldChunks) { if (fPruneMode) fCheckForPruning = true; if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos)) { FILE *file = OpenBlockFile(pos); if (file) { LogPrintf( "Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos); fclose(file); } } else { return state.Error("out of disk space"); } } } setDirtyFileInfo.insert(nFile); return true; } static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize) { pos.nFile = nFile; LOCK(cs_LastBlockFile); unsigned int nNewSize; pos.nPos = vinfoBlockFile[nFile].nUndoSize; nNewSize = vinfoBlockFile[nFile].nUndoSize += nAddSize; setDirtyFileInfo.insert(nFile); unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; if (nNewChunks > nOldChunks) { if (fPruneMode) fCheckForPruning = true; if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos)) { FILE *file = OpenUndoFile(pos); if (file) { LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile); AllocateFileRange(file, pos.nPos, nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos); fclose(file); } } else return state.Error("out of disk space"); } return true; } /** * Return true if the provided block header is valid. * Only verify PoW if blockValidationOptions is configured to do so. * This allows validation of headers on which the PoW hasn't been done. * For example: to validate template handed to mining software. * Do not call this for any check that depends on the context. * For context-dependant calls, see ContextualCheckBlockHeader. */ static bool CheckBlockHeader( const Config &config, const CBlockHeader &block, CValidationState &state, BlockValidationOptions validationOptions = BlockValidationOptions()) { // Check proof of work matches claimed amount if (validationOptions.shouldValidatePoW() && !CheckProofOfWork(block.GetHash(), block.nBits, config)) { return state.DoS(50, false, REJECT_INVALID, "high-hash", false, "proof of work failed"); } return true; } bool CheckBlock(const Config &config, const CBlock &block, CValidationState &state, BlockValidationOptions validationOptions) { // These are checks that are independent of context. if (block.fChecked) { return true; } // Check that the header is valid (particularly PoW). This is mostly // redundant with the call in AcceptBlockHeader. if (!CheckBlockHeader(config, block, state, validationOptions)) { return false; } // Check the merkle root. if (validationOptions.shouldValidateMerkleRoot()) { bool mutated; uint256 hashMerkleRoot2 = BlockMerkleRoot(block, &mutated); if (block.hashMerkleRoot != hashMerkleRoot2) { return state.DoS(100, false, REJECT_INVALID, "bad-txnmrklroot", true, "hashMerkleRoot mismatch"); } // Check for merkle tree malleability (CVE-2012-2459): repeating // sequences of transactions in a block without affecting the merkle // root of a block, while still invalidating it. if (mutated) { return state.DoS(100, false, REJECT_INVALID, "bad-txns-duplicate", true, "duplicate transaction"); } } // All potential-corruption validation must be done before we do any // transaction validation, as otherwise we may mark the header as invalid // because we receive the wrong transactions for it. // First transaction must be coinbase. if (block.vtx.empty()) { return state.DoS(100, false, REJECT_INVALID, "bad-cb-missing", false, "first tx is not coinbase"); } // Size limits. auto nMaxBlockSize = config.GetMaxBlockSize(); // Bail early if there is no way this block is of reasonable size. if ((block.vtx.size() * MIN_TRANSACTION_SIZE) > nMaxBlockSize) { return state.DoS(100, false, REJECT_INVALID, "bad-blk-length", false, "size limits failed"); } auto currentBlockSize = ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION); if (currentBlockSize > nMaxBlockSize) { return state.DoS(100, false, REJECT_INVALID, "bad-blk-length", false, "size limits failed"); } // And a valid coinbase. if (!CheckCoinbase(*block.vtx[0], state, false)) { return state.Invalid(false, state.GetRejectCode(), state.GetRejectReason(), strprintf("Coinbase check failed (txid %s) %s", block.vtx[0]->GetId().ToString(), state.GetDebugMessage())); } // Keep track of the sigops count. uint64_t nSigOps = 0; auto nMaxSigOpsCount = GetMaxBlockSigOpsCount(currentBlockSize); // Check transactions auto txCount = block.vtx.size(); auto *tx = block.vtx[0].get(); size_t i = 0; while (true) { // Count the sigops for the current transaction. If the total sigops // count is too high, the the block is invalid. nSigOps += GetSigOpCountWithoutP2SH(*tx); if (nSigOps > nMaxSigOpsCount) { return state.DoS(100, false, REJECT_INVALID, "bad-blk-sigops", false, "out-of-bounds SigOpCount"); } // Go to the next transaction. i++; // We reached the end of the block, success. if (i >= txCount) { break; } // Check that the transaction is valid. because this check differs for // the coinbase, the loos is arranged such as this only runs after at // least one increment. tx = block.vtx[i].get(); if (!CheckRegularTransaction(*tx, state, false)) { return state.Invalid( false, state.GetRejectCode(), state.GetRejectReason(), strprintf("Transaction check failed (txid %s) %s", tx->GetId().ToString(), state.GetDebugMessage())); } } if (validationOptions.shouldValidatePoW() && validationOptions.shouldValidateMerkleRoot()) { block.fChecked = true; } return true; } static bool CheckIndexAgainstCheckpoint(const CBlockIndex *pindexPrev, CValidationState &state, const CChainParams &chainparams, const uint256 &hash) { if (*pindexPrev->phashBlock == chainparams.GetConsensus().hashGenesisBlock) { return true; } int nHeight = pindexPrev->nHeight + 1; const CCheckpointData &checkpoints = chainparams.Checkpoints(); // Check that the block chain matches the known block chain up to a // checkpoint. if (!Checkpoints::CheckBlock(checkpoints, nHeight, hash)) { return state.DoS(100, error("%s: rejected by checkpoint lock-in at %d", __func__, nHeight), REJECT_CHECKPOINT, "checkpoint mismatch"); } // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in // our MapBlockIndex. CBlockIndex *pcheckpoint = Checkpoints::GetLastCheckpoint(checkpoints); if (pcheckpoint && nHeight < pcheckpoint->nHeight) { return state.DoS( 100, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight), REJECT_CHECKPOINT, "bad-fork-prior-to-checkpoint"); } return true; } static bool ContextualCheckBlockHeader(const Config &config, const CBlockHeader &block, CValidationState &state, const CBlockIndex *pindexPrev, int64_t nAdjustedTime) { const Consensus::Params &consensusParams = config.GetChainParams().GetConsensus(); const int nHeight = pindexPrev == nullptr ? 0 : pindexPrev->nHeight + 1; // Check proof of work if (block.nBits != GetNextWorkRequired(pindexPrev, &block, config)) { LogPrintf("bad bits after height: %d\n", pindexPrev->nHeight); return state.DoS(100, false, REJECT_INVALID, "bad-diffbits", false, "incorrect proof of work"); } // Check timestamp against prev if (block.GetBlockTime() <= pindexPrev->GetMedianTimePast()) { return state.Invalid(false, REJECT_INVALID, "time-too-old", "block's timestamp is too early"); } // Check timestamp if (block.GetBlockTime() > nAdjustedTime + MAX_FUTURE_BLOCK_TIME) { return state.Invalid(false, REJECT_INVALID, "time-too-new", "block timestamp too far in the future"); } // Reject outdated version blocks when 95% (75% on testnet) of the network // has upgraded: // check for version 2, 3 and 4 upgrades if ((block.nVersion < 2 && nHeight >= consensusParams.BIP34Height) || (block.nVersion < 3 && nHeight >= consensusParams.BIP66Height) || (block.nVersion < 4 && nHeight >= consensusParams.BIP65Height)) { return state.Invalid( false, REJECT_OBSOLETE, strprintf("bad-version(0x%08x)", block.nVersion), strprintf("rejected nVersion=0x%08x block", block.nVersion)); } return true; } bool ContextualCheckTransaction(const Config &config, const CTransaction &tx, CValidationState &state, int nHeight, int64_t nLockTimeCutoff) { if (!IsFinalTx(tx, nHeight, nLockTimeCutoff)) { // While this is only one transaction, we use txns in the error to // ensure continuity with other clients. return state.DoS(10, false, REJECT_INVALID, "bad-txns-nonfinal", false, "non-final transaction"); } const Consensus::Params &consensusParams = config.GetChainParams().GetConsensus(); if (IsUAHFenabled(config, nHeight) && nHeight <= consensusParams.antiReplayOpReturnSunsetHeight) { for (const CTxOut &o : tx.vout) { if (o.scriptPubKey.IsCommitment( consensusParams.antiReplayOpReturnCommitment)) { return state.DoS(10, false, REJECT_INVALID, "bad-txn-replay", false, "non playable transaction"); } } } return true; } bool ContextualCheckTransactionForCurrentBlock(const Config &config, const CTransaction &tx, CValidationState &state, int flags) { AssertLockHeld(cs_main); // By convention a negative value for flags indicates that the current // network-enforced consensus rules should be used. In a future soft-fork // scenario that would mean checking which rules would be enforced for the // next block and setting the appropriate flags. At the present time no // soft-forks are scheduled, so no flags are set. flags = std::max(flags, 0); // ContextualCheckTransactionForCurrentBlock() uses chainActive.Height()+1 // to evaluate nLockTime because when IsFinalTx() is called within // CBlock::AcceptBlock(), the height of the block *being* evaluated is what // is used. Thus if we want to know if a transaction can be part of the // *next* block, we need to call ContextualCheckTransaction() with one more // than chainActive.Height(). const int nBlockHeight = chainActive.Height() + 1; // BIP113 will require that time-locked transactions have nLockTime set to // less than the median time of the previous block they're contained in. // When the next block is created its previous block will be the current // chain tip, so we use that to calculate the median time passed to // ContextualCheckTransaction() if LOCKTIME_MEDIAN_TIME_PAST is set. const int64_t nLockTimeCutoff = (flags & LOCKTIME_MEDIAN_TIME_PAST) ? chainActive.Tip()->GetMedianTimePast() : GetAdjustedTime(); return ContextualCheckTransaction(config, tx, state, nBlockHeight, nLockTimeCutoff); } static bool ContextualCheckBlock(const Config &config, const CBlock &block, CValidationState &state, const CBlockIndex *pindexPrev) { const int nHeight = pindexPrev == nullptr ? 0 : pindexPrev->nHeight + 1; const Consensus::Params &consensusParams = config.GetChainParams().GetConsensus(); // Start enforcing BIP113 (Median Time Past) using versionbits logic. int nLockTimeFlags = 0; if (VersionBitsState(pindexPrev, consensusParams, Consensus::DEPLOYMENT_CSV, versionbitscache) == THRESHOLD_ACTIVE) { nLockTimeFlags |= LOCKTIME_MEDIAN_TIME_PAST; } if (!IsMonolithEnabled(config, pindexPrev)) { // When the May 15, 2018 HF is not enabled, block cannot be bigger // than 8MB . const uint64_t currentBlockSize = ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION); if (currentBlockSize > 8 * ONE_MEGABYTE) { return state.DoS(100, false, REJECT_INVALID, "bad-blk-length", false, "size limits failed"); } } const int64_t nMedianTimePast = pindexPrev == nullptr ? 0 : pindexPrev->GetMedianTimePast(); const int64_t nLockTimeCutoff = (nLockTimeFlags & LOCKTIME_MEDIAN_TIME_PAST) ? nMedianTimePast : block.GetBlockTime(); // Check that all transactions are finalized for (const auto &tx : block.vtx) { if (!ContextualCheckTransaction(config, *tx, state, nHeight, nLockTimeCutoff)) { // state set by ContextualCheckTransaction. return false; } } // Enforce rule that the coinbase starts with serialized block height if (nHeight >= consensusParams.BIP34Height) { CScript expect = CScript() << nHeight; if (block.vtx[0]->vin[0].scriptSig.size() < expect.size() || !std::equal(expect.begin(), expect.end(), block.vtx[0]->vin[0].scriptSig.begin())) { return state.DoS(100, false, REJECT_INVALID, "bad-cb-height", false, "block height mismatch in coinbase"); } } return true; } /** * If the provided block header is valid, add it to the block index. * * Returns true if the block is succesfully added to the block index. */ static bool AcceptBlockHeader(const Config &config, const CBlockHeader &block, CValidationState &state, CBlockIndex **ppindex) { AssertLockHeld(cs_main); const CChainParams &chainparams = config.GetChainParams(); // Check for duplicate uint256 hash = block.GetHash(); BlockMap::iterator miSelf = mapBlockIndex.find(hash); CBlockIndex *pindex = nullptr; if (hash != chainparams.GetConsensus().hashGenesisBlock) { if (miSelf != mapBlockIndex.end()) { // Block header is already known. pindex = miSelf->second; if (ppindex) { *ppindex = pindex; } if (pindex->nStatus & BLOCK_FAILED_MASK) { return state.Invalid(error("%s: block %s is marked invalid", __func__, hash.ToString()), 0, "duplicate"); } return true; } if (!CheckBlockHeader(config, block, state)) { return error("%s: Consensus::CheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); } // Get prev block index CBlockIndex *pindexPrev = nullptr; BlockMap::iterator mi = mapBlockIndex.find(block.hashPrevBlock); if (mi == mapBlockIndex.end()) { return state.DoS(10, error("%s: prev block not found", __func__), 0, "prev-blk-not-found"); } pindexPrev = (*mi).second; if (pindexPrev->nStatus & BLOCK_FAILED_MASK) { return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); } assert(pindexPrev); if (fCheckpointsEnabled && !CheckIndexAgainstCheckpoint(pindexPrev, state, chainparams, hash)) { return error("%s: CheckIndexAgainstCheckpoint(): %s", __func__, state.GetRejectReason().c_str()); } if (!ContextualCheckBlockHeader(config, block, state, pindexPrev, GetAdjustedTime())) { return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); } } if (pindex == nullptr) { pindex = AddToBlockIndex(block); } if (ppindex) { *ppindex = pindex; } CheckBlockIndex(chainparams.GetConsensus()); return true; } // Exposed wrapper for AcceptBlockHeader bool ProcessNewBlockHeaders(const Config &config, const std::vector &headers, CValidationState &state, const CBlockIndex **ppindex) { { LOCK(cs_main); for (const CBlockHeader &header : headers) { // Use a temp pindex instead of ppindex to avoid a const_cast CBlockIndex *pindex = nullptr; if (!AcceptBlockHeader(config, header, state, &pindex)) { return false; } if (ppindex) { *ppindex = pindex; } } } NotifyHeaderTip(); return true; } /** * Store block on disk. If dbp is non-null, the file is known to already reside * on disk. */ static bool AcceptBlock(const Config &config, const std::shared_ptr &pblock, CValidationState &state, CBlockIndex **ppindex, bool fRequested, const CDiskBlockPos *dbp, bool *fNewBlock) { AssertLockHeld(cs_main); const CBlock &block = *pblock; if (fNewBlock) { *fNewBlock = false; } CBlockIndex *pindexDummy = nullptr; CBlockIndex *&pindex = ppindex ? *ppindex : pindexDummy; if (!AcceptBlockHeader(config, block, state, &pindex)) { return false; } // Try to process all requested blocks that we don't have, but only // process an unrequested block if it's new and has enough work to // advance our tip, and isn't too many blocks ahead. bool fAlreadyHave = pindex->nStatus & BLOCK_HAVE_DATA; bool fHasMoreWork = (chainActive.Tip() ? pindex->nChainWork > chainActive.Tip()->nChainWork : true); // Blocks that are too out-of-order needlessly limit the effectiveness of // pruning, because pruning will not delete block files that contain any // blocks which are too close in height to the tip. Apply this test // regardless of whether pruning is enabled; it should generally be safe to // not process unrequested blocks. bool fTooFarAhead = (pindex->nHeight > int(chainActive.Height() + MIN_BLOCKS_TO_KEEP)); // TODO: Decouple this function from the block download logic by removing // fRequested // This requires some new chain datastructure to efficiently look up if a // block is in a chain leading to a candidate for best tip, despite not // being such a candidate itself. // TODO: deal better with return value and error conditions for duplicate // and unrequested blocks. if (fAlreadyHave) { return true; } // If we didn't ask for it: if (!fRequested) { // This is a previously-processed block that was pruned. if (pindex->nTx != 0) { return true; } // Don't process less-work chains. if (!fHasMoreWork) { return true; } // Block height is too high. if (fTooFarAhead) { return true; } } if (fNewBlock) { *fNewBlock = true; } if (!CheckBlock(config, block, state) || !ContextualCheckBlock(config, block, state, pindex->pprev)) { if (state.IsInvalid() && !state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(pindex); } return error("%s: %s (block %s)", __func__, FormatStateMessage(state), block.GetHash().ToString()); } // Header is valid/has work and the merkle tree is good. // Relay now, but if it does not build on our best tip, let the // SendMessages loop relay it. if (!IsInitialBlockDownload() && chainActive.Tip() == pindex->pprev) { GetMainSignals().NewPoWValidBlock(pindex, pblock); } int nHeight = pindex->nHeight; const CChainParams &chainparams = config.GetChainParams(); // Write block to history file try { unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); CDiskBlockPos blockPos; if (dbp != nullptr) { blockPos = *dbp; } if (!FindBlockPos(state, blockPos, nBlockSize + 8, nHeight, block.GetBlockTime(), dbp != nullptr)) { return error("AcceptBlock(): FindBlockPos failed"); } if (dbp == nullptr) { if (!WriteBlockToDisk(block, blockPos, chainparams.DiskMagic())) { AbortNode(state, "Failed to write block"); } } if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) { return error("AcceptBlock(): ReceivedBlockTransactions failed"); } } catch (const std::runtime_error &e) { return AbortNode(state, std::string("System error: ") + e.what()); } if (fCheckForPruning) { // we just allocated more disk space for block files. FlushStateToDisk(config.GetChainParams(), state, FLUSH_STATE_NONE); } return true; } bool ProcessNewBlock(const Config &config, const std::shared_ptr pblock, bool fForceProcessing, bool *fNewBlock) { { CBlockIndex *pindex = nullptr; if (fNewBlock) { *fNewBlock = false; } const CChainParams &chainparams = config.GetChainParams(); CValidationState state; // Ensure that CheckBlock() passes before calling AcceptBlock, as // belt-and-suspenders. bool ret = CheckBlock(config, *pblock, state); LOCK(cs_main); if (ret) { // Store to disk ret = AcceptBlock(config, pblock, state, &pindex, fForceProcessing, nullptr, fNewBlock); } CheckBlockIndex(chainparams.GetConsensus()); if (!ret) { GetMainSignals().BlockChecked(*pblock, state); return error("%s: AcceptBlock FAILED", __func__); } } NotifyHeaderTip(); // Only used to report errors, not invalidity - ignore it CValidationState state; if (!ActivateBestChain(config, state, pblock)) return error("%s: ActivateBestChain failed", __func__); return true; } bool TestBlockValidity(const Config &config, CValidationState &state, const CBlock &block, CBlockIndex *pindexPrev, BlockValidationOptions validationOptions) { AssertLockHeld(cs_main); const CChainParams &chainparams = config.GetChainParams(); assert(pindexPrev && pindexPrev == chainActive.Tip()); if (fCheckpointsEnabled && !CheckIndexAgainstCheckpoint(pindexPrev, state, chainparams, block.GetHash())) { return error("%s: CheckIndexAgainstCheckpoint(): %s", __func__, state.GetRejectReason().c_str()); } CCoinsViewCache viewNew(pcoinsTip); CBlockIndex indexDummy(block); indexDummy.pprev = pindexPrev; indexDummy.nHeight = pindexPrev->nHeight + 1; // NOTE: CheckBlockHeader is called by CheckBlock if (!ContextualCheckBlockHeader(config, block, state, pindexPrev, GetAdjustedTime())) { return error("%s: Consensus::ContextualCheckBlockHeader: %s", __func__, FormatStateMessage(state)); } if (!CheckBlock(config, block, state, validationOptions)) { return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); } if (!ContextualCheckBlock(config, block, state, pindexPrev)) { return error("%s: Consensus::ContextualCheckBlock: %s", __func__, FormatStateMessage(state)); } if (!ConnectBlock(config, block, state, &indexDummy, viewNew, true)) { return false; } assert(state.IsValid()); return true; } /** * BLOCK PRUNING CODE */ /* Calculate the amount of disk space the block & undo files currently use */ static uint64_t CalculateCurrentUsage() { uint64_t retval = 0; for (const CBlockFileInfo &file : vinfoBlockFile) { retval += file.nSize + file.nUndoSize; } return retval; } /* Prune a block file (modify associated database entries)*/ void PruneOneBlockFile(const int fileNumber) { for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); ++it) { CBlockIndex *pindex = it->second; if (pindex->nFile == fileNumber) { pindex->nStatus &= ~BLOCK_HAVE_DATA; pindex->nStatus &= ~BLOCK_HAVE_UNDO; pindex->nFile = 0; pindex->nDataPos = 0; pindex->nUndoPos = 0; setDirtyBlockIndex.insert(pindex); // Prune from mapBlocksUnlinked -- any block we prune would have // to be downloaded again in order to consider its chain, at which // point it would be considered as a candidate for // mapBlocksUnlinked or setBlockIndexCandidates. std::pair::iterator, std::multimap::iterator> range = mapBlocksUnlinked.equal_range(pindex->pprev); while (range.first != range.second) { std::multimap::iterator _it = range.first; range.first++; if (_it->second == pindex) { mapBlocksUnlinked.erase(_it); } } } } vinfoBlockFile[fileNumber].SetNull(); setDirtyFileInfo.insert(fileNumber); } void UnlinkPrunedFiles(const std::set &setFilesToPrune) { for (std::set::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { CDiskBlockPos pos(*it, 0); fs::remove(GetBlockPosFilename(pos, "blk")); fs::remove(GetBlockPosFilename(pos, "rev")); LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); } } /** * Calculate the block/rev files to delete based on height specified by user * with RPC command pruneblockchain */ static void FindFilesToPruneManual(std::set &setFilesToPrune, int nManualPruneHeight) { assert(fPruneMode && nManualPruneHeight > 0); LOCK2(cs_main, cs_LastBlockFile); if (chainActive.Tip() == nullptr) { return; } // last block to prune is the lesser of (user-specified height, // MIN_BLOCKS_TO_KEEP from the tip) unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP); int count = 0; for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { if (vinfoBlockFile[fileNumber].nSize == 0 || vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) { continue; } PruneOneBlockFile(fileNumber); setFilesToPrune.insert(fileNumber); count++; } LogPrintf("Prune (Manual): prune_height=%d removed %d blk/rev pairs\n", nLastBlockWeCanPrune, count); } /* This function is called from the RPC code for pruneblockchain */ void PruneBlockFilesManual(int nManualPruneHeight) { CValidationState state; const CChainParams &chainparams = Params(); FlushStateToDisk(chainparams, state, FLUSH_STATE_NONE, nManualPruneHeight); } /** * Prune block and undo files (blk???.dat and undo???.dat) so that the disk * space used is less than a user-defined target. The user sets the target (in * MB) on the command line or in config file. This will be run on startup and * whenever new space is allocated in a block or undo file, staying below the * target. Changing back to unpruned requires a reindex (which in this case * means the blockchain must be re-downloaded.) * * Pruning functions are called from FlushStateToDisk when the global * fCheckForPruning flag has been set. Block and undo files are deleted in * lock-step (when blk00003.dat is deleted, so is rev00003.dat.). Pruning cannot * take place until the longest chain is at least a certain length (100000 on * mainnet, 1000 on testnet, 1000 on regtest). Pruning will never delete a block * within a defined distance (currently 288) from the active chain's tip. The * block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks * that were stored in the deleted files. A db flag records the fact that at * least some block files have been pruned. * * @param[out] setFilesToPrune The set of file indices that can be unlinked * will be returned */ static void FindFilesToPrune(std::set &setFilesToPrune, uint64_t nPruneAfterHeight) { LOCK2(cs_main, cs_LastBlockFile); if (chainActive.Tip() == nullptr || nPruneTarget == 0) { return; } if (uint64_t(chainActive.Tip()->nHeight) <= nPruneAfterHeight) { return; } unsigned int nLastBlockWeCanPrune = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP; uint64_t nCurrentUsage = CalculateCurrentUsage(); // We don't check to prune until after we've allocated new space for files, // so we should leave a buffer under our target to account for another // allocation before the next pruning. uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE; uint64_t nBytesToPrune; int count = 0; if (nCurrentUsage + nBuffer >= nPruneTarget) { for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize; if (vinfoBlockFile[fileNumber].nSize == 0) { continue; } // are we below our target? if (nCurrentUsage + nBuffer < nPruneTarget) { break; } // don't prune files that could have a block within // MIN_BLOCKS_TO_KEEP of the main chain's tip but keep scanning if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) { continue; } PruneOneBlockFile(fileNumber); // Queue up the files for removal setFilesToPrune.insert(fileNumber); nCurrentUsage -= nBytesToPrune; count++; } } LogPrint(BCLog::PRUNE, "Prune: target=%dMiB actual=%dMiB diff=%dMiB " "max_prune_height=%d removed %d blk/rev pairs\n", nPruneTarget / 1024 / 1024, nCurrentUsage / 1024 / 1024, ((int64_t)nPruneTarget - (int64_t)nCurrentUsage) / 1024 / 1024, nLastBlockWeCanPrune, count); } bool CheckDiskSpace(uint64_t nAdditionalBytes) { uint64_t nFreeBytesAvailable = fs::space(GetDataDir()).available; // Check for nMinDiskSpace bytes (currently 50MB) if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes) return AbortNode("Disk space is low!", _("Error: Disk space is low!")); return true; } static FILE *OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) { if (pos.IsNull()) { return nullptr; } fs::path path = GetBlockPosFilename(pos, prefix); fs::create_directories(path.parent_path()); FILE *file = fsbridge::fopen(path, "rb+"); if (!file && !fReadOnly) { file = fsbridge::fopen(path, "wb+"); } if (!file) { LogPrintf("Unable to open file %s\n", path.string()); return nullptr; } if (pos.nPos) { if (fseek(file, pos.nPos, SEEK_SET)) { LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); fclose(file); return nullptr; } } return file; } FILE *OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) { return OpenDiskFile(pos, "blk", fReadOnly); } /** Open an undo file (rev?????.dat) */ static FILE *OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { return OpenDiskFile(pos, "rev", fReadOnly); } fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) { return GetDataDir() / "blocks" / strprintf("%s%05u.dat", prefix, pos.nFile); } CBlockIndex *InsertBlockIndex(uint256 hash) { if (hash.IsNull()) return nullptr; // Return existing BlockMap::iterator mi = mapBlockIndex.find(hash); if (mi != mapBlockIndex.end()) return (*mi).second; // Create new CBlockIndex *pindexNew = new CBlockIndex(); if (!pindexNew) throw std::runtime_error(std::string(__func__) + ": new CBlockIndex failed"); mi = mapBlockIndex.insert(std::make_pair(hash, pindexNew)).first; pindexNew->phashBlock = &((*mi).first); return pindexNew; } static bool LoadBlockIndexDB(const CChainParams &chainparams) { if (!pblocktree->LoadBlockIndexGuts(InsertBlockIndex)) return false; boost::this_thread::interruption_point(); // Calculate nChainWork std::vector> vSortedByHeight; vSortedByHeight.reserve(mapBlockIndex.size()); for (const std::pair &item : mapBlockIndex) { CBlockIndex *pindex = item.second; vSortedByHeight.push_back(std::make_pair(pindex->nHeight, pindex)); } sort(vSortedByHeight.begin(), vSortedByHeight.end()); for (const std::pair &item : vSortedByHeight) { CBlockIndex *pindex = item.second; pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime); // We can link the chain of blocks for which we've received transactions // at some point. Pruned nodes may have deleted the block. if (pindex->nTx > 0) { if (pindex->pprev) { if (pindex->pprev->nChainTx) { pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; } else { pindex->nChainTx = 0; mapBlocksUnlinked.insert( std::make_pair(pindex->pprev, pindex)); } } else { pindex->nChainTx = pindex->nTx; } } if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->nChainTx || pindex->pprev == nullptr)) { setBlockIndexCandidates.insert(pindex); } if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork)) { pindexBestInvalid = pindex; } if (pindex->pprev) { pindex->BuildSkip(); } if (pindex->IsValid(BLOCK_VALID_TREE) && (pindexBestHeader == nullptr || CBlockIndexWorkComparator()(pindexBestHeader, pindex))) { pindexBestHeader = pindex; } } // Load block file info pblocktree->ReadLastBlockFile(nLastBlockFile); vinfoBlockFile.resize(nLastBlockFile + 1); LogPrintf("%s: last block file = %i\n", __func__, nLastBlockFile); for (int nFile = 0; nFile <= nLastBlockFile; nFile++) { pblocktree->ReadBlockFileInfo(nFile, vinfoBlockFile[nFile]); } LogPrintf("%s: last block file info: %s\n", __func__, vinfoBlockFile[nLastBlockFile].ToString()); for (int nFile = nLastBlockFile + 1; true; nFile++) { CBlockFileInfo info; if (pblocktree->ReadBlockFileInfo(nFile, info)) { vinfoBlockFile.push_back(info); } else { break; } } // Check presence of blk files LogPrintf("Checking all blk files are present...\n"); std::set setBlkDataFiles; for (const std::pair &item : mapBlockIndex) { CBlockIndex *pindex = item.second; if (pindex->nStatus & BLOCK_HAVE_DATA) { setBlkDataFiles.insert(pindex->nFile); } } for (std::set::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) { CDiskBlockPos pos(*it, 0); if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION) .IsNull()) { return false; } } // Check whether we have ever pruned block & undo files pblocktree->ReadFlag("prunedblockfiles", fHavePruned); if (fHavePruned) { LogPrintf( "LoadBlockIndexDB(): Block files have previously been pruned\n"); } // Check whether we need to continue reindexing bool fReindexing = false; pblocktree->ReadReindexing(fReindexing); fReindex |= fReindexing; // Check whether we have a transaction index pblocktree->ReadFlag("txindex", fTxIndex); LogPrintf("%s: transaction index %s\n", __func__, fTxIndex ? "enabled" : "disabled"); return true; } void LoadChainTip(const CChainParams &chainparams) { if (chainActive.Tip() && chainActive.Tip()->GetBlockHash() == pcoinsTip->GetBestBlock()) { return; } // Load pointer to end of best chain BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); if (it == mapBlockIndex.end()) { return; } chainActive.SetTip(it->second); PruneBlockIndexCandidates(); LogPrintf( "Loaded best chain: hashBestChain=%s height=%d date=%s progress=%f\n", chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), GuessVerificationProgress(chainparams.TxData(), chainActive.Tip())); } CVerifyDB::CVerifyDB() { uiInterface.ShowProgress(_("Verifying blocks..."), 0); } CVerifyDB::~CVerifyDB() { uiInterface.ShowProgress("", 100); } bool CVerifyDB::VerifyDB(const Config &config, CCoinsView *coinsview, int nCheckLevel, int nCheckDepth) { LOCK(cs_main); if (chainActive.Tip() == nullptr || chainActive.Tip()->pprev == nullptr) { return true; } // Verify blocks in the best chain if (nCheckDepth <= 0) { // suffices until the year 19000 nCheckDepth = 1000000000; } if (nCheckDepth > chainActive.Height()) { nCheckDepth = chainActive.Height(); } nCheckLevel = std::max(0, std::min(4, nCheckLevel)); LogPrintf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); CCoinsViewCache coins(coinsview); CBlockIndex *pindexState = chainActive.Tip(); CBlockIndex *pindexFailure = nullptr; int nGoodTransactions = 0; CValidationState state; int reportDone = 0; LogPrintf("[0%%]..."); for (CBlockIndex *pindex = chainActive.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { boost::this_thread::interruption_point(); int percentageDone = std::max( 1, std::min( 99, (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); if (reportDone < percentageDone / 10) { // report every 10% step LogPrintf("[%d%%]...", percentageDone); reportDone = percentageDone / 10; } uiInterface.ShowProgress(_("Verifying blocks..."), percentageDone); if (pindex->nHeight < chainActive.Height() - nCheckDepth) { break; } if (fPruneMode && !(pindex->nStatus & BLOCK_HAVE_DATA)) { // If pruning, only go back as far as we have data. LogPrintf("VerifyDB(): block verification stopping at height %d " "(pruning, no data)\n", pindex->nHeight); break; } CBlock block; // check level 0: read from disk if (!ReadBlockFromDisk(block, pindex, config)) { return error( "VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } // check level 1: verify block validity if (nCheckLevel >= 1 && !CheckBlock(config, block, state)) { return error("%s: *** found bad block at %d, hash=%s (%s)\n", __func__, pindex->nHeight, pindex->GetBlockHash().ToString(), FormatStateMessage(state)); } // check level 2: verify undo validity if (nCheckLevel >= 2 && pindex) { CBlockUndo undo; CDiskBlockPos pos = pindex->GetUndoPos(); if (!pos.IsNull()) { if (!UndoReadFromDisk(undo, pos, pindex->pprev->GetBlockHash())) { return error( "VerifyDB(): *** found bad undo data at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); } } } // check level 3: check for inconsistencies during memory-only // disconnect of tip blocks if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { assert(coins.GetBestBlock() == pindex->GetBlockHash()); DisconnectResult res = DisconnectBlock(block, pindex, coins); if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in " "block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } pindexState = pindex->pprev; if (res == DISCONNECT_UNCLEAN) { nGoodTransactions = 0; pindexFailure = pindex; } else { nGoodTransactions += block.vtx.size(); } } if (ShutdownRequested()) { return true; } } if (pindexFailure) { return error("VerifyDB(): *** coin database inconsistencies found " "(last %i blocks, %i good transactions before that)\n", chainActive.Height() - pindexFailure->nHeight + 1, nGoodTransactions); } // check level 4: try reconnecting blocks if (nCheckLevel >= 4) { CBlockIndex *pindex = pindexState; while (pindex != chainActive.Tip()) { boost::this_thread::interruption_point(); uiInterface.ShowProgress( _("Verifying blocks..."), std::max(1, std::min(99, 100 - (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * 50)))); pindex = chainActive.Next(pindex); CBlock block; if (!ReadBlockFromDisk(block, pindex, config)) { return error( "VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } if (!ConnectBlock(config, block, state, pindex, coins)) { return error( "VerifyDB(): *** found unconnectable block at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } } } LogPrintf("[DONE].\n"); LogPrintf("No coin database inconsistencies in last %i blocks (%i " "transactions)\n", chainActive.Height() - pindexState->nHeight, nGoodTransactions); return true; } /** * Apply the effects of a block on the utxo cache, ignoring that it may already * have been applied. */ static bool RollforwardBlock(const CBlockIndex *pindex, CCoinsViewCache &inputs, const Config &config) { // TODO: merge with ConnectBlock CBlock block; if (!ReadBlockFromDisk(block, pindex, config)) { return error("ReplayBlock(): ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } for (const CTransactionRef &tx : block.vtx) { if (!tx->IsCoinBase()) { for (const CTxIn &txin : tx->vin) { inputs.SpendCoin(txin.prevout); } } // Pass check = true as every addition may be an overwrite. AddCoins(inputs, *tx, pindex->nHeight, true); } return true; } bool ReplayBlocks(const Config &config, CCoinsView *view) { LOCK(cs_main); CCoinsViewCache cache(view); std::vector hashHeads = view->GetHeadBlocks(); if (hashHeads.empty()) { // We're already in a consistent state. return true; } if (hashHeads.size() != 2) { return error("ReplayBlocks(): unknown inconsistent state"); } uiInterface.ShowProgress(_("Replaying blocks..."), 0); LogPrintf("Replaying blocks\n"); // Old tip during the interrupted flush. const CBlockIndex *pindexOld = nullptr; // New tip during the interrupted flush. const CBlockIndex *pindexNew; // Latest block common to both the old and the new tip. const CBlockIndex *pindexFork = nullptr; if (mapBlockIndex.count(hashHeads[0]) == 0) { return error( "ReplayBlocks(): reorganization to unknown block requested"); } pindexNew = mapBlockIndex[hashHeads[0]]; if (!hashHeads[1].IsNull()) { // The old tip is allowed to be 0, indicating it's the first flush. if (mapBlockIndex.count(hashHeads[1]) == 0) { return error( "ReplayBlocks(): reorganization from unknown block requested"); } pindexOld = mapBlockIndex[hashHeads[1]]; pindexFork = LastCommonAncestor(pindexOld, pindexNew); assert(pindexFork != nullptr); } // Rollback along the old branch. while (pindexOld != pindexFork) { if (pindexOld->nHeight > 0) { // Never disconnect the genesis block. CBlock block; if (!ReadBlockFromDisk(block, pindexOld, config)) { return error("RollbackBlock(): ReadBlockFromDisk() failed at " "%d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); } LogPrintf("Rolling back %s (%i)\n", pindexOld->GetBlockHash().ToString(), pindexOld->nHeight); DisconnectResult res = DisconnectBlock(block, pindexOld, cache); if (res == DISCONNECT_FAILED) { return error( "RollbackBlock(): DisconnectBlock failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); } // If DISCONNECT_UNCLEAN is returned, it means a non-existing UTXO // was deleted, or an existing UTXO was overwritten. It corresponds // to cases where the block-to-be-disconnect never had all its // operations applied to the UTXO set. However, as both writing a // UTXO and deleting a UTXO are idempotent operations, the result is // still a version of the UTXO set with the effects of that block // undone. } pindexOld = pindexOld->pprev; } // Roll forward from the forking point to the new tip. int nForkHeight = pindexFork ? pindexFork->nHeight : 0; for (int nHeight = nForkHeight + 1; nHeight <= pindexNew->nHeight; ++nHeight) { const CBlockIndex *pindex = pindexNew->GetAncestor(nHeight); LogPrintf("Rolling forward %s (%i)\n", pindex->GetBlockHash().ToString(), nHeight); if (!RollforwardBlock(pindex, cache, config)) { return false; } } cache.SetBestBlock(pindexNew->GetBlockHash()); cache.Flush(); uiInterface.ShowProgress("", 100); return true; } bool RewindBlockIndex(const Config &config) { LOCK(cs_main); const CChainParams ¶ms = config.GetChainParams(); int nHeight = chainActive.Height() + 1; // nHeight is now the height of the first insufficiently-validated block, or // tipheight + 1 CValidationState state; CBlockIndex *pindex = chainActive.Tip(); while (chainActive.Height() >= nHeight) { if (fPruneMode && !(chainActive.Tip()->nStatus & BLOCK_HAVE_DATA)) { // If pruning, don't try rewinding past the HAVE_DATA point; since // older blocks can't be served anyway, there's no need to walk // further, and trying to DisconnectTip() will fail (and require a // needless reindex/redownload of the blockchain). break; } if (!DisconnectTip(config, state, nullptr)) { return error( "RewindBlockIndex: unable to disconnect block at height %i", pindex->nHeight); } // Occasionally flush state to disk. if (!FlushStateToDisk(params, state, FLUSH_STATE_PERIODIC)) { return false; } } // Reduce validity flag and have-data flags. // We do this after actual disconnecting, otherwise we'll end up writing the // lack of data to disk before writing the chainstate, resulting in a // failure to continue if interrupted. for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { CBlockIndex *pindexIter = it->second; if (pindexIter->IsValid(BLOCK_VALID_TRANSACTIONS) && pindexIter->nChainTx) { setBlockIndexCandidates.insert(pindexIter); } } PruneBlockIndexCandidates(); CheckBlockIndex(params.GetConsensus()); if (!FlushStateToDisk(params, state, FLUSH_STATE_ALWAYS)) { return false; } return true; } // May NOT be used after any connections are up as much of the peer-processing // logic assumes a consistent block index state void UnloadBlockIndex() { LOCK(cs_main); setBlockIndexCandidates.clear(); chainActive.SetTip(nullptr); pindexBestInvalid = nullptr; pindexBestHeader = nullptr; mempool.clear(); mapBlocksUnlinked.clear(); vinfoBlockFile.clear(); nLastBlockFile = 0; nBlockSequenceId = 1; setDirtyBlockIndex.clear(); setDirtyFileInfo.clear(); versionbitscache.Clear(); for (int b = 0; b < VERSIONBITS_NUM_BITS; b++) { warningcache[b].clear(); } for (BlockMap::value_type &entry : mapBlockIndex) { delete entry.second; } mapBlockIndex.clear(); fHavePruned = false; } bool LoadBlockIndex(const CChainParams &chainparams) { // Load block index from databases if (!fReindex && !LoadBlockIndexDB(chainparams)) { return false; } return true; } bool InitBlockIndex(const Config &config) { LOCK(cs_main); // Check whether we're already initialized if (chainActive.Genesis() != nullptr) { return true; } // Use the provided setting for -txindex in the new database fTxIndex = gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX); pblocktree->WriteFlag("txindex", fTxIndex); LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the // one already on disk) if (!fReindex) { try { const CChainParams &chainparams = config.GetChainParams(); CBlock &block = const_cast(chainparams.GenesisBlock()); // Start new block file unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); CDiskBlockPos blockPos; CValidationState state; if (!FindBlockPos(state, blockPos, nBlockSize + 8, 0, block.GetBlockTime())) { return error("LoadBlockIndex(): FindBlockPos failed"); } if (!WriteBlockToDisk(block, blockPos, chainparams.DiskMagic())) { return error( "LoadBlockIndex(): writing genesis block to disk failed"); } CBlockIndex *pindex = AddToBlockIndex(block); if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) { return error("LoadBlockIndex(): genesis block not accepted"); } } catch (const std::runtime_error &e) { return error( "LoadBlockIndex(): failed to initialize block database: %s", e.what()); } } return true; } bool LoadExternalBlockFile(const Config &config, FILE *fileIn, CDiskBlockPos *dbp) { // Map of disk positions for blocks with unknown parent (only used for // reindex) static std::multimap mapBlocksUnknownParent; int64_t nStart = GetTimeMillis(); const CChainParams &chainparams = config.GetChainParams(); int nLoaded = 0; try { // This takes over fileIn and calls fclose() on it in the CBufferedFile // destructor. Make sure we have at least 2*MAX_TX_SIZE space in there // so any transaction can fit in the buffer. CBufferedFile blkdat(fileIn, 2 * MAX_TX_SIZE, MAX_TX_SIZE + 8, SER_DISK, CLIENT_VERSION); uint64_t nRewind = blkdat.GetPos(); while (!blkdat.eof()) { boost::this_thread::interruption_point(); blkdat.SetPos(nRewind); // Start one byte further next time, in case of failure. nRewind++; // Remove former limit. blkdat.SetLimit(); unsigned int nSize = 0; try { // Locate a header. uint8_t buf[CMessageHeader::MESSAGE_START_SIZE]; blkdat.FindByte(chainparams.DiskMagic()[0]); nRewind = blkdat.GetPos() + 1; blkdat >> FLATDATA(buf); if (memcmp(buf, std::begin(chainparams.DiskMagic()), CMessageHeader::MESSAGE_START_SIZE)) { continue; } // Read size. blkdat >> nSize; if (nSize < 80) { continue; } } catch (const std::exception &) { // No valid block header found; don't complain. break; } try { // read block uint64_t nBlockPos = blkdat.GetPos(); if (dbp) { dbp->nPos = nBlockPos; } blkdat.SetLimit(nBlockPos + nSize); blkdat.SetPos(nBlockPos); std::shared_ptr pblock = std::make_shared(); CBlock &block = *pblock; blkdat >> block; nRewind = blkdat.GetPos(); // detect out of order blocks, and store them for later uint256 hash = block.GetHash(); if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex.find(block.hashPrevBlock) == mapBlockIndex.end()) { LogPrint(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), block.hashPrevBlock.ToString()); if (dbp) { mapBlocksUnknownParent.insert( std::make_pair(block.hashPrevBlock, *dbp)); } continue; } // process in case the block isn't known yet if (mapBlockIndex.count(hash) == 0 || (mapBlockIndex[hash]->nStatus & BLOCK_HAVE_DATA) == 0) { LOCK(cs_main); CValidationState state; if (AcceptBlock(config, pblock, state, nullptr, true, dbp, nullptr)) { nLoaded++; } if (state.IsError()) { break; } } else if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex[hash]->nHeight % 1000 == 0) { LogPrint( BCLog::REINDEX, "Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight); } // Activate the genesis block so normal node progress can // continue if (hash == chainparams.GetConsensus().hashGenesisBlock) { CValidationState state; if (!ActivateBestChain(config, state)) { break; } } NotifyHeaderTip(); // Recursively process earlier encountered successors of this // block std::deque queue; queue.push_back(hash); while (!queue.empty()) { uint256 head = queue.front(); queue.pop_front(); std::pair::iterator, std::multimap::iterator> range = mapBlocksUnknownParent.equal_range(head); while (range.first != range.second) { std::multimap::iterator it = range.first; std::shared_ptr pblockrecursive = std::make_shared(); if (ReadBlockFromDisk(*pblockrecursive, it->second, config)) { LogPrint( BCLog::REINDEX, "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), head.ToString()); LOCK(cs_main); CValidationState dummy; if (AcceptBlock(config, pblockrecursive, dummy, nullptr, true, &it->second, nullptr)) { nLoaded++; queue.push_back(pblockrecursive->GetHash()); } } range.first++; mapBlocksUnknownParent.erase(it); NotifyHeaderTip(); } } } catch (const std::exception &e) { LogPrintf("%s: Deserialize or I/O error - %s\n", __func__, e.what()); } } } catch (const std::runtime_error &e) { AbortNode(std::string("System error: ") + e.what()); } if (nLoaded > 0) { LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, GetTimeMillis() - nStart); } return nLoaded > 0; } static void CheckBlockIndex(const Consensus::Params &consensusParams) { if (!fCheckBlockIndex) { return; } LOCK(cs_main); // During a reindex, we read the genesis block and call CheckBlockIndex // before ActivateBestChain, so we have the genesis block in mapBlockIndex // but no active chain. (A few of the tests when iterating the block tree // require that chainActive has been initialized.) if (chainActive.Height() < 0) { assert(mapBlockIndex.size() <= 1); return; } // Build forward-pointing map of the entire block tree. std::multimap forward; for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { forward.insert(std::make_pair(it->second->pprev, it->second)); } assert(forward.size() == mapBlockIndex.size()); std::pair::iterator, std::multimap::iterator> rangeGenesis = forward.equal_range(nullptr); CBlockIndex *pindex = rangeGenesis.first->second; rangeGenesis.first++; // There is only one index entry with parent nullptr. assert(rangeGenesis.first == rangeGenesis.second); // Iterate over the entire block tree, using depth-first search. // Along the way, remember whether there are blocks on the path from genesis // block being explored which are the first to have certain properties. size_t nNodes = 0; int nHeight = 0; // Oldest ancestor of pindex which is invalid. CBlockIndex *pindexFirstInvalid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_HAVE_DATA. CBlockIndex *pindexFirstMissing = nullptr; // Oldest ancestor of pindex for which nTx == 0. CBlockIndex *pindexFirstNeverProcessed = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_TREE // (regardless of being valid or not). CBlockIndex *pindexFirstNotTreeValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS // (regardless of being valid or not). CBlockIndex *pindexFirstNotTransactionsValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN // (regardless of being valid or not). CBlockIndex *pindexFirstNotChainValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_SCRIPTS // (regardless of being valid or not). CBlockIndex *pindexFirstNotScriptsValid = nullptr; while (pindex != nullptr) { nNodes++; if (pindexFirstInvalid == nullptr && pindex->nStatus & BLOCK_FAILED_VALID) { pindexFirstInvalid = pindex; } if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA)) { pindexFirstMissing = pindex; } if (pindexFirstNeverProcessed == nullptr && pindex->nTx == 0) { pindexFirstNeverProcessed = pindex; } if (pindex->pprev != nullptr && pindexFirstNotTreeValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) { pindexFirstNotTreeValid = pindex; } if (pindex->pprev != nullptr && pindexFirstNotTransactionsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) { pindexFirstNotTransactionsValid = pindex; } if (pindex->pprev != nullptr && pindexFirstNotChainValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) { pindexFirstNotChainValid = pindex; } if (pindex->pprev != nullptr && pindexFirstNotScriptsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) { pindexFirstNotScriptsValid = pindex; } // Begin: actual consistency checks. if (pindex->pprev == nullptr) { // Genesis block checks. // Genesis block's hash must match. assert(pindex->GetBlockHash() == consensusParams.hashGenesisBlock); // The current active chain's genesis block must be this block. assert(pindex == chainActive.Genesis()); } if (pindex->nChainTx == 0) { // nSequenceId can't be set positive for blocks that aren't linked // (negative is used for preciousblock) assert(pindex->nSequenceId <= 0); } // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or // not pruning has occurred). HAVE_DATA is only equivalent to nTx > 0 // (or VALID_TRANSACTIONS) if no pruning has occurred. if (!fHavePruned) { // If we've never pruned, then HAVE_DATA should be equivalent to nTx // > 0 assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); assert(pindexFirstMissing == pindexFirstNeverProcessed); } else { // If we have pruned, then we can only say that HAVE_DATA implies // nTx > 0 if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0); } if (pindex->nStatus & BLOCK_HAVE_UNDO) { assert(pindex->nStatus & BLOCK_HAVE_DATA); } // This is pruning-independent. assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // All parents having had data (at some point) is equivalent to all // parents being VALID_TRANSACTIONS, which is equivalent to nChainTx // being set. // nChainTx != 0 is used to signal that all parent blocks have been // processed (but may have been pruned). assert((pindexFirstNeverProcessed != nullptr) == (pindex->nChainTx == 0)); assert((pindexFirstNotTransactionsValid != nullptr) == (pindex->nChainTx == 0)); // nHeight must be consistent. assert(pindex->nHeight == nHeight); // For every block except the genesis block, the chainwork must be // larger than the parent's. assert(pindex->pprev == nullptr || pindex->nChainWork >= pindex->pprev->nChainWork); // The pskip pointer must point back for all but the first 2 blocks. assert(nHeight < 2 || (pindex->pskip && (pindex->pskip->nHeight < nHeight))); // All mapBlockIndex entries must at least be TREE valid assert(pindexFirstNotTreeValid == nullptr); if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TREE) { // TREE valid implies all parents are TREE valid assert(pindexFirstNotTreeValid == nullptr); } if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_CHAIN) { // CHAIN valid implies all parents are CHAIN valid assert(pindexFirstNotChainValid == nullptr); } if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_SCRIPTS) { // SCRIPTS valid implies all parents are SCRIPTS valid assert(pindexFirstNotScriptsValid == nullptr); } if (pindexFirstInvalid == nullptr) { // Checks for not-invalid blocks. // The failed mask cannot be set for blocks without invalid parents. assert((pindex->nStatus & BLOCK_FAILED_MASK) == 0); } if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstNeverProcessed == nullptr) { if (pindexFirstInvalid == nullptr) { // If this block sorts at least as good as the current tip and // is valid and we have all data for its parents, it must be in // setBlockIndexCandidates. chainActive.Tip() must also be there // even if some data has been pruned. if (pindexFirstMissing == nullptr || pindex == chainActive.Tip()) { assert(setBlockIndexCandidates.count(pindex)); } // If some parent is missing, then it could be that this block // was in setBlockIndexCandidates but had to be removed because // of the missing data. In this case it must be in // mapBlocksUnlinked -- see test below. } } else { // If this block sorts worse than the current tip or some ancestor's // block has never been seen, it cannot be in // setBlockIndexCandidates. assert(setBlockIndexCandidates.count(pindex) == 0); } // Check whether this block is in mapBlocksUnlinked. std::pair::iterator, std::multimap::iterator> rangeUnlinked = mapBlocksUnlinked.equal_range(pindex->pprev); bool foundInUnlinked = false; while (rangeUnlinked.first != rangeUnlinked.second) { assert(rangeUnlinked.first->first == pindex->pprev); if (rangeUnlinked.first->second == pindex) { foundInUnlinked = true; break; } rangeUnlinked.first++; } if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed != nullptr && pindexFirstInvalid == nullptr) { // If this block has block data available, some parent was never // received, and has no invalid parents, it must be in // mapBlocksUnlinked. assert(foundInUnlinked); } if (!(pindex->nStatus & BLOCK_HAVE_DATA)) { // Can't be in mapBlocksUnlinked if we don't HAVE_DATA assert(!foundInUnlinked); } if (pindexFirstMissing == nullptr) { // We aren't missing data for any parent -- cannot be in // mapBlocksUnlinked. assert(!foundInUnlinked); } if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == nullptr && pindexFirstMissing != nullptr) { // We HAVE_DATA for this block, have received data for all parents // at some point, but we're currently missing data for some parent. // We must have pruned. assert(fHavePruned); // This block may have entered mapBlocksUnlinked if: // - it has a descendant that at some point had more work than the // tip, and // - we tried switching to that descendant but were missing // data for some intermediate block between chainActive and the // tip. // So if this block is itself better than chainActive.Tip() and it // wasn't in // setBlockIndexCandidates, then it must be in mapBlocksUnlinked. if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && setBlockIndexCandidates.count(pindex) == 0) { if (pindexFirstInvalid == nullptr) { assert(foundInUnlinked); } } } // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // // Perhaps too slow // End: actual consistency checks. // Try descending into the first subnode. std::pair::iterator, std::multimap::iterator> range = forward.equal_range(pindex); if (range.first != range.second) { // A subnode was found. pindex = range.first->second; nHeight++; continue; } // This is a leaf node. Move upwards until we reach a node of which we // have not yet visited the last child. while (pindex) { // We are going to either move to a parent or a sibling of pindex. // If pindex was the first with a certain property, unset the // corresponding variable. if (pindex == pindexFirstInvalid) { pindexFirstInvalid = nullptr; } if (pindex == pindexFirstMissing) { pindexFirstMissing = nullptr; } if (pindex == pindexFirstNeverProcessed) { pindexFirstNeverProcessed = nullptr; } if (pindex == pindexFirstNotTreeValid) { pindexFirstNotTreeValid = nullptr; } if (pindex == pindexFirstNotTransactionsValid) { pindexFirstNotTransactionsValid = nullptr; } if (pindex == pindexFirstNotChainValid) { pindexFirstNotChainValid = nullptr; } if (pindex == pindexFirstNotScriptsValid) { pindexFirstNotScriptsValid = nullptr; } // Find our parent. CBlockIndex *pindexPar = pindex->pprev; // Find which child we just visited. std::pair::iterator, std::multimap::iterator> rangePar = forward.equal_range(pindexPar); while (rangePar.first->second != pindex) { // Our parent must have at least the node we're coming from as // child. assert(rangePar.first != rangePar.second); rangePar.first++; } // Proceed to the next one. rangePar.first++; if (rangePar.first != rangePar.second) { // Move to the sibling. pindex = rangePar.first->second; break; } else { // Move up further. pindex = pindexPar; nHeight--; continue; } } } // Check that we actually traversed the entire map. assert(nNodes == forward.size()); } std::string CBlockFileInfo::ToString() const { return strprintf( "CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, DateTimeStrFormat("%Y-%m-%d", nTimeFirst), DateTimeStrFormat("%Y-%m-%d", nTimeLast)); } CBlockFileInfo *GetBlockFileInfo(size_t n) { return &vinfoBlockFile.at(n); } ThresholdState VersionBitsTipState(const Consensus::Params ¶ms, Consensus::DeploymentPos pos) { LOCK(cs_main); return VersionBitsState(chainActive.Tip(), params, pos, versionbitscache); } int VersionBitsTipStateSinceHeight(const Consensus::Params ¶ms, Consensus::DeploymentPos pos) { LOCK(cs_main); return VersionBitsStateSinceHeight(chainActive.Tip(), params, pos, versionbitscache); } static const uint64_t MEMPOOL_DUMP_VERSION = 1; bool LoadMempool(const Config &config) { int64_t nExpiryTimeout = gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; FILE *filestr = fsbridge::fopen(GetDataDir() / "mempool.dat", "rb"); CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); if (file.IsNull()) { LogPrintf( "Failed to open mempool file from disk. Continuing anyway.\n"); return false; } int64_t count = 0; int64_t skipped = 0; int64_t failed = 0; int64_t nNow = GetTime(); try { uint64_t version; file >> version; if (version != MEMPOOL_DUMP_VERSION) { return false; } uint64_t num; file >> num; double prioritydummy = 0; while (num--) { CTransactionRef tx; int64_t nTime; int64_t nFeeDelta; file >> tx; file >> nTime; file >> nFeeDelta; Amount amountdelta(nFeeDelta); if (amountdelta != Amount(0)) { mempool.PrioritiseTransaction(tx->GetId(), tx->GetId().ToString(), prioritydummy, amountdelta); } CValidationState state; if (nTime + nExpiryTimeout > nNow) { LOCK(cs_main); AcceptToMemoryPoolWithTime(config, mempool, state, tx, true, nullptr, nTime); if (state.IsValid()) { ++count; } else { ++failed; } } else { ++skipped; } if (ShutdownRequested()) return false; } std::map mapDeltas; file >> mapDeltas; for (const auto &i : mapDeltas) { mempool.PrioritiseTransaction(i.first, i.first.ToString(), prioritydummy, i.second); } } catch (const std::exception &e) { LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing " "anyway.\n", e.what()); return false; } LogPrintf("Imported mempool transactions from disk: %i successes, %i " "failed, %i expired\n", count, failed, skipped); return true; } void DumpMempool(void) { int64_t start = GetTimeMicros(); std::map mapDeltas; std::vector vinfo; { LOCK(mempool.cs); for (const auto &i : mempool.mapDeltas) { mapDeltas[i.first] = i.second.second; } vinfo = mempool.infoAll(); } int64_t mid = GetTimeMicros(); try { FILE *filestr = fsbridge::fopen(GetDataDir() / "mempool.dat.new", "wb"); if (!filestr) { return; } CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); uint64_t version = MEMPOOL_DUMP_VERSION; file << version; file << (uint64_t)vinfo.size(); for (const auto &i : vinfo) { file << *(i.tx); file << (int64_t)i.nTime; file << (int64_t)i.nFeeDelta.GetSatoshis(); mapDeltas.erase(i.tx->GetId()); } file << mapDeltas; FileCommit(file.Get()); file.fclose(); RenameOver(GetDataDir() / "mempool.dat.new", GetDataDir() / "mempool.dat"); int64_t last = GetTimeMicros(); LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", (mid - start) * 0.000001, (last - mid) * 0.000001); } catch (const std::exception &e) { LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); } } //! Guess how far we are in the verification process at the given block index double GuessVerificationProgress(const ChainTxData &data, CBlockIndex *pindex) { if (pindex == nullptr) { return 0.0; } int64_t nNow = time(nullptr); double fTxTotal; if (pindex->nChainTx <= data.nTxCount) { fTxTotal = data.nTxCount + (nNow - data.nTime) * data.dTxRate; } else { fTxTotal = pindex->nChainTx + (nNow - pindex->GetBlockTime()) * data.dTxRate; } return pindex->nChainTx / fTxTotal; } class CMainCleanup { public: CMainCleanup() {} ~CMainCleanup() { // block headers BlockMap::iterator it1 = mapBlockIndex.begin(); for (; it1 != mapBlockIndex.end(); it1++) delete (*it1).second; mapBlockIndex.clear(); } } instance_of_cmaincleanup;