diff --git a/src/coins.cpp b/src/coins.cpp index 35834a080..ac3808680 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -1,335 +1,342 @@ // Copyright (c) 2012-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } BlockHash CCoinsView::GetBestBlock() const { return BlockHash(); } std::vector CCoinsView::GetHeadBlocks() const { return std::vector(); } bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const BlockHash &hashBlock) { return false; } CCoinsViewCursor *CCoinsView::Cursor() const { return nullptr; } bool CCoinsView::HaveCoin(const COutPoint &outpoint) const { Coin coin; return GetCoin(outpoint, coin); } CCoinsViewBacked::CCoinsViewBacked(CCoinsView *viewIn) : base(viewIn) {} bool CCoinsViewBacked::GetCoin(const COutPoint &outpoint, Coin &coin) const { return base->GetCoin(outpoint, coin); } bool CCoinsViewBacked::HaveCoin(const COutPoint &outpoint) const { return base->HaveCoin(outpoint); } BlockHash CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); } std::vector CCoinsViewBacked::GetHeadBlocks() const { return base->GetHeadBlocks(); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const BlockHash &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); } size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); } SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), cachedCoinsUsage(0) {} size_t CCoinsViewCache::DynamicMemoryUsage() const { return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; } CCoinsMap::iterator CCoinsViewCache::FetchCoin(const COutPoint &outpoint) const { CCoinsMap::iterator it = cacheCoins.find(outpoint); if (it != cacheCoins.end()) { return it; } Coin tmp; if (!base->GetCoin(outpoint, tmp)) { return cacheCoins.end(); } CCoinsMap::iterator ret = cacheCoins .emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::forward_as_tuple(std::move(tmp))) .first; if (ret->second.coin.IsSpent()) { // The parent only has an empty entry for this outpoint; we can consider // our version as fresh. ret->second.flags = CCoinsCacheEntry::FRESH; } cachedCoinsUsage += ret->second.coin.DynamicMemoryUsage(); return ret; } bool CCoinsViewCache::GetCoin(const COutPoint &outpoint, Coin &coin) const { CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) { return false; } coin = it->second.coin; return !coin.IsSpent(); } void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin coin, bool possible_overwrite) { assert(!coin.IsSpent()); if (coin.GetTxOut().scriptPubKey.IsUnspendable()) { return; } CCoinsMap::iterator it; bool inserted; std::tie(it, inserted) = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::tuple<>()); bool fresh = false; if (!inserted) { cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); } if (!possible_overwrite) { if (!it->second.coin.IsSpent()) { throw std::logic_error("Attempted to overwrite an unspent coin " "(when possible_overwrite is false)"); } // If the coin exists in this cache as a spent coin and is DIRTY, then // its spentness hasn't been flushed to the parent cache. We're // re-adding the coin to this cache now but we can't mark it as FRESH. // If we mark it FRESH and then spend it before the cache is flushed // we would remove it from this cache and would never flush spentness // to the parent cache. // // Re-adding a spent coin can happen in the case of a re-org (the coin // is 'spent' when the block adding it is disconnected and then // re-added when it is also added in a newly connected block). // // If the coin doesn't exist in the current cache, or is spent but not // DIRTY, then it can be marked FRESH. fresh = !(it->second.flags & CCoinsCacheEntry::DIRTY); } it->second.coin = std::move(coin); it->second.flags |= CCoinsCacheEntry::DIRTY | (fresh ? CCoinsCacheEntry::FRESH : 0); cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); } void AddCoins(CCoinsViewCache &cache, const CTransaction &tx, int nHeight, bool check_for_overwrite) { bool fCoinbase = tx.IsCoinBase(); const TxId txid = tx.GetId(); for (size_t i = 0; i < tx.vout.size(); ++i) { const COutPoint outpoint(txid, i); bool overwrite = check_for_overwrite ? cache.HaveCoin(outpoint) : fCoinbase; // Coinbase transactions can always be overwritten, // in order to correctly deal with the pre-BIP30 occurrences of // duplicate coinbase transactions. cache.AddCoin(outpoint, Coin(tx.vout[i], nHeight, fCoinbase), overwrite); } } bool CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin *moveout) { CCoinsMap::iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) { return false; } cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); if (moveout) { *moveout = std::move(it->second.coin); } if (it->second.flags & CCoinsCacheEntry::FRESH) { cacheCoins.erase(it); } else { it->second.flags |= CCoinsCacheEntry::DIRTY; it->second.coin.Clear(); } return true; } static const Coin coinEmpty; const Coin &CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const { CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) { return coinEmpty; } return it->second.coin; } bool CCoinsViewCache::HaveCoin(const COutPoint &outpoint) const { CCoinsMap::const_iterator it = FetchCoin(outpoint); return it != cacheCoins.end() && !it->second.coin.IsSpent(); } bool CCoinsViewCache::HaveCoinInCache(const COutPoint &outpoint) const { CCoinsMap::const_iterator it = cacheCoins.find(outpoint); return (it != cacheCoins.end() && !it->second.coin.IsSpent()); } BlockHash CCoinsViewCache::GetBestBlock() const { if (hashBlock.IsNull()) { hashBlock = base->GetBestBlock(); } return hashBlock; } void CCoinsViewCache::SetBestBlock(const BlockHash &hashBlockIn) { hashBlock = hashBlockIn; } bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const BlockHash &hashBlockIn) { for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); it = mapCoins.erase(it)) { // Ignore non-dirty entries (optimization). if (!(it->second.flags & CCoinsCacheEntry::DIRTY)) { continue; } CCoinsMap::iterator itUs = cacheCoins.find(it->first); if (itUs == cacheCoins.end()) { // The parent cache does not have an entry, while the child cache // does. We can ignore it if it's both spent and FRESH in the child if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coin.IsSpent())) { // Create the coin in the parent cache, move the data up // and mark it as dirty. CCoinsCacheEntry &entry = cacheCoins[it->first]; entry.coin = std::move(it->second.coin); cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); entry.flags = CCoinsCacheEntry::DIRTY; // We can mark it FRESH in the parent if it was FRESH in the // child. Otherwise it might have just been flushed from the // parent's cache and already exist in the grandparent if (it->second.flags & CCoinsCacheEntry::FRESH) { entry.flags |= CCoinsCacheEntry::FRESH; } } } else { // Found the entry in the parent cache if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coin.IsSpent()) { // The coin was marked FRESH in the child cache, but the coin // exists in the parent cache. If this ever happens, it means // the FRESH flag was misapplied and there is a logic error in // the calling code. throw std::logic_error("FRESH flag misapplied to coin that " "exists in parent cache"); } if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coin.IsSpent()) { // The grandparent cache does not have an entry, and the coin // has been spent. We can just delete it from the parent cache. cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); cacheCoins.erase(itUs); } else { // A normal modification. cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); itUs->second.coin = std::move(it->second.coin); cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); itUs->second.flags |= CCoinsCacheEntry::DIRTY; // NOTE: It isn't safe to mark the coin as FRESH in the parent // cache. If it already existed and was spent in the parent // cache then marking it FRESH would prevent that spentness // from being flushed to the grandparent. } } } hashBlock = hashBlockIn; return true; } bool CCoinsViewCache::Flush() { bool fOk = base->BatchWrite(cacheCoins, hashBlock); cacheCoins.clear(); cachedCoinsUsage = 0; return fOk; } void CCoinsViewCache::Uncache(const COutPoint &outpoint) { CCoinsMap::iterator it = cacheCoins.find(outpoint); if (it != cacheCoins.end() && it->second.flags == 0) { cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); cacheCoins.erase(it); } } unsigned int CCoinsViewCache::GetCacheSize() const { return cacheCoins.size(); } bool CCoinsViewCache::HaveInputs(const CTransaction &tx) const { if (tx.IsCoinBase()) { return true; } for (size_t i = 0; i < tx.vin.size(); i++) { if (!HaveCoin(tx.vin[i].prevout)) { return false; } } return true; } +void CCoinsViewCache::ReallocateCache() { + // Cache should be empty when we're calling this. + assert(cacheCoins.size() == 0); + cacheCoins.~CCoinsMap(); + ::new (&cacheCoins) CCoinsMap(); +} + // TODO: merge with similar definition in undo.h. static const size_t MAX_OUTPUTS_PER_TX = MAX_TX_SIZE / ::GetSerializeSize(CTxOut(), PROTOCOL_VERSION); const Coin &AccessByTxid(const CCoinsViewCache &view, const TxId &txid) { for (uint32_t n = 0; n < MAX_OUTPUTS_PER_TX; n++) { const Coin &alternate = view.AccessCoin(COutPoint(txid, n)); if (!alternate.IsSpent()) { return alternate; } } return coinEmpty; } bool CCoinsViewErrorCatcher::GetCoin(const COutPoint &outpoint, Coin &coin) const { try { return CCoinsViewBacked::GetCoin(outpoint, coin); } catch (const std::runtime_error &e) { for (auto f : m_err_callbacks) { f(); } LogPrintf("Error reading from database: %s\n", e.what()); // Starting the shutdown sequence and returning false to the caller // would be interpreted as 'entry not found' (as opposed to unable to // read data), and could lead to invalid interpretation. Just exit // immediately, as we can't continue anyway, and all writes should be // atomic. std::abort(); } } diff --git a/src/coins.h b/src/coins.h index 1e0174741..be0ceb496 100644 --- a/src/coins.h +++ b/src/coins.h @@ -1,368 +1,376 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_COINS_H #define BITCOIN_COINS_H #include #include #include #include #include #include #include #include #include /** * A UTXO entry. * * Serialized format: * - VARINT((coinbase ? 1 : 0) | (height << 1)) * - the non-spent CTxOut (via TxOutCompression) */ class Coin { //! Unspent transaction output. CTxOut out; //! Whether containing transaction was a coinbase and height at which the //! transaction was included into a block. uint32_t nHeightAndIsCoinBase; public: //! Empty constructor Coin() : nHeightAndIsCoinBase(0) {} //! Constructor from a CTxOut and height/coinbase information. Coin(CTxOut outIn, uint32_t nHeightIn, bool IsCoinbase) : out(std::move(outIn)), nHeightAndIsCoinBase((nHeightIn << 1) | IsCoinbase) {} uint32_t GetHeight() const { return nHeightAndIsCoinBase >> 1; } bool IsCoinBase() const { return nHeightAndIsCoinBase & 0x01; } bool IsSpent() const { return out.IsNull(); } CTxOut &GetTxOut() { return out; } const CTxOut &GetTxOut() const { return out; } void Clear() { out.SetNull(); nHeightAndIsCoinBase = 0; } template void Serialize(Stream &s) const { assert(!IsSpent()); ::Serialize(s, VARINT(nHeightAndIsCoinBase)); ::Serialize(s, Using(out)); } template void Unserialize(Stream &s) { ::Unserialize(s, VARINT(nHeightAndIsCoinBase)); ::Unserialize(s, Using(out)); } size_t DynamicMemoryUsage() const { return memusage::DynamicUsage(out.scriptPubKey); } }; class SaltedOutpointHasher { private: /** Salt */ const uint64_t k0, k1; public: SaltedOutpointHasher(); /** * This *must* return size_t. With Boost 1.46 on 32-bit systems the * unordered_map will behave unpredictably if the custom hasher returns a * uint64_t, resulting in failures when syncing the chain (#4634). * Note: This information above might be outdated as the unordered map * container type has meanwhile been switched to the C++ standard library * implementation. * * Having the hash noexcept allows libstdc++'s unordered_map to recalculate * the hash during rehash, so it does not have to cache the value. This * reduces node's memory by sizeof(size_t). The required recalculation has * a slight performance penalty (around 1.6%), but this is compensated by * memory savings of about 9% which allow for a larger dbcache setting. * * @see * https://gcc.gnu.org/onlinedocs/gcc-9.2.0/libstdc++/manual/manual/unordered_associative.html */ size_t operator()(const COutPoint &outpoint) const noexcept { return SipHashUint256Extra(k0, k1, outpoint.GetTxId(), outpoint.GetN()); } }; /** * A Coin in one level of the coins database caching hierarchy. * * A coin can either be: * - unspent or spent (in which case the Coin object will be nulled out - see * Coin.Clear()) * - DIRTY or not DIRTY * - FRESH or not FRESH * * Out of these 2^3 = 8 states, only some combinations are valid: * - unspent, FRESH, DIRTY (e.g. a new coin created in the cache) * - unspent, not FRESH, DIRTY (e.g. a coin changed in the cache during a reorg) * - unspent, not FRESH, not DIRTY (e.g. an unspent coin fetched from the parent * cache) * - spent, FRESH, not DIRTY (e.g. a spent coin fetched from the parent cache) * - spent, not FRESH, DIRTY (e.g. a coin is spent and spentness needs to be * flushed to the parent) */ struct CCoinsCacheEntry { // The actual cached data. Coin coin; uint8_t flags; enum Flags { /** * DIRTY means the CCoinsCacheEntry is potentially different from the * version in the parent cache. Failure to mark a coin as DIRTY when * it is potentially different from the parent cache will cause a * consensus failure, since the coin's state won't get written to the * parent when the cache is flushed. */ DIRTY = (1 << 0), /** * FRESH means the parent cache does not have this coin or that it is a * spent coin in the parent cache. If a FRESH coin in the cache is * later spent, it can be deleted entirely and doesn't ever need to be * flushed to the parent. This is a performance optimization. Marking a * coin as FRESH when it exists unspent in the parent cache will cause a * consensus failure, since it might not be deleted from the parent * when this cache is flushed. */ FRESH = (1 << 1), }; CCoinsCacheEntry() : flags(0) {} explicit CCoinsCacheEntry(Coin coinIn) : coin(std::move(coinIn)), flags(0) {} }; typedef std::unordered_map CCoinsMap; /** Cursor for iterating over CoinsView state */ class CCoinsViewCursor { public: CCoinsViewCursor(const BlockHash &hashBlockIn) : hashBlock(hashBlockIn) {} virtual ~CCoinsViewCursor() {} virtual bool GetKey(COutPoint &key) const = 0; virtual bool GetValue(Coin &coin) const = 0; virtual unsigned int GetValueSize() const = 0; virtual bool Valid() const = 0; virtual void Next() = 0; //! Get best block at the time this cursor was created const BlockHash &GetBestBlock() const { return hashBlock; } private: BlockHash hashBlock; }; /** Abstract view on the open txout dataset. */ class CCoinsView { public: /** * Retrieve the Coin (unspent transaction output) for a given outpoint. * Returns true only when an unspent coin was found, which is returned in * coin. When false is returned, coin's value is unspecified. */ virtual bool GetCoin(const COutPoint &outpoint, Coin &coin) const; //! Just check whether a given outpoint is unspent. virtual bool HaveCoin(const COutPoint &outpoint) const; //! Retrieve the block hash whose state this CCoinsView currently represents virtual BlockHash GetBestBlock() const; //! Retrieve the range of blocks that may have been only partially written. //! If the database is in a consistent state, the result is the empty //! vector. //! Otherwise, a two-element vector is returned consisting of the new and //! the old block hash, in that order. virtual std::vector GetHeadBlocks() const; //! Do a bulk modification (multiple Coin changes + BestBlock change). //! The passed mapCoins can be modified. virtual bool BatchWrite(CCoinsMap &mapCoins, const BlockHash &hashBlock); //! Get a cursor to iterate over the whole state virtual CCoinsViewCursor *Cursor() const; //! As we use CCoinsViews polymorphically, have a virtual destructor virtual ~CCoinsView() {} //! Estimate database size (0 if not implemented) virtual size_t EstimateSize() const { return 0; } }; /** CCoinsView backed by another CCoinsView */ class CCoinsViewBacked : public CCoinsView { protected: CCoinsView *base; public: CCoinsViewBacked(CCoinsView *viewIn); bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; bool HaveCoin(const COutPoint &outpoint) const override; BlockHash GetBestBlock() const override; std::vector GetHeadBlocks() const override; void SetBackend(CCoinsView &viewIn); bool BatchWrite(CCoinsMap &mapCoins, const BlockHash &hashBlock) override; CCoinsViewCursor *Cursor() const override; size_t EstimateSize() const override; }; /** * CCoinsView that adds a memory cache for transactions to another CCoinsView */ class CCoinsViewCache : public CCoinsViewBacked { protected: /** * Make mutable so that we can "fill the cache" even from Get-methods * declared as "const". */ mutable BlockHash hashBlock; mutable CCoinsMap cacheCoins; /* Cached dynamic memory usage for the inner Coin objects. */ mutable size_t cachedCoinsUsage; public: CCoinsViewCache(CCoinsView *baseIn); /** * By deleting the copy constructor, we prevent accidentally using it when * one intends to create a cache on top of a base cache. */ CCoinsViewCache(const CCoinsViewCache &) = delete; // Standard CCoinsView methods bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; bool HaveCoin(const COutPoint &outpoint) const override; BlockHash GetBestBlock() const override; void SetBestBlock(const BlockHash &hashBlock); bool BatchWrite(CCoinsMap &mapCoins, const BlockHash &hashBlock) override; CCoinsViewCursor *Cursor() const override { throw std::logic_error( "CCoinsViewCache cursor iteration not supported."); } /** * Check if we have the given utxo already loaded in this cache. * The semantics are the same as HaveCoin(), but no calls to the backing * CCoinsView are made. */ bool HaveCoinInCache(const COutPoint &outpoint) const; /** * Return a reference to Coin in the cache, or coinEmpty if not found. * This is more efficient than GetCoin. * * Generally, do not hold the reference returned for more than a short * scope. While the current implementation allows for modifications to the * contents of the cache while holding the reference, this behavior should * not be relied on! To be safe, best to not hold the returned reference * through any other calls to this cache. */ const Coin &AccessCoin(const COutPoint &output) const; /** * Add a coin. Set possible_overwrite to true if an unspent version may * already exist in the cache. */ void AddCoin(const COutPoint &outpoint, Coin coin, bool possible_overwrite); /** * Spend a coin. Pass moveto in order to get the deleted data. * If no unspent output exists for the passed outpoint, this call has no * effect. */ bool SpendCoin(const COutPoint &outpoint, Coin *moveto = nullptr); /** * Push the modifications applied to this cache to its base. * Failure to call this method before destruction will cause the changes to * be forgotten. If false is returned, the state of this cache (and its * backing view) will be undefined. */ bool Flush(); /** * Removes the UTXO with the given outpoint from the cache, if it is not * modified. */ void Uncache(const COutPoint &outpoint); //! Calculate the size of the cache (in number of transaction outputs) unsigned int GetCacheSize() const; //! Calculate the size of the cache (in bytes) size_t DynamicMemoryUsage() const; //! Check whether all prevouts of the transaction are present in the UTXO //! set represented by this view bool HaveInputs(const CTransaction &tx) const; + //! Force a reallocation of the cache map. This is required when downsizing + //! the cache because the map's allocator may be hanging onto a lot of + //! memory despite having called .clear(). + //! + //! See: + //! https://stackoverflow.com/questions/42114044/how-to-release-unordered-map-memory + void ReallocateCache(); + private: /** * @note this is marked const, but may actually append to `cacheCoins`, * increasing memory usage. */ CCoinsMap::iterator FetchCoin(const COutPoint &outpoint) const; }; //! Utility function to add all of a transaction's outputs to a cache. //! When check is false, this assumes that overwrites are only possible for //! coinbase transactions. When check is true, the underlying view may be //! queried to determine whether an addition is an overwrite. // TODO: pass in a boolean to limit these possible overwrites to known // (pre-BIP34) cases. void AddCoins(CCoinsViewCache &cache, const CTransaction &tx, int nHeight, bool check = false); //! Utility function to find any unspent output with a given txid. //! This function can be quite expensive because in the event of a transaction //! which is not found in the cache, it can cause up to MAX_OUTPUTS_PER_BLOCK //! lookups to database, so it should be used with care. const Coin &AccessByTxid(const CCoinsViewCache &cache, const TxId &txid); /** * This is a minimally invasive approach to shutdown on LevelDB read errors from * the chainstate, while keeping user interface out of the common library, which * is shared between bitcoind, and bitcoin-qt and non-server tools. * * Writes do not need similar protection, as failure to write is handled by the * caller. */ class CCoinsViewErrorCatcher final : public CCoinsViewBacked { public: explicit CCoinsViewErrorCatcher(CCoinsView *view) : CCoinsViewBacked(view) {} void AddReadErrCallback(std::function f) { m_err_callbacks.emplace_back(std::move(f)); } bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; private: /** * A list of callbacks to execute upon leveldb read error. */ std::vector> m_err_callbacks; }; #endif // BITCOIN_COINS_H diff --git a/src/init.cpp b/src/init.cpp index 3d5980b88..30c6ff409 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1,2965 +1,2968 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2018 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include