diff --git a/src/coins.h b/src/coins.h --- a/src/coins.h +++ b/src/coins.h @@ -299,13 +299,13 @@ } }; -class SaltedTxidHasher { +class SaltedOutpointHasher { private: /** Salt */ const uint64_t k0, k1; public: - SaltedTxidHasher(); + SaltedOutpointHasher(); /** * This *must* return size_t. With Boost 1.46 on 32-bit systems the @@ -315,13 +315,14 @@ * container type has meanwhile been switched to the C++ standard library * implementation. */ - size_t operator()(const uint256 &txid) const { - return SipHashUint256(k0, k1, txid); + size_t operator()(const COutPoint &outpoint) const { + return SipHashUint256Extra(k0, k1, outpoint.hash, outpoint.n); } }; struct CCoinsCacheEntry { - CCoins coins; // The actual cached data. + // The actual cached data. + Coin coin; uint8_t flags; enum Flags { @@ -336,21 +337,22 @@ that condition is not guaranteed. */ }; - CCoinsCacheEntry() : coins(), flags(0) {} + CCoinsCacheEntry() : flags(0) {} + explicit CCoinsCacheEntry(Coin coinIn) + : coin(std::move(coinIn)), flags(0) {} }; -typedef std::unordered_map +typedef std::unordered_map CCoinsMap; /** Cursor for iterating over CoinsView state */ class CCoinsViewCursor { public: CCoinsViewCursor(const uint256 &hashBlockIn) : hashBlock(hashBlockIn) {} - virtual ~CCoinsViewCursor(); + virtual ~CCoinsViewCursor() {} - virtual bool GetKey(uint256 &key) const = 0; - virtual bool GetValue(CCoins &coins) const = 0; - /* Don't care about GetKeySize here */ + virtual bool GetKey(COutPoint &key) const = 0; + virtual bool GetValue(Coin &coin) const = 0; virtual unsigned int GetValueSize() const = 0; virtual bool Valid() const = 0; @@ -365,50 +367,18 @@ /** Abstract view on the open txout dataset. */ class CCoinsView { -protected: - //! Retrieve the CCoins (unspent transaction outputs) for a given txid - virtual bool GetCoins(const uint256 &txid, CCoins &coins) const; - - //! Just check whether we have data for a given txid. - //! This may (but cannot always) return true for fully spent transactions - virtual bool HaveCoins(const uint256 &txid) const; - public: - //! Transitional function to move from GetCoins to GetCoin. - bool GetCoins_DONOTUSE(const uint256 &txid, CCoins &coins) const { - return GetCoins(txid, coins); - } - //! Retrieve the Coin (unspent transaction output) for a given outpoint. - bool GetCoin(const COutPoint &outpoint, Coin &coin) const { - CCoins coins; - if (!GetCoins(outpoint.hash, coins) || !coins.IsAvailable(outpoint.n)) { - return false; - } - - coin = Coin(coins.vout[outpoint.n], coins.nHeight, coins.fCoinBase); - return true; - } - - //! Transitional function to move from HaveCoins to HaveCoin. - bool HaveCoins_DONOTUSE(const uint256 &txid) const { - return HaveCoins(txid); - } + virtual bool GetCoin(const COutPoint &outpoint, Coin &coin) const; //! Just check whether we have data for a given outpoint. //! This may (but cannot always) return true for spent outputs. - bool HaveCoin(const COutPoint &outpoint) const { - CCoins coins; - if (!GetCoins(outpoint.hash, coins)) { - return false; - } - return coins.IsAvailable(outpoint.n); - } + virtual bool HaveCoin(const COutPoint &outpoint) const; //! Retrieve the block hash whose state this CCoinsView currently represents virtual uint256 GetBestBlock() const; - //! Do a bulk modification (multiple CCoins changes + BestBlock change). + //! Do a bulk modification (multiple Coin changes + BestBlock change). //! The passed mapCoins can be modified. virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); @@ -427,11 +397,10 @@ protected: CCoinsView *base; - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - public: CCoinsViewBacked(CCoinsView *viewIn); + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; uint256 GetBestBlock() const; void SetBackend(CCoinsView &viewIn); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); @@ -451,23 +420,15 @@ mutable uint256 hashBlock; mutable CCoinsMap cacheCoins; - /* Cached dynamic memory usage for the inner CCoins objects. */ + /* Cached dynamic memory usage for the inner Coin objects. */ mutable size_t cachedCoinsUsage; - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - - /** - * Return a pointer to CCoins in the cache, or nullptr if not found. This is - * more efficient than GetCoins. Modifications to other cache entries are - * allowed while accessing the returned pointer. - */ - const CCoins *AccessCoins(const uint256 &txid) const; - public: CCoinsViewCache(CCoinsView *baseIn); // Standard CCoinsView methods + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; uint256 GetBestBlock() const; void SetBestBlock(const uint256 &hashBlock); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); @@ -479,25 +440,18 @@ */ bool HaveCoinInCache(const COutPoint &outpoint) const; - //! Transitional function to move from AccessCoins to AccessCoin. - const CCoins *AccessCoins_DONOTUSE(const uint256 &txid) const { - return AccessCoins(txid); - } - /** - * Return a copy of a Coin in the cache, or a pruned one if not found. This - * is more efficient than GetCoin. Modifications to other cache entries are - * allowed while accessing the returned pointer. - * TODO: return a reference instead of a value once underlying storage is - * updated. + * Return a reference to a Coin in the cache, or a pruned one if not found. + * This is more efficient than GetCoin. Modifications to other cache entries + * are allowed while accessing the returned pointer. */ - const Coin AccessCoin(const COutPoint &output) const; + const Coin &AccessCoin(const COutPoint &output) const; /** * Add a coin. Set potential_overwrite to true if a non-pruned version may * already exist. */ - void AddCoin(const COutPoint &outpoint, const Coin &coin, + void AddCoin(const COutPoint &outpoint, Coin coin, bool potential_overwrite); /** @@ -521,7 +475,7 @@ */ void Uncache(const COutPoint &outpoint); - //! Calculate the size of the cache (in number of transactions) + //! Calculate the size of the cache (in number of transaction outputs) unsigned int GetCacheSize() const; //! Calculate the size of the cache (in bytes) @@ -551,10 +505,8 @@ const CTxOut &GetOutputFor(const CTxIn &input) const; - friend class CCoinsModifier; - private: - CCoinsMap::iterator FetchCoins(const uint256 &txid) const; + CCoinsMap::iterator FetchCoin(const COutPoint &outpoint) const; /** * By making the copy constructor private, we prevent accidentally using it @@ -570,6 +522,6 @@ void AddCoins(CCoinsViewCache &cache, const CTransaction &tx, int nHeight); //! Utility function to find any unspent output with a given txid. -const Coin AccessByTxid(const CCoinsViewCache &cache, const uint256 &txid); +const Coin &AccessByTxid(const CCoinsViewCache &cache, const uint256 &txid); #endif // BITCOIN_COINS_H diff --git a/src/coins.cpp b/src/coins.cpp --- a/src/coins.cpp +++ b/src/coins.cpp @@ -41,10 +41,10 @@ return true; } -bool CCoinsView::GetCoins(const uint256 &txid, CCoins &coins) const { +bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } -bool CCoinsView::HaveCoins(const uint256 &txid) const { +bool CCoinsView::HaveCoin(const COutPoint &outpoint) const { return false; } uint256 CCoinsView::GetBestBlock() const { @@ -54,15 +54,15 @@ return false; } CCoinsViewCursor *CCoinsView::Cursor() const { - return 0; + return nullptr; } CCoinsViewBacked::CCoinsViewBacked(CCoinsView *viewIn) : base(viewIn) {} -bool CCoinsViewBacked::GetCoins(const uint256 &txid, CCoins &coins) const { - return base->GetCoins_DONOTUSE(txid, coins); +bool CCoinsViewBacked::GetCoin(const COutPoint &outpoint, Coin &coin) const { + return base->GetCoin(outpoint, coin); } -bool CCoinsViewBacked::HaveCoins(const uint256 &txid) const { - return base->HaveCoins_DONOTUSE(txid); +bool CCoinsViewBacked::HaveCoin(const COutPoint &outpoint) const { + return base->HaveCoin(outpoint); } uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); @@ -81,7 +81,7 @@ return base->EstimateSize(); } -SaltedTxidHasher::SaltedTxidHasher() +SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} @@ -92,37 +92,40 @@ return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; } -CCoinsMap::iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const { - CCoinsMap::iterator it = cacheCoins.find(txid); +CCoinsMap::iterator +CCoinsViewCache::FetchCoin(const COutPoint &outpoint) const { + CCoinsMap::iterator it = cacheCoins.find(outpoint); if (it != cacheCoins.end()) { return it; } - CCoins tmp; - if (!base->GetCoins_DONOTUSE(txid, tmp)) { + Coin tmp; + if (!base->GetCoin(outpoint, tmp)) { return cacheCoins.end(); } CCoinsMap::iterator ret = - cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())).first; - tmp.swap(ret->second.coins); - if (ret->second.coins.IsPruned()) { - // The parent only has an empty entry for this txid; we can consider our - // version as fresh. + cacheCoins + .emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), + std::forward_as_tuple(std::move(tmp))) + .first; + if (ret->second.coin.IsSpent()) { + // The parent only has an empty entry for this outpoint; we can consider + // our version as fresh. ret->second.flags = CCoinsCacheEntry::FRESH; } - cachedCoinsUsage += ret->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage += ret->second.coin.DynamicMemoryUsage(); return ret; } -bool CCoinsViewCache::GetCoins(const uint256 &txid, CCoins &coins) const { - CCoinsMap::const_iterator it = FetchCoins(txid); - if (it != cacheCoins.end()) { - coins = it->second.coins; - return true; +bool CCoinsViewCache::GetCoin(const COutPoint &outpoint, Coin &coin) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); + if (it == cacheCoins.end()) { + return false; } - return false; + coin = it->second.coin; + return true; } -void CCoinsViewCache::AddCoin(const COutPoint &outpoint, const Coin &coin, +void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin coin, bool possible_overwrite) { assert(!coin.IsSpent()); if (coin.GetTxOut().scriptPubKey.IsUnspendable()) { @@ -130,30 +133,24 @@ } CCoinsMap::iterator it; bool inserted; - std::tie(it, inserted) = cacheCoins.emplace( - std::piecewise_construct, std::forward_as_tuple(outpoint.hash), - std::tuple<>()); + std::tie(it, inserted) = + cacheCoins.emplace(std::piecewise_construct, + std::forward_as_tuple(outpoint), std::tuple<>()); bool fresh = false; if (!inserted) { - cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); } if (!possible_overwrite) { - if (it->second.coins.IsAvailable(outpoint.n)) { + if (!it->second.coin.IsSpent()) { throw std::logic_error( "Adding new coin that replaces non-pruned entry"); } - fresh = it->second.coins.IsPruned() && - !(it->second.flags & CCoinsCacheEntry::DIRTY); + fresh = !(it->second.flags & CCoinsCacheEntry::DIRTY); } - if (it->second.coins.vout.size() <= outpoint.n) { - it->second.coins.vout.resize(outpoint.n + 1); - } - it->second.coins.vout[outpoint.n] = coin.GetTxOut(); - it->second.coins.nHeight = coin.GetHeight(); - it->second.coins.fCoinBase = coin.IsCoinBase(); + it->second.coin = std::move(coin); it->second.flags |= CCoinsCacheEntry::DIRTY | (fresh ? CCoinsCacheEntry::FRESH : 0); - cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); } void AddCoins(CCoinsViewCache &cache, const CTransaction &tx, int nHeight) { @@ -169,63 +166,47 @@ } bool CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin *moveout) { - CCoinsMap::iterator it = FetchCoins(outpoint.hash); + CCoinsMap::iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) { return false; } - cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); - if (moveout && it->second.coins.IsAvailable(outpoint.n)) { - *moveout = Coin(it->second.coins.vout[outpoint.n], - it->second.coins.nHeight, it->second.coins.fCoinBase); + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + if (moveout) { + *moveout = std::move(it->second.coin); } - // Ignore return value: SpendCoin has no effect if no UTXO found. - it->second.coins.Spend(outpoint.n); - if (it->second.coins.IsPruned() && - it->second.flags & CCoinsCacheEntry::FRESH) { + if (it->second.flags & CCoinsCacheEntry::FRESH) { cacheCoins.erase(it); } else { - cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); it->second.flags |= CCoinsCacheEntry::DIRTY; + it->second.coin.Clear(); } return true; } -const CCoins *CCoinsViewCache::AccessCoins(const uint256 &txid) const { - CCoinsMap::const_iterator it = FetchCoins(txid); - if (it == cacheCoins.end()) { - return nullptr; - } - - return &it->second.coins; -} - static const Coin coinEmpty; -const Coin CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const { - CCoinsMap::const_iterator it = FetchCoins(outpoint.hash); - if (it == cacheCoins.end() || !it->second.coins.IsAvailable(outpoint.n)) { +const Coin &CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); + if (it == cacheCoins.end()) { return coinEmpty; } - return Coin(it->second.coins.vout[outpoint.n], it->second.coins.nHeight, - it->second.coins.fCoinBase); + return it->second.coin; } -bool CCoinsViewCache::HaveCoins(const uint256 &txid) const { - CCoinsMap::const_iterator it = FetchCoins(txid); - // We're using vtx.empty() instead of IsPruned here for performance reasons, - // as we only care about the case where a transaction was replaced entirely - // in a reorganization (which wipes vout entirely, as opposed to spending - // which just cleans individual outputs). - return (it != cacheCoins.end() && !it->second.coins.vout.empty()); +bool CCoinsViewCache::HaveCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); + return it != cacheCoins.end() && !it->second.coin.IsSpent(); } bool CCoinsViewCache::HaveCoinInCache(const COutPoint &outpoint) const { - CCoinsMap::const_iterator it = cacheCoins.find(outpoint.hash); - return it != cacheCoins.end() && it->second.coins.IsAvailable(outpoint.n); + CCoinsMap::const_iterator it = cacheCoins.find(outpoint); + return it != cacheCoins.end(); } uint256 CCoinsViewCache::GetBestBlock() const { - if (hashBlock.IsNull()) hashBlock = base->GetBestBlock(); + if (hashBlock.IsNull()) { + hashBlock = base->GetBestBlock(); + } return hashBlock; } @@ -243,12 +224,12 @@ // The parent cache does not have an entry, while the child does // We can ignore it if it's both FRESH and pruned in the child if (!(it->second.flags & CCoinsCacheEntry::FRESH && - it->second.coins.IsPruned())) { + it->second.coin.IsSpent())) { // Otherwise we will need to create it in the parent and // move the data up and mark it as dirty CCoinsCacheEntry &entry = cacheCoins[it->first]; - entry.coins.swap(it->second.coins); - cachedCoinsUsage += entry.coins.DynamicMemoryUsage(); + entry.coin = std::move(it->second.coin); + cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); entry.flags = CCoinsCacheEntry::DIRTY; // We can mark it FRESH in the parent if it was FRESH in the // child. Otherwise it might have just been flushed from the @@ -262,24 +243,24 @@ // it means the FRESH flag was misapplied and there is a logic // error in the calling code. if ((it->second.flags & CCoinsCacheEntry::FRESH) && - !itUs->second.coins.IsPruned()) + !itUs->second.coin.IsSpent()) throw std::logic_error("FRESH flag misapplied to cache " "entry for base transaction with " "spendable outputs"); // Found the entry in the parent cache if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && - it->second.coins.IsPruned()) { + it->second.coin.IsSpent()) { // The grandparent does not have an entry, and the child is // modified and being pruned. This means we can just delete // it from the parent. - cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); cacheCoins.erase(itUs); } else { // A normal modification. - cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); - itUs->second.coins.swap(it->second.coins); - cachedCoinsUsage += itUs->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); + itUs->second.coin = std::move(it->second.coin); + cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); itUs->second.flags |= CCoinsCacheEntry::DIRTY; // NOTE: It is possible the child has a FRESH flag here in // the event the entry we found in the parent is pruned. But @@ -304,9 +285,9 @@ } void CCoinsViewCache::Uncache(const COutPoint &outpoint) { - CCoinsMap::iterator it = cacheCoins.find(outpoint.hash); + CCoinsMap::iterator it = cacheCoins.find(outpoint); if (it != cacheCoins.end() && it->second.flags == 0) { - cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); cacheCoins.erase(it); } } @@ -316,17 +297,20 @@ } const CTxOut &CCoinsViewCache::GetOutputFor(const CTxIn &input) const { - const CCoins *coins = AccessCoins(input.prevout.hash); - assert(coins && coins->IsAvailable(input.prevout.n)); - return coins->vout[input.prevout.n]; + const Coin &coin = AccessCoin(input.prevout); + assert(!coin.IsSpent()); + return coin.GetTxOut(); } CAmount CCoinsViewCache::GetValueIn(const CTransaction &tx) const { - if (tx.IsCoinBase()) return 0; + if (tx.IsCoinBase()) { + return 0; + } CAmount nResult = 0; - for (unsigned int i = 0; i < tx.vin.size(); i++) + for (size_t i = 0; i < tx.vin.size(); i++) { nResult += GetOutputFor(tx.vin[i]).nValue; + } return nResult; } @@ -336,10 +320,8 @@ return true; } - for (unsigned int i = 0; i < tx.vin.size(); i++) { - const COutPoint &prevout = tx.vin[i].prevout; - const CCoins *coins = AccessCoins(prevout.hash); - if (!coins || !coins->IsAvailable(prevout.n)) { + for (size_t i = 0; i < tx.vin.size(); i++) { + if (!HaveCoin(tx.vin[i].prevout)) { return false; } } @@ -350,28 +332,29 @@ double CCoinsViewCache::GetPriority(const CTransaction &tx, int nHeight, CAmount &inChainInputValue) const { inChainInputValue = 0; - if (tx.IsCoinBase()) return 0.0; + if (tx.IsCoinBase()) { + return 0.0; + } double dResult = 0.0; for (const CTxIn &txin : tx.vin) { - const CCoins *coins = AccessCoins(txin.prevout.hash); - assert(coins); - if (!coins->IsAvailable(txin.prevout.n)) continue; - if (coins->nHeight <= nHeight) { - dResult += (double)(coins->vout[txin.prevout.n].nValue) * - (nHeight - coins->nHeight); - inChainInputValue += coins->vout[txin.prevout.n].nValue; + const Coin &coin = AccessCoin(txin.prevout); + if (coin.IsSpent()) { + continue; + } + if (int64_t(coin.GetHeight()) <= nHeight) { + dResult += + double(coin.GetTxOut().nValue) * (nHeight - coin.GetHeight()); + inChainInputValue += coin.GetTxOut().nValue; } } return tx.ComputePriority(dResult); } -CCoinsViewCursor::~CCoinsViewCursor() {} - // TODO: merge with similar definition in undo.h. static const size_t MAX_OUTPUTS_PER_TX = MAX_TX_SIZE / ::GetSerializeSize(CTxOut(), SER_NETWORK, PROTOCOL_VERSION); -const Coin AccessByTxid(const CCoinsViewCache &view, const uint256 &txid) { +const Coin &AccessByTxid(const CCoinsViewCache &view, const uint256 &txid) { COutPoint iter(txid, 0); while (iter.n < MAX_OUTPUTS_PER_TX) { const Coin &alternate = view.AccessCoin(iter); diff --git a/src/dbwrapper.h b/src/dbwrapper.h --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -66,6 +66,11 @@ : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION), size_estimate(0){}; + void Clear() { + batch.Clear(); + size_estimate = 0; + } + template void Write(const K &key, const V &value) { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; diff --git a/src/init.cpp b/src/init.cpp --- a/src/init.cpp +++ b/src/init.cpp @@ -141,9 +141,9 @@ class CCoinsViewErrorCatcher : public CCoinsViewBacked { public: CCoinsViewErrorCatcher(CCoinsView *view) : CCoinsViewBacked(view) {} - bool GetCoins(const uint256 &txid, CCoins &coins) const { + bool GetCoin(const COutPoint &outpoint, Coin &coin) const { try { - return CCoinsViewBacked::GetCoins(txid, coins); + return CCoinsViewBacked::GetCoin(outpoint, coin); } catch (const std::runtime_error &e) { uiInterface.ThreadSafeMessageBox( _("Error reading from database, shutting down."), "", @@ -1942,7 +1942,12 @@ pblocktree->WriteReindexing(true); // If we're reindexing in prune mode, wipe away unusable // block files and all undo data files - if (fPruneMode) CleanupBlockRevFiles(); + if (fPruneMode) { + CleanupBlockRevFiles(); + } + } else if (!pcoinsdbview->Upgrade()) { + strLoadError = _("Error upgrading chainstate database"); + break; } if (!LoadBlockIndex(chainparams)) { diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -867,6 +867,27 @@ nDiskSize(0), nTotalAmount(0) {} }; +static void ApplyStats(CCoinsStats &stats, CHashWriter &ss, const uint256 &hash, + const std::map &outputs) { + assert(!outputs.empty()); + ss << hash; + ss << VARINT(outputs.begin()->second.GetHeight() * 2 + + outputs.begin()->second.IsCoinBase()); + stats.nTransactions++; + for (const auto output : outputs) { + ss << VARINT(output.first + 1); + ss << *(const CScriptBase *)(&output.second.GetTxOut().scriptPubKey); + ss << VARINT(output.second.GetTxOut().nValue); + stats.nTransactionOutputs++; + stats.nTotalAmount += output.second.GetTxOut().nValue; + stats.nBogoSize += + 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + + 8 /* amount */ + 2 /* scriptPubKey len */ + + output.second.GetTxOut().scriptPubKey.size() /* scriptPubKey */; + } + ss << VARINT(0); +} + //! Calculate statistics about the unspent transaction output set static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) { std::unique_ptr pcursor(view->Cursor()); @@ -878,37 +899,28 @@ stats.nHeight = mapBlockIndex.find(stats.hashBlock)->second->nHeight; } ss << stats.hashBlock; - CAmount nTotalAmount = 0; + uint256 prevkey; + std::map outputs; while (pcursor->Valid()) { boost::this_thread::interruption_point(); - uint256 key; - CCoins coins; - if (pcursor->GetKey(key) && pcursor->GetValue(coins)) { - stats.nTransactions++; - ss << key; - ss << VARINT(coins.nHeight * 2 + coins.fCoinBase); - for (size_t i = 0; i < coins.vout.size(); i++) { - const CTxOut &out = coins.vout[i]; - if (!out.IsNull()) { - stats.nTransactionOutputs++; - ss << VARINT(i + 1); - ss << out; - nTotalAmount += out.nValue; - stats.nBogoSize += - 32 /* txid */ + 4 /* vout index */ + - 4 /* height + coinbase */ + 8 /* amount */ + - 2 /* scriptPubKey len */ + - out.scriptPubKey.size() /* scriptPubKey */; - } + COutPoint key; + Coin coin; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + if (!outputs.empty() && key.hash != prevkey) { + ApplyStats(stats, ss, prevkey, outputs); + outputs.clear(); } - ss << VARINT(0); + prevkey = key.hash; + outputs[key.n] = std::move(coin); } else { return error("%s: unable to read value", __func__); } pcursor->Next(); } + if (!outputs.empty()) { + ApplyStats(stats, ss, prevkey, outputs); + } stats.hashSerialized = ss.GetHash(); - stats.nTotalAmount = nTotalAmount; stats.nDiskSize = view->EstimateSize(); return true; } @@ -1053,7 +1065,6 @@ " ,...\n" " ]\n" " },\n" - " \"version\" : n, (numeric) The version\n" " \"coinbase\" : true|false (boolean) Coinbase or not\n" "}\n" diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -315,10 +315,11 @@ throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); pblockindex = mapBlockIndex[hashBlock]; } else { - CCoins coins; - if (pcoinsTip->GetCoins_DONOTUSE(oneTxid, coins) && coins.nHeight > 0 && - coins.nHeight <= chainActive.Height()) - pblockindex = chainActive[coins.nHeight]; + const Coin &coin = AccessByTxid(*pcoinsTip, oneTxid); + if (!coin.IsSpent() && coin.GetHeight() > 0 && + int64_t(coin.GetHeight()) <= chainActive.Height()) { + pblockindex = chainActive[coin.GetHeight()]; + } } if (pblockindex == nullptr) { diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -32,25 +32,25 @@ class CCoinsViewTest : public CCoinsView { uint256 hashBestBlock_; - std::map map_; + std::map map_; public: - bool GetCoins(const uint256 &txid, CCoins &coins) const { - std::map::const_iterator it = map_.find(txid); + bool GetCoin(const COutPoint &outpoint, Coin &coin) const { + std::map::const_iterator it = map_.find(outpoint); if (it == map_.end()) { return false; } - coins = it->second; - if (coins.IsPruned() && insecure_rand() % 2 == 0) { + coin = it->second; + if (coin.IsSpent() && insecure_rand() % 2 == 0) { // Randomly return false in case of an empty entry. return false; } return true; } - bool HaveCoins(const uint256 &txid) const { - CCoins coins; - return GetCoins(txid, coins); + bool HaveCoin(const COutPoint outpoint) const { + Coin coin; + return GetCoin(outpoint, coin); } uint256 GetBestBlock() const { return hashBestBlock_; } @@ -60,15 +60,17 @@ if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Same optimization used in CCoinsViewDB is to only write dirty // entries. - map_[it->first] = it->second.coins; - if (it->second.coins.IsPruned() && insecure_rand() % 3 == 0) { + map_[it->first] = it->second.coin; + if (it->second.coin.IsSpent() && insecure_rand() % 3 == 0) { // Randomly delete empty entries on write. map_.erase(it->first); } } mapCoins.erase(it++); } - if (!hashBlock.IsNull()) hashBestBlock_ = hashBlock; + if (!hashBlock.IsNull()) { + hashBestBlock_ = hashBlock; + } return true; } }; @@ -83,7 +85,7 @@ size_t ret = memusage::DynamicUsage(cacheCoins); for (CCoinsMap::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) { - ret += it->second.coins.DynamicMemoryUsage(); + ret += it->second.coin.DynamicMemoryUsage(); } BOOST_CHECK_EQUAL(DynamicMemoryUsage(), ret); } @@ -622,9 +624,7 @@ } } -// TODO: Remove TXID once the migration is over. -static const uint256 TXID; -static const COutPoint OUTPOINT = {uint256(), 0}; +static const COutPoint OUTPOINT; static const CAmount PRUNED = -1; static const CAmount ABSENT = -2; static const CAmount FAIL = -3; @@ -639,14 +639,15 @@ static const auto CLEAN_FLAGS = {char(0), FRESH}; static const auto ABSENT_FLAGS = {NO_ENTRY}; -void SetCoinsValue(CAmount value, CCoins &coins) { +void SetCoinsValue(CAmount value, Coin &coin) { assert(value != ABSENT); - coins.Clear(); - assert(coins.IsPruned()); + coin.Clear(); + assert(coin.IsSpent()); if (value != PRUNED) { - coins.vout.emplace_back(); - coins.vout.back().nValue = value; - assert(!coins.IsPruned()); + CTxOut out; + out.nValue = value; + coin = Coin(std::move(out), 1, false); + assert(!coin.IsSpent()); } } @@ -658,24 +659,22 @@ assert(flags != NO_ENTRY); CCoinsCacheEntry entry; entry.flags = flags; - SetCoinsValue(value, entry.coins); - auto inserted = map.emplace(TXID, std::move(entry)); + SetCoinsValue(value, entry.coin); + auto inserted = map.emplace(OUTPOINT, std::move(entry)); assert(inserted.second); - return inserted.first->second.coins.DynamicMemoryUsage(); + return inserted.first->second.coin.DynamicMemoryUsage(); } void GetCoinsMapEntry(const CCoinsMap &map, CAmount &value, char &flags) { - auto it = map.find(TXID); + auto it = map.find(OUTPOINT); if (it == map.end()) { value = ABSENT; flags = NO_ENTRY; } else { - if (it->second.coins.IsPruned()) { - assert(it->second.coins.vout.size() == 0); + if (it->second.coin.IsSpent()) { value = PRUNED; } else { - assert(it->second.coins.vout.size() == 1); - value = it->second.coins.vout[0].nValue; + value = it->second.coin.GetTxOut().nValue; } flags = it->second.flags; assert(flags != NO_ENTRY); diff --git a/src/txdb.h b/src/txdb.h --- a/src/txdb.h +++ b/src/txdb.h @@ -71,12 +71,15 @@ public: CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; uint256 GetBestBlock() const; bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); CCoinsViewCursor *Cursor() const; + //! Attempt to update from an older database format. + //! Returns whether an error occurred. + bool Upgrade(); size_t EstimateSize() const override; }; @@ -85,8 +88,8 @@ public: ~CCoinsViewDBCursor() {} - bool GetKey(uint256 &key) const; - bool GetValue(CCoins &coins) const; + bool GetKey(COutPoint &key) const; + bool GetValue(Coin &coin) const; unsigned int GetValueSize() const; bool Valid() const; @@ -96,7 +99,7 @@ CCoinsViewDBCursor(CDBIterator *pcursorIn, const uint256 &hashBlockIn) : CCoinsViewCursor(hashBlockIn), pcursor(pcursorIn) {} std::unique_ptr pcursor; - std::pair keyTmp; + std::pair keyTmp; friend class CCoinsViewDB; }; diff --git a/src/txdb.cpp b/src/txdb.cpp --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -14,6 +14,7 @@ #include +static const char DB_COIN = 'C'; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; @@ -24,15 +25,37 @@ static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; +namespace { + +struct CoinEntry { + COutPoint *outpoint; + char key; + CoinEntry(const COutPoint *ptr) + : outpoint(const_cast(ptr)), key(DB_COIN) {} + + template void Serialize(Stream &s) const { + s << key; + s << outpoint->hash; + s << VARINT(outpoint->n); + } + + template void Unserialize(Stream &s) { + s >> key; + s >> outpoint->hash; + s >> VARINT(outpoint->n); + } +}; +} + CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) {} -bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const { - return db.Read(std::make_pair(DB_COINS, txid), coins); +bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { + return db.Read(CoinEntry(&outpoint), coin); } -bool CCoinsViewDB::HaveCoins(const uint256 &txid) const { - return db.Exists(std::make_pair(DB_COINS, txid)); +bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const { + return db.Exists(CoinEntry(&outpoint)); } uint256 CCoinsViewDB::GetBestBlock() const { @@ -47,28 +70,31 @@ size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { - if (it->second.coins.IsPruned()) - batch.Erase(std::make_pair(DB_COINS, it->first)); - else - batch.Write(std::make_pair(DB_COINS, it->first), - it->second.coins); + CoinEntry entry(&it->first); + if (it->second.coin.IsSpent()) { + batch.Erase(entry); + } else { + batch.Write(entry, it->second.coin); + } changed++; } count++; CCoinsMap::iterator itOld = it++; mapCoins.erase(itOld); } - if (!hashBlock.IsNull()) batch.Write(DB_BEST_BLOCK, hashBlock); + if (!hashBlock.IsNull()) { + batch.Write(DB_BEST_BLOCK, hashBlock); + } - LogPrint( - "coindb", - "Committing %u changed transactions (out of %u) to coin database...\n", - (unsigned int)changed, (unsigned int)count); - return db.WriteBatch(batch); + bool ret = db.WriteBatch(batch); + LogPrint("coindb", "Committed %u changed transaction outputs (out of %u) " + "to coin database...\n", + (unsigned int)changed, (unsigned int)count); + return ret; } size_t CCoinsViewDB::EstimateSize() const { - return db.EstimateSize(DB_COINS, char(DB_COINS + 1)); + return db.EstimateSize(DB_COIN, char(DB_COIN + 1)); } CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) @@ -103,10 +129,12 @@ * need read operations on it, use a const-cast to get around that * restriction. */ - i->pcursor->Seek(DB_COINS); + i->pcursor->Seek(DB_COIN); // Cache key of first record if (i->pcursor->Valid()) { - i->pcursor->GetKey(i->keyTmp); + CoinEntry entry(&i->keyTmp.second); + i->pcursor->GetKey(entry); + i->keyTmp.first = entry.key; } else { // Make sure Valid() and GetKey() return false i->keyTmp.first = 0; @@ -114,17 +142,17 @@ return i; } -bool CCoinsViewDBCursor::GetKey(uint256 &key) const { +bool CCoinsViewDBCursor::GetKey(COutPoint &key) const { // Return cached key - if (keyTmp.first == DB_COINS) { + if (keyTmp.first == DB_COIN) { key = keyTmp.second; return true; } return false; } -bool CCoinsViewDBCursor::GetValue(CCoins &coins) const { - return pcursor->GetValue(coins); +bool CCoinsViewDBCursor::GetValue(Coin &coin) const { + return pcursor->GetValue(coin); } unsigned int CCoinsViewDBCursor::GetValueSize() const { @@ -132,15 +160,19 @@ } bool CCoinsViewDBCursor::Valid() const { - return keyTmp.first == DB_COINS; + return keyTmp.first == DB_COIN; } void CCoinsViewDBCursor::Next() { pcursor->Next(); - if (!pcursor->Valid() || !pcursor->GetKey(keyTmp)) + CoinEntry entry(&keyTmp.second); + if (!pcursor->Valid() || !pcursor->GetKey(entry)) { // Invalidate cached key after last record so that Valid() and GetKey() // return false keyTmp.first = 0; + } else { + keyTmp.first = entry.key; + } } bool CBlockTreeDB::WriteBatchSync( @@ -231,3 +263,55 @@ return true; } + +/** + * Upgrade the database from older formats. + * + * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout. + */ +bool CCoinsViewDB::Upgrade() { + std::unique_ptr pcursor(db.NewIterator()); + pcursor->Seek(std::make_pair(DB_COINS, uint256())); + if (!pcursor->Valid()) { + return true; + } + + LogPrintf("Upgrading database...\n"); + size_t batch_size = 1 << 24; + CDBBatch batch(db); + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (!pcursor->GetKey(key) || key.first != DB_COINS) { + break; + } + + CCoins old_coins; + if (!pcursor->GetValue(old_coins)) { + return error("%s: cannot parse CCoins record", __func__); + } + + COutPoint outpoint(key.second, 0); + for (size_t i = 0; i < old_coins.vout.size(); ++i) { + if (!old_coins.vout[i].IsNull() && + !old_coins.vout[i].scriptPubKey.IsUnspendable()) { + Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, + old_coins.fCoinBase); + outpoint.n = i; + CoinEntry entry(&outpoint); + batch.Write(entry, newcoin); + } + } + + batch.Erase(key); + if (batch.SizeEstimate() > batch_size) { + db.WriteBatch(batch); + batch.Clear(); + } + + pcursor->Next(); + } + + db.WriteBatch(batch); + return true; +} diff --git a/src/txmempool.h b/src/txmempool.h --- a/src/txmempool.h +++ b/src/txmempool.h @@ -370,6 +370,19 @@ REPLACED }; +class SaltedTxidHasher { +private: + /** Salt */ + const uint64_t k0, k1; + +public: + SaltedTxidHasher(); + + size_t operator()(const uint256 &txid) const { + return SipHashUint256(k0, k1, txid); + } +}; + /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions that * may be included in the next block. @@ -683,7 +696,13 @@ bool exists(uint256 hash) const { LOCK(cs); - return (mapTx.count(hash) != 0); + return mapTx.count(hash) != 0; + } + + bool exists(const COutPoint &outpoint) const { + LOCK(cs); + auto it = mapTx.find(outpoint.hash); + return it != mapTx.end() && outpoint.n < it->GetTx().vout.size(); } CTransactionRef get(const uint256 &hash) const; @@ -773,11 +792,10 @@ protected: const CTxMemPool &mempool; - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - public: CCoinsViewMemPool(CCoinsView *baseIn, const CTxMemPool &mempoolIn); + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; }; // We want to sort transactions by coin age priority diff --git a/src/txmempool.cpp b/src/txmempool.cpp --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -585,7 +585,7 @@ if (nCheckFrequency != 0) assert(!coin.IsSpent()); if (coin.IsSpent() || (coin.IsCoinBase() && - ((signed long)nMemPoolHeight) - coin.GetHeight() < + int64_t(nMemPoolHeight) - coin.GetHeight() < COINBASE_MATURITY)) { txToRemove.insert(it); break; @@ -1017,21 +1017,25 @@ const CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) {} -bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) const { +bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { // If an entry in the mempool exists, always return that one, as it's // guaranteed to never conflict with the underlying cache, and it cannot // have pruned entries (as it contains full) transactions. First checking // the underlying cache risks returning a pruned entry instead. - CTransactionRef ptx = mempool.get(txid); + CTransactionRef ptx = mempool.get(outpoint.hash); if (ptx) { - coins = CCoins(*ptx, MEMPOOL_HEIGHT); - return true; + if (outpoint.n < ptx->vout.size()) { + coin = Coin(ptx->vout[outpoint.n], MEMPOOL_HEIGHT, false); + return true; + } + return false; } - return (base->GetCoins_DONOTUSE(txid, coins) && !coins.IsPruned()); + + return base->GetCoin(outpoint, coin) && !coin.IsSpent(); } -bool CCoinsViewMemPool::HaveCoins(const uint256 &txid) const { - return mempool.exists(txid) || base->HaveCoins_DONOTUSE(txid); +bool CCoinsViewMemPool::HaveCoin(const COutPoint &outpoint) const { + return mempool.exists(outpoint) || base->HaveCoin(outpoint); } size_t CTxMemPool::DynamicMemoryUsage() const { @@ -1192,12 +1196,9 @@ if (exists(txin.prevout.hash)) { continue; } - - auto iter = - mapNextTx.lower_bound(COutPoint(txin.prevout.hash, 0)); - if (iter == mapNextTx.end() || - iter->first->hash != txin.prevout.hash) + if (!mapNextTx.count(txin.prevout)) { pvNoSpendsRemaining->push_back(txin.prevout); + } } } } @@ -1216,3 +1217,7 @@ return it == mapTx.end() || (it->GetCountWithAncestors() < chainLimit && it->GetCountWithDescendants() < chainLimit); } + +SaltedTxidHasher::SaltedTxidHasher() + : k0(GetRand(std::numeric_limits::max())), + k1(GetRand(std::numeric_limits::max())) {} diff --git a/src/validation.cpp b/src/validation.cpp --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2182,16 +2182,18 @@ // Flush best chain related state. This can only be done if the blocks / // block index write was also done. if (fDoFullFlush) { - // Typical CCoins structures on disk are around 128 bytes in size. + // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(128 * 2 * 2 * pcoinsTip->GetCacheSize())) + if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) { return state.Error("out of disk space"); + } // Flush the chainstate (which may refer to block index entries). - if (!pcoinsTip->Flush()) + if (!pcoinsTip->Flush()) { return AbortNode(state, "Failed to write to coin database"); + } nLastFlush = nNow; } if (fDoFullFlush || @@ -2287,7 +2289,7 @@ } LogPrintf( "%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu " - "date='%s' progress=%f cache=%.1fMiB(%utx)", + "date='%s' progress=%f cache=%.1fMiB(%utxo)", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, log(chainActive.Tip()->nChainWork.getdouble()) / log(2.0),