diff --git a/doc/release-notes.md b/doc/release-notes.md index 84de37e8b..96b13bd11 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,14 +1,16 @@ # Bitcoin ABC 0.25.8 Release Notes Bitcoin ABC version 0.25.8 is now available from: This release includes the following features and fixes: - Users can start their node with the option `-coinstatsindex` which syncs an index of coin statistics in the background. After the index is synced the user can use `gettxoutsetinfo` with hash_type=none or hash_type=muhash and will get the response instantly out of the index - Users can specify a height or block hash when calling `gettxoutsetinfo` to see coin statistics at a specific block height when they use the `-coinstatsindex` option. +- Additional amount tracking information has been added to the output of + `gettxoutsetinfo` when the `-coinstatsindex` option is set. diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index d21f05520..552fb8d84 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -1,390 +1,522 @@ // Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include #include #include #include #include static constexpr char DB_BLOCK_HASH = 's'; static constexpr char DB_BLOCK_HEIGHT = 't'; static constexpr char DB_MUHASH = 'M'; namespace { struct DBVal { uint256 muhash; uint64_t transaction_output_count; uint64_t bogo_size; Amount total_amount; + Amount total_subsidy; + Amount total_unspendable_amount; + Amount total_prevout_spent_amount; + Amount total_new_outputs_ex_coinbase_amount; + Amount total_coinbase_amount; + Amount total_unspendables_genesis_block; + Amount total_unspendables_bip30; + Amount total_unspendables_scripts; + Amount total_unspendables_unclaimed_rewards; SERIALIZE_METHODS(DBVal, obj) { READWRITE(obj.muhash); READWRITE(obj.transaction_output_count); READWRITE(obj.bogo_size); READWRITE(obj.total_amount); + READWRITE(obj.total_subsidy); + READWRITE(obj.total_unspendable_amount); + READWRITE(obj.total_prevout_spent_amount); + READWRITE(obj.total_new_outputs_ex_coinbase_amount); + READWRITE(obj.total_coinbase_amount); + READWRITE(obj.total_unspendables_genesis_block); + READWRITE(obj.total_unspendables_bip30); + READWRITE(obj.total_unspendables_scripts); + READWRITE(obj.total_unspendables_unclaimed_rewards); } }; struct DBHeightKey { int height; explicit DBHeightKey(int height_in) : height(height_in) {} template void Serialize(Stream &s) const { ser_writedata8(s, DB_BLOCK_HEIGHT); ser_writedata32be(s, height); } template void Unserialize(Stream &s) { char prefix{static_cast(ser_readdata8(s))}; if (prefix != DB_BLOCK_HEIGHT) { throw std::ios_base::failure( "Invalid format for coinstatsindex DB height key"); } height = ser_readdata32be(s); } }; struct DBHashKey { BlockHash block_hash; explicit DBHashKey(const BlockHash &hash_in) : block_hash(hash_in) {} SERIALIZE_METHODS(DBHashKey, obj) { char prefix{DB_BLOCK_HASH}; READWRITE(prefix); if (prefix != DB_BLOCK_HASH) { throw std::ios_base::failure( "Invalid format for coinstatsindex DB hash key"); } READWRITE(obj.block_hash); } }; }; // namespace std::unique_ptr g_coin_stats_index; CoinStatsIndex::CoinStatsIndex(size_t n_cache_size, bool f_memory, bool f_wipe) { fs::path path{GetDataDir() / "indexes" / "coinstats"}; fs::create_directories(path); m_db = std::make_unique(path / "db", n_cache_size, f_memory, f_wipe); } bool CoinStatsIndex::WriteBlock(const CBlock &block, const CBlockIndex *pindex) { CBlockUndo block_undo; + const Amount block_subsidy{ + GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())}; + m_total_subsidy += block_subsidy; // Ignore genesis block if (pindex->nHeight > 0) { if (!UndoReadFromDisk(block_undo, pindex)) { return false; } std::pair read_out; if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { return false; } BlockHash expected_block_hash{pindex->pprev->GetBlockHash()}; if (read_out.first != expected_block_hash) { if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { return error("%s: previous block header belongs to unexpected " "block %s; expected %s", __func__, read_out.first.ToString(), expected_block_hash.ToString()); } } // TODO: Deduplicate BIP30 related code bool is_bip30_block{ (pindex->nHeight == 91722 && pindex->GetBlockHash() == BlockHash{uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc" "6955e5a6c6cdf3f2574dd08e")}) || (pindex->nHeight == 91812 && pindex->GetBlockHash() == BlockHash{uint256S("0x00000000000af0aed4792b1acee3d966af36cf5d" "ef14935db8de83d6f9306f2f")})}; // Add the new utxos created from the block for (size_t i = 0; i < block.vtx.size(); ++i) { const auto &tx{block.vtx.at(i)}; // Skip duplicate txid coinbase transactions (BIP30). if (is_bip30_block && tx->IsCoinBase()) { + m_total_unspendable_amount += block_subsidy; + m_total_unspendables_bip30 += block_subsidy; continue; } for (uint32_t j = 0; j < tx->vout.size(); ++j) { const CTxOut &out{tx->vout[j]}; Coin coin{out, static_cast(pindex->nHeight), tx->IsCoinBase()}; COutPoint outpoint{tx->GetId(), j}; // Skip unspendable coins if (coin.GetTxOut().scriptPubKey.IsUnspendable()) { + m_total_unspendable_amount += coin.GetTxOut().nValue; + m_total_unspendables_scripts += coin.GetTxOut().nValue; continue; } m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); + if (tx->IsCoinBase()) { + m_total_coinbase_amount += coin.GetTxOut().nValue; + } else { + m_total_new_outputs_ex_coinbase_amount += + coin.GetTxOut().nValue; + } + ++m_transaction_output_count; m_total_amount += coin.GetTxOut().nValue; m_bogo_size += GetBogoSize(coin.GetTxOut().scriptPubKey); } // The coinbase tx has no undo data since no former output is spent if (!tx->IsCoinBase()) { const auto &tx_undo{block_undo.vtxundo.at(i - 1)}; for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { Coin coin{tx_undo.vprevout[j]}; COutPoint outpoint{tx->vin[j].prevout.GetTxId(), tx->vin[j].prevout.GetN()}; m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); + m_total_prevout_spent_amount += coin.GetTxOut().nValue; + --m_transaction_output_count; m_total_amount -= coin.GetTxOut().nValue; m_bogo_size -= GetBogoSize(coin.GetTxOut().scriptPubKey); } } } + } else { + // genesis block + m_total_unspendable_amount += block_subsidy; + m_total_unspendables_genesis_block += block_subsidy; } + // If spent prevouts + block subsidy are still a higher amount than + // new outputs + coinbase + current unspendable amount this means + // the miner did not claim the full block reward. Unclaimed block + // rewards are also unspendable. + const Amount unclaimed_rewards{ + (m_total_prevout_spent_amount + m_total_subsidy) - + (m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + + m_total_unspendable_amount)}; + m_total_unspendable_amount += unclaimed_rewards; + m_total_unspendables_unclaimed_rewards += unclaimed_rewards; + std::pair value; value.first = pindex->GetBlockHash(); value.second.transaction_output_count = m_transaction_output_count; value.second.bogo_size = m_bogo_size; value.second.total_amount = m_total_amount; + value.second.total_subsidy = m_total_subsidy; + value.second.total_unspendable_amount = m_total_unspendable_amount; + value.second.total_prevout_spent_amount = m_total_prevout_spent_amount; + value.second.total_new_outputs_ex_coinbase_amount = + m_total_new_outputs_ex_coinbase_amount; + value.second.total_coinbase_amount = m_total_coinbase_amount; + value.second.total_unspendables_genesis_block = + m_total_unspendables_genesis_block; + value.second.total_unspendables_bip30 = m_total_unspendables_bip30; + value.second.total_unspendables_scripts = m_total_unspendables_scripts; + value.second.total_unspendables_unclaimed_rewards = + m_total_unspendables_unclaimed_rewards; uint256 out; m_muhash.Finalize(out); value.second.muhash = out; CDBBatch batch(*m_db); batch.Write(DBHeightKey(pindex->nHeight), value); batch.Write(DB_MUHASH, m_muhash); return m_db->WriteBatch(batch); } static bool CopyHeightIndexToHashIndex(CDBIterator &db_it, CDBBatch &batch, const std::string &index_name, int start_height, int stop_height) { DBHeightKey key{start_height}; db_it.Seek(key); for (int height = start_height; height <= stop_height; ++height) { if (!db_it.GetKey(key) || key.height != height) { return error("%s: unexpected key in %s: expected (%c, %d)", __func__, index_name, DB_BLOCK_HEIGHT, height); } std::pair value; if (!db_it.GetValue(value)) { return error("%s: unable to read value in %s at key (%c, %d)", __func__, index_name, DB_BLOCK_HEIGHT, height); } batch.Write(DBHashKey(value.first), std::move(value.second)); db_it.Next(); } return true; } bool CoinStatsIndex::Rewind(const CBlockIndex *current_tip, const CBlockIndex *new_tip) { assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); CDBBatch batch(*m_db); std::unique_ptr db_it(m_db->NewIterator()); // During a reorg, we need to copy all hash digests for blocks that are // getting disconnected from the height index to the hash index so we can // still find them when the height index entries are overwritten. if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) { return false; } if (!m_db->WriteBatch(batch)) { return false; } { LOCK(cs_main); CBlockIndex *iter_tip{g_chainman.m_blockman.LookupBlockIndex( current_tip->GetBlockHash())}; const auto &consensus_params{Params().GetConsensus()}; do { CBlock block; if (!ReadBlockFromDisk(block, iter_tip, consensus_params)) { return error("%s: Failed to read block %s from disk", __func__, iter_tip->GetBlockHash().ToString()); } ReverseBlock(block, iter_tip); iter_tip = iter_tip->GetAncestor(iter_tip->nHeight - 1); } while (new_tip != iter_tip); } return BaseIndex::Rewind(current_tip, new_tip); } static bool LookUpOne(const CDBWrapper &db, const CBlockIndex *block_index, DBVal &result) { // First check if the result is stored under the height index and the value // there matches the block hash. This should be the case if the block is on // the active chain. std::pair read_out; if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) { return false; } if (read_out.first == block_index->GetBlockHash()) { result = std::move(read_out.second); return true; } // If value at the height index corresponds to an different block, the // result will be stored in the hash index. return db.Read(DBHashKey(block_index->GetBlockHash()), result); } bool CoinStatsIndex::LookUpStats(const CBlockIndex *block_index, CCoinsStats &coins_stats) const { DBVal entry; if (!LookUpOne(*m_db, block_index, entry)) { return false; } coins_stats.hashSerialized = entry.muhash; coins_stats.nTransactionOutputs = entry.transaction_output_count; coins_stats.nBogoSize = entry.bogo_size; coins_stats.nTotalAmount = entry.total_amount; + coins_stats.total_subsidy = entry.total_subsidy; + coins_stats.total_unspendable_amount = entry.total_unspendable_amount; + coins_stats.total_prevout_spent_amount = entry.total_prevout_spent_amount; + coins_stats.total_new_outputs_ex_coinbase_amount = + entry.total_new_outputs_ex_coinbase_amount; + coins_stats.total_coinbase_amount = entry.total_coinbase_amount; + coins_stats.total_unspendables_genesis_block = + entry.total_unspendables_genesis_block; + coins_stats.total_unspendables_bip30 = entry.total_unspendables_bip30; + coins_stats.total_unspendables_scripts = entry.total_unspendables_scripts; + coins_stats.total_unspendables_unclaimed_rewards = + entry.total_unspendables_unclaimed_rewards; return true; } bool CoinStatsIndex::Init() { if (!m_db->Read(DB_MUHASH, m_muhash)) { // Check that the cause of the read failure is that the key does not // exist. Any other errors indicate database corruption or a disk // failure, and starting the index would cause further corruption. if (m_db->Exists(DB_MUHASH)) { return error( "%s: Cannot read current %s state; index may be corrupted", __func__, GetName()); } } if (BaseIndex::Init()) { const CBlockIndex *pindex{CurrentIndex()}; if (pindex) { DBVal entry; if (!LookUpOne(*m_db, pindex, entry)) { return false; } m_transaction_output_count = entry.transaction_output_count; m_bogo_size = entry.bogo_size; m_total_amount = entry.total_amount; + m_total_subsidy = entry.total_subsidy; + m_total_unspendable_amount = entry.total_unspendable_amount; + m_total_prevout_spent_amount = entry.total_prevout_spent_amount; + m_total_new_outputs_ex_coinbase_amount = + entry.total_new_outputs_ex_coinbase_amount; + m_total_coinbase_amount = entry.total_coinbase_amount; + m_total_unspendables_genesis_block = + entry.total_unspendables_genesis_block; + m_total_unspendables_bip30 = entry.total_unspendables_bip30; + m_total_unspendables_scripts = entry.total_unspendables_scripts; + m_total_unspendables_unclaimed_rewards = + entry.total_unspendables_unclaimed_rewards; } return true; } return false; } // Reverse a single block as part of a reorg bool CoinStatsIndex::ReverseBlock(const CBlock &block, const CBlockIndex *pindex) { CBlockUndo block_undo; std::pair read_out; + const Amount block_subsidy{ + GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())}; + m_total_subsidy -= block_subsidy; + // Ignore genesis block if (pindex->nHeight > 0) { if (!UndoReadFromDisk(block_undo, pindex)) { return false; } if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { return false; } BlockHash expected_block_hash{pindex->pprev->GetBlockHash()}; if (read_out.first != expected_block_hash) { if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { return error("%s: previous block header belongs to unexpected " "block %s; expected %s", __func__, read_out.first.ToString(), expected_block_hash.ToString()); } } } // Remove the new UTXOs that were created from the block for (size_t i = 0; i < block.vtx.size(); ++i) { const auto &tx{block.vtx.at(i)}; for (uint32_t j = 0; j < tx->vout.size(); ++j) { const CTxOut &out{tx->vout[j]}; COutPoint outpoint{tx->GetId(), j}; Coin coin{out, static_cast(pindex->nHeight), tx->IsCoinBase()}; // Skip unspendable coins if (coin.GetTxOut().scriptPubKey.IsUnspendable()) { + m_total_unspendable_amount -= coin.GetTxOut().nValue; + m_total_unspendables_scripts -= coin.GetTxOut().nValue; continue; } m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); + + if (tx->IsCoinBase()) { + m_total_coinbase_amount -= coin.GetTxOut().nValue; + } else { + m_total_new_outputs_ex_coinbase_amount -= + coin.GetTxOut().nValue; + } + + --m_transaction_output_count; + m_total_amount -= coin.GetTxOut().nValue; + m_bogo_size -= GetBogoSize(coin.GetTxOut().scriptPubKey); } // The coinbase tx has no undo data since no former output is spent if (!tx->IsCoinBase()) { const auto &tx_undo{block_undo.vtxundo.at(i - 1)}; for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { Coin coin{tx_undo.vprevout[j]}; COutPoint outpoint{tx->vin[j].prevout.GetTxId(), tx->vin[j].prevout.GetN()}; m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); + + m_total_prevout_spent_amount -= coin.GetTxOut().nValue; + + m_transaction_output_count++; + m_total_amount += coin.GetTxOut().nValue; + m_bogo_size += GetBogoSize(coin.GetTxOut().scriptPubKey); } } } - // Check that the rolled back internal value of muhash is consistent with - // the DB read out + const Amount unclaimed_rewards{ + (m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + + m_total_unspendable_amount) - + (m_total_prevout_spent_amount + m_total_subsidy)}; + m_total_unspendable_amount -= unclaimed_rewards; + m_total_unspendables_unclaimed_rewards -= unclaimed_rewards; + + // Check that the rolled back internal values are consistent with the DB + // read out uint256 out; m_muhash.Finalize(out); Assert(read_out.second.muhash == out); - m_transaction_output_count = read_out.second.transaction_output_count; - m_total_amount = read_out.second.total_amount; - m_bogo_size = read_out.second.bogo_size; + Assert(m_transaction_output_count == + read_out.second.transaction_output_count); + Assert(m_total_amount == read_out.second.total_amount); + Assert(m_bogo_size == read_out.second.bogo_size); + Assert(m_total_subsidy == read_out.second.total_subsidy); + Assert(m_total_unspendable_amount == + read_out.second.total_unspendable_amount); + Assert(m_total_prevout_spent_amount == + read_out.second.total_prevout_spent_amount); + Assert(m_total_new_outputs_ex_coinbase_amount == + read_out.second.total_new_outputs_ex_coinbase_amount); + Assert(m_total_coinbase_amount == read_out.second.total_coinbase_amount); + Assert(m_total_unspendables_genesis_block == + read_out.second.total_unspendables_genesis_block); + Assert(m_total_unspendables_bip30 == + read_out.second.total_unspendables_bip30); + Assert(m_total_unspendables_scripts == + read_out.second.total_unspendables_scripts); + Assert(m_total_unspendables_unclaimed_rewards == + read_out.second.total_unspendables_unclaimed_rewards); return m_db->Write(DB_MUHASH, m_muhash); } diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h index 9180d39d8..bb72324f6 100644 --- a/src/index/coinstatsindex.h +++ b/src/index/coinstatsindex.h @@ -1,56 +1,65 @@ // Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_INDEX_COINSTATSINDEX_H #define BITCOIN_INDEX_COINSTATSINDEX_H #include #include #include #include #include struct Amount; /** * CoinStatsIndex maintains statistics on the UTXO set. */ class CoinStatsIndex final : public BaseIndex { private: std::string m_name; std::unique_ptr m_db; MuHash3072 m_muhash; uint64_t m_transaction_output_count{0}; uint64_t m_bogo_size{0}; Amount m_total_amount{Amount::zero()}; + Amount m_total_subsidy{Amount::zero()}; + Amount m_total_unspendable_amount{Amount::zero()}; + Amount m_total_prevout_spent_amount{Amount::zero()}; + Amount m_total_new_outputs_ex_coinbase_amount{Amount::zero()}; + Amount m_total_coinbase_amount{Amount::zero()}; + Amount m_total_unspendables_genesis_block{Amount::zero()}; + Amount m_total_unspendables_bip30{Amount::zero()}; + Amount m_total_unspendables_scripts{Amount::zero()}; + Amount m_total_unspendables_unclaimed_rewards{Amount::zero()}; bool ReverseBlock(const CBlock &block, const CBlockIndex *pindex); protected: bool Init() override; bool WriteBlock(const CBlock &block, const CBlockIndex *pindex) override; bool Rewind(const CBlockIndex *current_tip, const CBlockIndex *new_tip) override; BaseIndex::DB &GetDB() const override { return *m_db; } const char *GetName() const override { return "coinstatsindex"; } public: // Constructs the index, which becomes available to be queried. explicit CoinStatsIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); // Look up stats for a specific block using CBlockIndex bool LookUpStats(const CBlockIndex *block_index, CCoinsStats &coins_stats) const; }; /// The global UTXO set hash object. extern std::unique_ptr g_coin_stats_index; #endif // BITCOIN_INDEX_COINSTATSINDEX_H diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 55c072e00..fde224098 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -1,56 +1,67 @@ // Copyright (c) 2010 Satoshi Nakamoto // Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_NODE_COINSTATS_H #define BITCOIN_NODE_COINSTATS_H #include #include #include #include #include #include #include #include class BlockManager; class CCoinsView; enum class CoinStatsHashType { HASH_SERIALIZED, MUHASH, NONE, }; struct CCoinsStats { CoinStatsHashType m_hash_type; int nHeight{0}; BlockHash hashBlock{}; uint64_t nTransactions{0}; uint64_t nTransactionOutputs{0}; uint64_t nBogoSize{0}; uint256 hashSerialized{}; uint64_t nDiskSize{0}; Amount nTotalAmount{Amount::zero()}; //! The number of coins contained. uint64_t coins_count{0}; bool from_index{false}; + // Following values are only available from coinstats index + Amount total_subsidy{Amount::zero()}; + Amount total_unspendable_amount{Amount::zero()}; + Amount total_prevout_spent_amount{Amount::zero()}; + Amount total_new_outputs_ex_coinbase_amount{Amount::zero()}; + Amount total_coinbase_amount{Amount::zero()}; + Amount total_unspendables_genesis_block{Amount::zero()}; + Amount total_unspendables_bip30{Amount::zero()}; + Amount total_unspendables_scripts{Amount::zero()}; + Amount total_unspendables_unclaimed_rewards{Amount::zero()}; + CCoinsStats(CoinStatsHashType hash_type) : m_hash_type(hash_type) {} }; //! Calculate statistics about the unspent transaction output set bool GetUTXOStats(CCoinsView *view, BlockManager &blockman, CCoinsStats &stats, const std::function &interruption_point = {}, const CBlockIndex *pindex = nullptr); uint64_t GetBogoSize(const CScript &script_pub_key); CDataStream TxOutSer(const COutPoint &outpoint, const Coin &coin); #endif // BITCOIN_NODE_COINSTATS_H diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 30f87fd9a..861f9851a 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1,3244 +1,3320 @@ // Copyright (c) 2010 Satoshi Nakamoto // Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include