diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index 484d0b925..53f36391a 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -1,544 +1,551 @@ // Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include #include #include #include #include using node::CCoinsStats; using node::GetBogoSize; using node::ReadBlockFromDisk; using node::TxOutSer; using node::UndoReadFromDisk; static constexpr char DB_BLOCK_HASH = 's'; static constexpr char DB_BLOCK_HEIGHT = 't'; static constexpr char DB_MUHASH = 'M'; namespace { struct DBVal { uint256 muhash; uint64_t transaction_output_count; uint64_t bogo_size; Amount total_amount; Amount total_subsidy; Amount total_unspendable_amount; Amount total_prevout_spent_amount; Amount total_new_outputs_ex_coinbase_amount; Amount total_coinbase_amount; Amount total_unspendables_genesis_block; Amount total_unspendables_bip30; Amount total_unspendables_scripts; Amount total_unspendables_unclaimed_rewards; SERIALIZE_METHODS(DBVal, obj) { READWRITE(obj.muhash); READWRITE(obj.transaction_output_count); READWRITE(obj.bogo_size); READWRITE(obj.total_amount); READWRITE(obj.total_subsidy); READWRITE(obj.total_unspendable_amount); READWRITE(obj.total_prevout_spent_amount); READWRITE(obj.total_new_outputs_ex_coinbase_amount); READWRITE(obj.total_coinbase_amount); READWRITE(obj.total_unspendables_genesis_block); READWRITE(obj.total_unspendables_bip30); READWRITE(obj.total_unspendables_scripts); READWRITE(obj.total_unspendables_unclaimed_rewards); } }; struct DBHeightKey { int height; explicit DBHeightKey(int height_in) : height(height_in) {} template void Serialize(Stream &s) const { ser_writedata8(s, DB_BLOCK_HEIGHT); ser_writedata32be(s, height); } template void Unserialize(Stream &s) { char prefix{static_cast(ser_readdata8(s))}; if (prefix != DB_BLOCK_HEIGHT) { throw std::ios_base::failure( "Invalid format for coinstatsindex DB height key"); } height = ser_readdata32be(s); } }; struct DBHashKey { BlockHash block_hash; explicit DBHashKey(const BlockHash &hash_in) : block_hash(hash_in) {} SERIALIZE_METHODS(DBHashKey, obj) { char prefix{DB_BLOCK_HASH}; READWRITE(prefix); if (prefix != DB_BLOCK_HASH) { throw std::ios_base::failure( "Invalid format for coinstatsindex DB hash key"); } READWRITE(obj.block_hash); } }; }; // namespace std::unique_ptr g_coin_stats_index; CoinStatsIndex::CoinStatsIndex(size_t n_cache_size, bool f_memory, bool f_wipe) { fs::path path{gArgs.GetDataDirNet() / "indexes" / "coinstats"}; fs::create_directories(path); m_db = std::make_unique(path / "db", n_cache_size, f_memory, f_wipe); } bool CoinStatsIndex::WriteBlock(const CBlock &block, const CBlockIndex *pindex) { CBlockUndo block_undo; const Amount block_subsidy{ GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())}; m_total_subsidy += block_subsidy; // Ignore genesis block if (pindex->nHeight > 0) { if (!UndoReadFromDisk(block_undo, pindex)) { return false; } std::pair read_out; if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { return false; } BlockHash expected_block_hash{pindex->pprev->GetBlockHash()}; if (read_out.first != expected_block_hash) { LogPrintf("WARNING: previous block header belongs to unexpected " "block %s; expected %s\n", read_out.first.ToString(), expected_block_hash.ToString()); if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { return error("%s: previous block header not found; expected %s", __func__, expected_block_hash.ToString()); } } // TODO: Deduplicate BIP30 related code bool is_bip30_block{ (pindex->nHeight == 91722 && pindex->GetBlockHash() == BlockHash{uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc" "6955e5a6c6cdf3f2574dd08e")}) || (pindex->nHeight == 91812 && pindex->GetBlockHash() == BlockHash{uint256S("0x00000000000af0aed4792b1acee3d966af36cf5d" "ef14935db8de83d6f9306f2f")})}; // Add the new utxos created from the block for (size_t i = 0; i < block.vtx.size(); ++i) { const auto &tx{block.vtx.at(i)}; // Skip duplicate txid coinbase transactions (BIP30). if (is_bip30_block && tx->IsCoinBase()) { m_total_unspendable_amount += block_subsidy; m_total_unspendables_bip30 += block_subsidy; continue; } for (uint32_t j = 0; j < tx->vout.size(); ++j) { const CTxOut &out{tx->vout[j]}; Coin coin{out, static_cast(pindex->nHeight), tx->IsCoinBase()}; COutPoint outpoint{tx->GetId(), j}; // Skip unspendable coins if (coin.GetTxOut().scriptPubKey.IsUnspendable()) { m_total_unspendable_amount += coin.GetTxOut().nValue; m_total_unspendables_scripts += coin.GetTxOut().nValue; continue; } m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); if (tx->IsCoinBase()) { m_total_coinbase_amount += coin.GetTxOut().nValue; } else { m_total_new_outputs_ex_coinbase_amount += coin.GetTxOut().nValue; } ++m_transaction_output_count; m_total_amount += coin.GetTxOut().nValue; m_bogo_size += GetBogoSize(coin.GetTxOut().scriptPubKey); } // The coinbase tx has no undo data since no former output is spent if (!tx->IsCoinBase()) { const auto &tx_undo{block_undo.vtxundo.at(i - 1)}; for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { Coin coin{tx_undo.vprevout[j]}; COutPoint outpoint{tx->vin[j].prevout.GetTxId(), tx->vin[j].prevout.GetN()}; m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); m_total_prevout_spent_amount += coin.GetTxOut().nValue; --m_transaction_output_count; m_total_amount -= coin.GetTxOut().nValue; m_bogo_size -= GetBogoSize(coin.GetTxOut().scriptPubKey); } } } } else { // genesis block m_total_unspendable_amount += block_subsidy; m_total_unspendables_genesis_block += block_subsidy; } // If spent prevouts + block subsidy are still a higher amount than // new outputs + coinbase + current unspendable amount this means // the miner did not claim the full block reward. Unclaimed block // rewards are also unspendable. const Amount unclaimed_rewards{ (m_total_prevout_spent_amount + m_total_subsidy) - (m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + m_total_unspendable_amount)}; m_total_unspendable_amount += unclaimed_rewards; m_total_unspendables_unclaimed_rewards += unclaimed_rewards; std::pair value; value.first = pindex->GetBlockHash(); value.second.transaction_output_count = m_transaction_output_count; value.second.bogo_size = m_bogo_size; value.second.total_amount = m_total_amount; value.second.total_subsidy = m_total_subsidy; value.second.total_unspendable_amount = m_total_unspendable_amount; value.second.total_prevout_spent_amount = m_total_prevout_spent_amount; value.second.total_new_outputs_ex_coinbase_amount = m_total_new_outputs_ex_coinbase_amount; value.second.total_coinbase_amount = m_total_coinbase_amount; value.second.total_unspendables_genesis_block = m_total_unspendables_genesis_block; value.second.total_unspendables_bip30 = m_total_unspendables_bip30; value.second.total_unspendables_scripts = m_total_unspendables_scripts; value.second.total_unspendables_unclaimed_rewards = m_total_unspendables_unclaimed_rewards; uint256 out; m_muhash.Finalize(out); value.second.muhash = out; - CDBBatch batch(*m_db); - batch.Write(DBHeightKey(pindex->nHeight), value); - batch.Write(DB_MUHASH, m_muhash); - return m_db->WriteBatch(batch); + // Intentionally do not update DB_MUHASH here so it stays in sync with + // DB_BEST_BLOCK, and the index is not corrupted if there is an unclean + // shutdown. + return m_db->Write(DBHeightKey(pindex->nHeight), value); } static bool CopyHeightIndexToHashIndex(CDBIterator &db_it, CDBBatch &batch, const std::string &index_name, int start_height, int stop_height) { DBHeightKey key{start_height}; db_it.Seek(key); for (int height = start_height; height <= stop_height; ++height) { if (!db_it.GetKey(key) || key.height != height) { return error("%s: unexpected key in %s: expected (%c, %d)", __func__, index_name, DB_BLOCK_HEIGHT, height); } std::pair value; if (!db_it.GetValue(value)) { return error("%s: unable to read value in %s at key (%c, %d)", __func__, index_name, DB_BLOCK_HEIGHT, height); } batch.Write(DBHashKey(value.first), std::move(value.second)); db_it.Next(); } return true; } bool CoinStatsIndex::Rewind(const CBlockIndex *current_tip, const CBlockIndex *new_tip) { assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); CDBBatch batch(*m_db); std::unique_ptr db_it(m_db->NewIterator()); // During a reorg, we need to copy all hash digests for blocks that are // getting disconnected from the height index to the hash index so we can // still find them when the height index entries are overwritten. if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) { return false; } if (!m_db->WriteBatch(batch)) { return false; } { LOCK(cs_main); const CBlockIndex *iter_tip{m_chainstate->m_blockman.LookupBlockIndex( current_tip->GetBlockHash())}; const auto &consensus_params{Params().GetConsensus()}; do { CBlock block; if (!ReadBlockFromDisk(block, iter_tip, consensus_params)) { return error("%s: Failed to read block %s from disk", __func__, iter_tip->GetBlockHash().ToString()); } ReverseBlock(block, iter_tip); iter_tip = iter_tip->GetAncestor(iter_tip->nHeight - 1); } while (new_tip != iter_tip); } return BaseIndex::Rewind(current_tip, new_tip); } static bool LookUpOne(const CDBWrapper &db, const CBlockIndex *block_index, DBVal &result) { // First check if the result is stored under the height index and the value // there matches the block hash. This should be the case if the block is on // the active chain. std::pair read_out; if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) { return false; } if (read_out.first == block_index->GetBlockHash()) { result = std::move(read_out.second); return true; } // If value at the height index corresponds to an different block, the // result will be stored in the hash index. return db.Read(DBHashKey(block_index->GetBlockHash()), result); } bool CoinStatsIndex::LookUpStats(const CBlockIndex *block_index, CCoinsStats &coins_stats) const { DBVal entry; if (!LookUpOne(*m_db, block_index, entry)) { return false; } coins_stats.hashSerialized = entry.muhash; coins_stats.nTransactionOutputs = entry.transaction_output_count; coins_stats.nBogoSize = entry.bogo_size; coins_stats.nTotalAmount = entry.total_amount; coins_stats.total_subsidy = entry.total_subsidy; coins_stats.total_unspendable_amount = entry.total_unspendable_amount; coins_stats.total_prevout_spent_amount = entry.total_prevout_spent_amount; coins_stats.total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount; coins_stats.total_coinbase_amount = entry.total_coinbase_amount; coins_stats.total_unspendables_genesis_block = entry.total_unspendables_genesis_block; coins_stats.total_unspendables_bip30 = entry.total_unspendables_bip30; coins_stats.total_unspendables_scripts = entry.total_unspendables_scripts; coins_stats.total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards; return true; } bool CoinStatsIndex::Init() { if (!m_db->Read(DB_MUHASH, m_muhash)) { // Check that the cause of the read failure is that the key does not // exist. Any other errors indicate database corruption or a disk // failure, and starting the index would cause further corruption. if (m_db->Exists(DB_MUHASH)) { return error( "%s: Cannot read current %s state; index may be corrupted", __func__, GetName()); } } if (!BaseIndex::Init()) { return false; } const CBlockIndex *pindex{CurrentIndex()}; if (pindex) { DBVal entry; if (!LookUpOne(*m_db, pindex, entry)) { return error( "%s: Cannot read current %s state; index may be corrupted", __func__, GetName()); } uint256 out; m_muhash.Finalize(out); if (entry.muhash != out) { return error( "%s: Cannot read current %s state; index may be corrupted", __func__, GetName()); } m_transaction_output_count = entry.transaction_output_count; m_bogo_size = entry.bogo_size; m_total_amount = entry.total_amount; m_total_subsidy = entry.total_subsidy; m_total_unspendable_amount = entry.total_unspendable_amount; m_total_prevout_spent_amount = entry.total_prevout_spent_amount; m_total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount; m_total_coinbase_amount = entry.total_coinbase_amount; m_total_unspendables_genesis_block = entry.total_unspendables_genesis_block; m_total_unspendables_bip30 = entry.total_unspendables_bip30; m_total_unspendables_scripts = entry.total_unspendables_scripts; m_total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards; } return true; } +bool CoinStatsIndex::CommitInternal(CDBBatch &batch) { + // DB_MUHASH should always be committed in a batch together with + // DB_BEST_BLOCK to prevent an inconsistent state of the DB. + batch.Write(DB_MUHASH, m_muhash); + return BaseIndex::CommitInternal(batch); +} + // Reverse a single block as part of a reorg bool CoinStatsIndex::ReverseBlock(const CBlock &block, const CBlockIndex *pindex) { CBlockUndo block_undo; std::pair read_out; const Amount block_subsidy{ GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())}; m_total_subsidy -= block_subsidy; // Ignore genesis block if (pindex->nHeight > 0) { if (!UndoReadFromDisk(block_undo, pindex)) { return false; } if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { return false; } BlockHash expected_block_hash{pindex->pprev->GetBlockHash()}; if (read_out.first != expected_block_hash) { LogPrintf("WARNING: previous block header belongs to unexpected " "block %s; expected %s\n", read_out.first.ToString(), expected_block_hash.ToString()); if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { return error("%s: previous block header not found; expected %s", __func__, expected_block_hash.ToString()); } } } // Remove the new UTXOs that were created from the block for (size_t i = 0; i < block.vtx.size(); ++i) { const auto &tx{block.vtx.at(i)}; for (uint32_t j = 0; j < tx->vout.size(); ++j) { const CTxOut &out{tx->vout[j]}; COutPoint outpoint{tx->GetId(), j}; Coin coin{out, static_cast(pindex->nHeight), tx->IsCoinBase()}; // Skip unspendable coins if (coin.GetTxOut().scriptPubKey.IsUnspendable()) { m_total_unspendable_amount -= coin.GetTxOut().nValue; m_total_unspendables_scripts -= coin.GetTxOut().nValue; continue; } m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); if (tx->IsCoinBase()) { m_total_coinbase_amount -= coin.GetTxOut().nValue; } else { m_total_new_outputs_ex_coinbase_amount -= coin.GetTxOut().nValue; } --m_transaction_output_count; m_total_amount -= coin.GetTxOut().nValue; m_bogo_size -= GetBogoSize(coin.GetTxOut().scriptPubKey); } // The coinbase tx has no undo data since no former output is spent if (!tx->IsCoinBase()) { const auto &tx_undo{block_undo.vtxundo.at(i - 1)}; for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { Coin coin{tx_undo.vprevout[j]}; COutPoint outpoint{tx->vin[j].prevout.GetTxId(), tx->vin[j].prevout.GetN()}; m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); m_total_prevout_spent_amount -= coin.GetTxOut().nValue; m_transaction_output_count++; m_total_amount += coin.GetTxOut().nValue; m_bogo_size += GetBogoSize(coin.GetTxOut().scriptPubKey); } } } const Amount unclaimed_rewards{ (m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + m_total_unspendable_amount) - (m_total_prevout_spent_amount + m_total_subsidy)}; m_total_unspendable_amount -= unclaimed_rewards; m_total_unspendables_unclaimed_rewards -= unclaimed_rewards; // Check that the rolled back internal values are consistent with the DB // read out uint256 out; m_muhash.Finalize(out); Assert(read_out.second.muhash == out); Assert(m_transaction_output_count == read_out.second.transaction_output_count); Assert(m_total_amount == read_out.second.total_amount); Assert(m_bogo_size == read_out.second.bogo_size); Assert(m_total_subsidy == read_out.second.total_subsidy); Assert(m_total_unspendable_amount == read_out.second.total_unspendable_amount); Assert(m_total_prevout_spent_amount == read_out.second.total_prevout_spent_amount); Assert(m_total_new_outputs_ex_coinbase_amount == read_out.second.total_new_outputs_ex_coinbase_amount); Assert(m_total_coinbase_amount == read_out.second.total_coinbase_amount); Assert(m_total_unspendables_genesis_block == read_out.second.total_unspendables_genesis_block); Assert(m_total_unspendables_bip30 == read_out.second.total_unspendables_bip30); Assert(m_total_unspendables_scripts == read_out.second.total_unspendables_scripts); Assert(m_total_unspendables_unclaimed_rewards == read_out.second.total_unspendables_unclaimed_rewards); - return m_db->Write(DB_MUHASH, m_muhash); + return true; } diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h index e1519e8d4..f9c41cbdd 100644 --- a/src/index/coinstatsindex.h +++ b/src/index/coinstatsindex.h @@ -1,65 +1,67 @@ // Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_INDEX_COINSTATSINDEX_H #define BITCOIN_INDEX_COINSTATSINDEX_H #include #include #include #include #include struct Amount; /** * CoinStatsIndex maintains statistics on the UTXO set. */ class CoinStatsIndex final : public BaseIndex { private: std::string m_name; std::unique_ptr m_db; MuHash3072 m_muhash; uint64_t m_transaction_output_count{0}; uint64_t m_bogo_size{0}; Amount m_total_amount{Amount::zero()}; Amount m_total_subsidy{Amount::zero()}; Amount m_total_unspendable_amount{Amount::zero()}; Amount m_total_prevout_spent_amount{Amount::zero()}; Amount m_total_new_outputs_ex_coinbase_amount{Amount::zero()}; Amount m_total_coinbase_amount{Amount::zero()}; Amount m_total_unspendables_genesis_block{Amount::zero()}; Amount m_total_unspendables_bip30{Amount::zero()}; Amount m_total_unspendables_scripts{Amount::zero()}; Amount m_total_unspendables_unclaimed_rewards{Amount::zero()}; bool ReverseBlock(const CBlock &block, const CBlockIndex *pindex); protected: bool Init() override; + bool CommitInternal(CDBBatch &batch) override; + bool WriteBlock(const CBlock &block, const CBlockIndex *pindex) override; bool Rewind(const CBlockIndex *current_tip, const CBlockIndex *new_tip) override; BaseIndex::DB &GetDB() const override { return *m_db; } const char *GetName() const override { return "coinstatsindex"; } public: // Constructs the index, which becomes available to be queried. explicit CoinStatsIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); // Look up stats for a specific block using CBlockIndex bool LookUpStats(const CBlockIndex *block_index, node::CCoinsStats &coins_stats) const; }; /// The global UTXO set hash object. extern std::unique_ptr g_coin_stats_index; #endif // BITCOIN_INDEX_COINSTATSINDEX_H diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 2b4f36b17..ea69fdf8a 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -1,305 +1,306 @@ # Copyright (c) 2018 The Bitcoin developers project(bitcoin-test) # Process json files. file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/data") function(gen_json_header NAME) set(HEADERS "") foreach(f ${ARGN}) set(h "${CMAKE_CURRENT_BINARY_DIR}/${f}.h") # Get the proper name for the test variable. get_filename_component(TEST_NAME ${f} NAME_WE) add_custom_command(OUTPUT ${h} COMMAND "${Python_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/data/generate_header.py" "${TEST_NAME}" "${CMAKE_CURRENT_SOURCE_DIR}/${f}" > ${h} MAIN_DEPENDENCY ${f} DEPENDS "data/generate_header.py" VERBATIM ) list(APPEND HEADERS ${h}) endforeach(f) set(${NAME} "${HEADERS}" PARENT_SCOPE) endfunction() gen_json_header(JSON_HEADERS data/base58_encode_decode.json data/blockfilters.json data/key_io_valid.json data/key_io_invalid.json data/script_tests.json data/sighash.json data/tx_invalid.json data/tx_valid.json ) include(TestSuite) create_test_suite(bitcoin) add_dependencies(check check-bitcoin) # An utility library for bitcoin related test suites. add_library(testutil OBJECT util/blockfilter.cpp util/blockindex.cpp util/logging.cpp util/mining.cpp util/net.cpp util/setup_common.cpp util/str.cpp util/transaction_utils.cpp + util/validation.cpp util/wallet.cpp ) target_link_libraries(testutil server) if(BUILD_BITCOIN_WALLET) set(BITCOIN_WALLET_TEST_FIXTURE ../wallet/test/init_test_fixture.cpp ../wallet/test/wallet_test_fixture.cpp ) set(BITCOIN_WALLET_TESTS ../wallet/test/db_tests.cpp ../wallet/test/coinselector_tests.cpp ../wallet/test/init_tests.cpp ../wallet/test/ismine_tests.cpp ../wallet/test/psbt_wallet_tests.cpp ../wallet/test/scriptpubkeyman_tests.cpp ../wallet/test/wallet_tests.cpp ../wallet/test/walletdb_tests.cpp ../wallet/test/wallet_crypto_tests.cpp ) endif() function(gen_asmap_headers HEADERS_VAR) foreach(INPUT_FILE ${ARGN}) set(OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/${INPUT_FILE}.h") add_custom_command( OUTPUT "${OUTPUT_FILE}" COMMENT "Generate ASMAP header from ${INPUT_FILE}" COMMAND "${Python_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/data/generate_asmap.py" "${CMAKE_CURRENT_SOURCE_DIR}/${INPUT_FILE}" "${OUTPUT_FILE}" MAIN_DEPENDENCY "${INPUT_FILE}" DEPENDS "data/generate_header.py" VERBATIM ) list(APPEND ${HEADERS_VAR} "${OUTPUT_FILE}") endforeach() set(${HEADERS_VAR} ${${HEADERS_VAR}} PARENT_SCOPE) endfunction() gen_asmap_headers(ASMAP_HEADERS data/asmap.raw ) if(BUILD_BITCOIN_CHRONIK) set(BITCOIN_CHRONIK_TESTS ../../chronik/test/bridgecompression_tests.cpp ../../chronik/test/bridgeprimitives_tests.cpp ../../chronik/test/chronikbridge_tests.cpp ) endif() add_boost_unit_tests_to_suite(bitcoin test_bitcoin fixture.cpp jsonutil.cpp scriptflags.cpp sigutil.cpp ${ASMAP_HEADERS} # Tests generated from JSON ${JSON_HEADERS} # Wallet test fixture ${BITCOIN_WALLET_TEST_FIXTURE} TESTS activation_tests.cpp addrman_tests.cpp allocator_tests.cpp amount_tests.cpp arith_uint256_tests.cpp base32_tests.cpp base58_tests.cpp base64_tests.cpp bip32_tests.cpp bitmanip_tests.cpp blockchain_tests.cpp blockcheck_tests.cpp blockencodings_tests.cpp blockfilter_tests.cpp blockfilter_index_tests.cpp blockindex_tests.cpp blockstatus_tests.cpp blockstorage_tests.cpp bloom_tests.cpp bswap_tests.cpp cashaddr_tests.cpp cashaddrenc_tests.cpp checkdatasig_tests.cpp checkpoints_tests.cpp checkqueue_tests.cpp coins_tests.cpp coinstatsindex_tests.cpp compilerbug_tests.cpp compress_tests.cpp config_tests.cpp core_io_tests.cpp crypto_tests.cpp cuckoocache_tests.cpp dbwrapper_tests.cpp denialofservice_tests.cpp descriptor_tests.cpp dnsseeds_tests.cpp dstencode_tests.cpp feerate_tests.cpp flatfile_tests.cpp fs_tests.cpp getarg_tests.cpp hash_tests.cpp hasher_tests.cpp i2p_tests.cpp interfaces_tests.cpp intmath_tests.cpp inv_tests.cpp key_io_tests.cpp key_tests.cpp lcg_tests.cpp logging_tests.cpp mempool_tests.cpp merkle_tests.cpp merkleblock_tests.cpp miner_tests.cpp minerfund_tests.cpp monolith_opcodes_tests.cpp multisig_tests.cpp net_peer_eviction_tests.cpp net_tests.cpp netbase_tests.cpp op_reversebytes_tests.cpp pmt_tests.cpp policy_block_tests.cpp policy_fee_tests.cpp policyestimator_tests.cpp prevector_tests.cpp radix_tests.cpp raii_event_tests.cpp random_tests.cpp rcu_tests.cpp reverselock_tests.cpp rpc_tests.cpp rpc_server_tests.cpp rwcollection_tests.cpp sanity_tests.cpp scheduler_tests.cpp schnorr_tests.cpp script_bitfield_tests.cpp script_commitment_tests.cpp script_p2sh_tests.cpp script_standard_tests.cpp script_tests.cpp scriptnum_tests.cpp serialize_tests.cpp settings_tests.cpp shortidprocessor_tests.cpp sigcache_tests.cpp sigencoding_tests.cpp sighash_tests.cpp sighashtype_tests.cpp sigcheckcount_tests.cpp skiplist_tests.cpp sock_tests.cpp streams_tests.cpp sync_tests.cpp timedata_tests.cpp torcontrol_tests.cpp transaction_tests.cpp txindex_tests.cpp txpackage_tests.cpp txrequest_tests.cpp txvalidation_tests.cpp txvalidationcache_tests.cpp uint256_tests.cpp undo_tests.cpp util_tests.cpp util_threadnames_tests.cpp validation_block_tests.cpp validation_chainstate_tests.cpp validation_chainstatemanager_tests.cpp validation_flush_tests.cpp validation_tests.cpp validationinterface_tests.cpp blockindex_comparator_tests.cpp # RPC Tests ../rpc/test/server_tests.cpp # Wallet tests ${BITCOIN_WALLET_TESTS} # Chronik tests ${BITCOIN_CHRONIK_TESTS} ) function(add_boost_test_runners_with_upgrade_activated SUITE EXECUTABLE) set(SUITE_UPGRADE_ACTIVATED "${SUITE}-upgrade-activated") get_target_from_suite(${SUITE_UPGRADE_ACTIVATED} TARGET_UPGRADE_ACTIVATED) if(NOT TARGET ${TARGET_UPGRADE_ACTIVATED}) create_test_suite_with_parent_targets( ${SUITE_UPGRADE_ACTIVATED} check-upgrade-activated check-upgrade-activated-extended ) add_dependencies(${TARGET_UPGRADE_ACTIVATED} ${EXECUTABLE}) endif() get_target_from_suite(${SUITE} SUITE_TARGET) get_target_property(BOOST_TESTS ${SUITE_TARGET} UNIT_TESTS) get_target_from_suite(${SUITE_UPGRADE_ACTIVATED} SUITE_UPGRADE_ACTIVATED_TARGET) set(HRF_LOGGER "HRF,test_suite") foreach(_test_name ${BOOST_TESTS}) if(ENABLE_JUNIT_REPORT) set(JUNIT_LOGGER ":JUNIT,message,${SUITE_UPGRADE_ACTIVATED}-${_test_name}.xml") endif() add_test_runner( ${SUITE_UPGRADE_ACTIVATED} "${_test_name}" ${EXECUTABLE} JUNIT "--run_test=${_test_name}" "--logger=${HRF_LOGGER}${JUNIT_LOGGER}" "--catch_system_errors=no" -- "-testsuitename=Bitcoin ABC unit tests with next upgrade activated" # Sep. 20th, 2022 at 12:00:00 -wellingtonactivationtime=1663675200 ) endforeach() endfunction() add_boost_test_runners_with_upgrade_activated(bitcoin test_bitcoin) target_link_libraries(test_bitcoin rpcclient testutil) if(TARGET bitcoinconsensus-shared) target_link_libraries(test_bitcoin bitcoinconsensus-shared) else() target_link_libraries(test_bitcoin bitcoinconsensus) endif() add_subdirectory(fuzz) diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index 9bce0199d..06e3f52dc 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -1,81 +1,143 @@ // Copyright (c) 2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include +#include #include #include +#include #include #include #include #include using node::CCoinsStats; using node::CoinStatsHashType; BOOST_AUTO_TEST_SUITE(coinstatsindex_tests) +static void IndexWaitSynced(BaseIndex &index) { + // Allow the CoinStatsIndex to catch up with the block index that is syncing + // in a background thread. + const auto timeout = GetTime() + 120s; + while (!index.BlockUntilSyncedToCurrentChain()) { + BOOST_REQUIRE(timeout > GetTime()); + UninterruptibleSleep(100ms); + } +} + BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) { CoinStatsIndex coin_stats_index{1 << 20, true}; CCoinsStats coin_stats{CoinStatsHashType::MUHASH}; const CBlockIndex *block_index; { LOCK(cs_main); block_index = m_node.chainman->ActiveTip(); } // CoinStatsIndex should not be found before it is started. BOOST_CHECK(!coin_stats_index.LookUpStats(block_index, coin_stats)); // BlockUntilSyncedToCurrentChain should return false before CoinStatsIndex // is started. BOOST_CHECK(!coin_stats_index.BlockUntilSyncedToCurrentChain()); BOOST_REQUIRE(coin_stats_index.Start(m_node.chainman->ActiveChainstate())); - // Allow the CoinStatsIndex to catch up with the block index that is syncing - // in a background thread. - const auto timeout = GetTime() + 120s; - while (!coin_stats_index.BlockUntilSyncedToCurrentChain()) { - BOOST_REQUIRE(timeout > GetTime()); - UninterruptibleSleep(100ms); - } + IndexWaitSynced(coin_stats_index); // Check that CoinStatsIndex works for genesis block. const CBlockIndex *genesis_block_index; { LOCK(cs_main); genesis_block_index = m_node.chainman->ActiveChain().Genesis(); } BOOST_CHECK(coin_stats_index.LookUpStats(genesis_block_index, coin_stats)); // Check that CoinStatsIndex updates with new blocks. coin_stats_index.LookUpStats(block_index, coin_stats); const CScript script_pub_key{ CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG}; std::vector noTxns; CreateAndProcessBlock(noTxns, script_pub_key); // Let the CoinStatsIndex to catch up again. BOOST_CHECK(coin_stats_index.BlockUntilSyncedToCurrentChain()); CCoinsStats new_coin_stats{CoinStatsHashType::MUHASH}; const CBlockIndex *new_block_index; { LOCK(cs_main); new_block_index = m_node.chainman->ActiveTip(); } coin_stats_index.LookUpStats(new_block_index, new_coin_stats); BOOST_CHECK(block_index != new_block_index); // Shutdown sequence (c.f. Shutdown() in init.cpp) coin_stats_index.Stop(); // Rest of shutdown sequence and destructors happen in ~TestingSetup() } +// Test shutdown between BlockConnected and ChainStateFlushed notifications, +// make sure index is not corrupted and is able to reload. +BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup) { + Chainstate &chainstate = Assert(m_node.chainman)->ActiveChainstate(); + const Config &config = GetConfig(); + { + CoinStatsIndex index{1 << 20}; + BOOST_REQUIRE(index.Start(chainstate)); + IndexWaitSynced(index); + std::shared_ptr new_block; + CBlockIndex *new_block_index = nullptr; + { + const CScript script_pub_key{ + CScript() << ToByteVector(coinbaseKey.GetPubKey()) + << OP_CHECKSIG}; + const CBlock block = + this->CreateBlock({}, script_pub_key, chainstate); + + new_block = std::make_shared(block); + + LOCK(cs_main); + BlockValidationState state; + BlockValidationOptions options{config}; + BOOST_CHECK(CheckBlock( + block, state, config.GetChainParams().GetConsensus(), options)); + BOOST_CHECK(chainstate.AcceptBlock(config, new_block, state, true, + nullptr, nullptr)); + + // Get the block index (not returned by AcceptBlock since D2127) + auto it{m_node.chainman->m_blockman.m_block_index.find( + new_block->GetHash())}; + if (it != m_node.chainman->m_blockman.m_block_index.end()) { + new_block_index = &(it->second); + } + + CCoinsViewCache view(&chainstate.CoinsTip()); + BOOST_CHECK(chainstate.ConnectBlock(block, state, new_block_index, + view, options)); + } + // Send block connected notification, then stop the index without + // sending a chainstate flushed notification. Prior to #24138, this + // would cause the index to be corrupted and fail to reload. + ValidationInterfaceTest::BlockConnected(index, new_block, + new_block_index); + index.Stop(); + } + + { + CoinStatsIndex index{1 << 20}; + // Make sure the index can be loaded. + BOOST_REQUIRE(index.Start(chainstate)); + index.Stop(); + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util/validation.cpp b/src/test/util/validation.cpp new file mode 100644 index 000000000..7bb1eb1ed --- /dev/null +++ b/src/test/util/validation.cpp @@ -0,0 +1,14 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include + +void ValidationInterfaceTest::BlockConnected( + CValidationInterface &obj, const std::shared_ptr &block, + const CBlockIndex *pindex) { + obj.BlockConnected(block, pindex); +} diff --git a/src/test/util/validation.h b/src/test/util/validation.h new file mode 100644 index 000000000..8d73e7cf2 --- /dev/null +++ b/src/test/util/validation.h @@ -0,0 +1,19 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_VALIDATION_H +#define BITCOIN_TEST_UTIL_VALIDATION_H + +#include + +class CValidationInterface; + +class ValidationInterfaceTest { +public: + static void BlockConnected(CValidationInterface &obj, + const std::shared_ptr &block, + const CBlockIndex *pindex); +}; + +#endif // BITCOIN_TEST_UTIL_VALIDATION_H diff --git a/src/validationinterface.h b/src/validationinterface.h index b1e70db0a..acf457efc 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -1,244 +1,245 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_VALIDATIONINTERFACE_H #define BITCOIN_VALIDATIONINTERFACE_H #include // CTransaction(Ref) #include #include #include extern RecursiveMutex cs_main; class BlockValidationState; class CBlock; class CBlockIndex; struct CBlockLocator; class CValidationInterface; class CScheduler; enum class MemPoolRemovalReason; /** Register subscriber */ void RegisterValidationInterface(CValidationInterface *callbacks); /** * Unregister subscriber. DEPRECATED. This is not safe to use when the RPC * server or main message handler thread is running. */ void UnregisterValidationInterface(CValidationInterface *callbacks); /** Unregister all subscribers */ void UnregisterAllValidationInterfaces(); // Alternate registration functions that release a shared_ptr after the last // notification is sent. These are useful for race-free cleanup, since // unregistration is nonblocking and can return before the last notification is // processed. /** Register subscriber */ void RegisterSharedValidationInterface( std::shared_ptr callbacks); /** Unregister subscriber */ void UnregisterSharedValidationInterface( std::shared_ptr callbacks); /** * Pushes a function to callback onto the notification queue, guaranteeing any * callbacks generated prior to now are finished when the function is called. * * Be very careful blocking on func to be called if any locks are held - * validation interface clients may not be able to make progress as they often * wait for things like cs_main, so blocking until func is called with cs_main * will result in a deadlock (that DEBUG_LOCKORDER will miss). */ void CallFunctionInValidationInterfaceQueue(std::function func); /** * This is a synonym for the following, which asserts certain locks are not * held: * std::promise promise; * CallFunctionInValidationInterfaceQueue([&promise] { * promise.set_value(); * }); * promise.get_future().wait(); */ void SyncWithValidationInterfaceQueue() LOCKS_EXCLUDED(cs_main); /** * Implement this to subscribe to events generated in validation * * Each CValidationInterface() subscriber will receive event callbacks * in the order in which the events were generated by validation. * Furthermore, each ValidationInterface() subscriber may assume that * callbacks effectively run in a single thread with single-threaded * memory consistency. That is, for a given ValidationInterface() * instantiation, each callback will complete before the next one is * invoked. This means, for example when a block is connected that the * UpdatedBlockTip() callback may depend on an operation performed in * the BlockConnected() callback without worrying about explicit * synchronization. No ordering should be assumed across * ValidationInterface() subscribers. */ class CValidationInterface { protected: /** * Protected destructor so that instances can only be deleted by derived * classes. If that restriction is no longer desired, this should be made * public and virtual. */ ~CValidationInterface() = default; /** * Notifies listeners when the block chain tip advances. * * When multiple blocks are connected at once, UpdatedBlockTip will be * called on the final tip but may not be called on every intermediate tip. * If the latter behavior is desired, subscribe to BlockConnected() instead. * * Called on a background thread. */ virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) {} /** * Notifies listeners of a transaction having been added to mempool. * * Called on a background thread. */ virtual void TransactionAddedToMempool(const CTransactionRef &tx, uint64_t mempool_sequence) {} /** * Notifies listeners of a transaction leaving mempool. * * This notification fires for transactions that are removed from the * mempool for the following reasons: * * - EXPIRY (expired from mempool after -mempoolexpiry hours) * - SIZELIMIT (removed in size limiting if the mempool exceeds -maxmempool * megabytes) * - REORG (removed during a reorg) * - CONFLICT (removed because it conflicts with in-block transaction) * * This does not fire for transactions that are removed from the mempool * because they have been included in a block. Any client that is interested * in transactions removed from the mempool for inclusion in a block can * learn about those transactions from the BlockConnected notification. * * Transactions that are removed from the mempool because they conflict * with a transaction in the new block will have * TransactionRemovedFromMempool events fired *before* the BlockConnected * event is fired. If multiple blocks are connected in one step, then the * ordering could be: * * - TransactionRemovedFromMempool(tx1 from block A) * - TransactionRemovedFromMempool(tx2 from block A) * - TransactionRemovedFromMempool(tx1 from block B) * - TransactionRemovedFromMempool(tx2 from block B) * - BlockConnected(A) * - BlockConnected(B) * * Called on a background thread. */ virtual void TransactionRemovedFromMempool(const CTransactionRef &tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) {} /** * Notifies listeners of a block being connected. * Provides a vector of transactions evicted from the mempool as a result. * * Called on a background thread. */ virtual void BlockConnected(const std::shared_ptr &block, const CBlockIndex *pindex) {} /** * Notifies listeners of a block being disconnected * * Called on a background thread. */ virtual void BlockDisconnected(const std::shared_ptr &block, const CBlockIndex *pindex) {} /** * Notifies listeners of the new active block chain on-disk. * * Prior to this callback, any updates are not guaranteed to persist on disk * (ie clients need to handle shutdown/restart safety by being able to * understand when some updates were lost due to unclean shutdown). * * When this callback is invoked, the validation changes done by any prior * callback are guaranteed to exist on disk and survive a restart, including * an unclean shutdown. * * Provides a locator describing the best chain, which is likely useful for * storing current state on disk in client DBs. * * Called on a background thread. */ virtual void ChainStateFlushed(const CBlockLocator &locator) {} /** * Notifies listeners of a block validation result. * If the provided BlockValidationState IsValid, the provided block * is guaranteed to be the current best block at the time the * callback was generated (not necessarily now) */ virtual void BlockChecked(const CBlock &, const BlockValidationState &) {} /** * Notifies listeners that a block which builds directly on our current tip * has been received and connected to the headers tree, though not validated * yet. */ virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr &block){}; virtual void BlockFinalized(const CBlockIndex *pindex){}; friend class CMainSignals; + friend class ValidationInterfaceTest; }; class MainSignalsImpl; class CMainSignals { private: std::unique_ptr m_internals; friend void ::RegisterSharedValidationInterface( std::shared_ptr); friend void ::UnregisterValidationInterface(CValidationInterface *); friend void ::UnregisterAllValidationInterfaces(); friend void ::CallFunctionInValidationInterfaceQueue( std::function func); public: /** * Register a CScheduler to give callbacks which should run in the * background (may only be called once) */ void RegisterBackgroundSignalScheduler(CScheduler &scheduler); /** * Unregister a CScheduler to give callbacks which should run in the * background - these callbacks will now be dropped! */ void UnregisterBackgroundSignalScheduler(); /** Call any remaining callbacks on the calling thread */ void FlushBackgroundCallbacks(); size_t CallbacksPending(); void UpdatedBlockTip(const CBlockIndex *, const CBlockIndex *, bool fInitialDownload); void TransactionAddedToMempool(const CTransactionRef &, uint64_t mempool_sequence); void TransactionRemovedFromMempool(const CTransactionRef &, MemPoolRemovalReason, uint64_t mempool_sequence); void BlockConnected(const std::shared_ptr &, const CBlockIndex *pindex); void BlockDisconnected(const std::shared_ptr &, const CBlockIndex *pindex); void ChainStateFlushed(const CBlockLocator &); void BlockChecked(const CBlock &, const BlockValidationState &); void NewPoWValidBlock(const CBlockIndex *, const std::shared_ptr &); void BlockFinalized(const CBlockIndex *); }; CMainSignals &GetMainSignals(); #endif // BITCOIN_VALIDATIONINTERFACE_H