diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index c5983015f..0a4465d02 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -1,281 +1,282 @@ # Copyright (c) 2018 The Bitcoin developers project(bitcoin-test) # Process json files. file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/data") function(gen_json_header NAME) set(HEADERS "") foreach(f ${ARGN}) set(h "${CMAKE_CURRENT_BINARY_DIR}/${f}.h") # Get the proper name for the test variable. get_filename_component(TEST_NAME ${f} NAME_WE) add_custom_command(OUTPUT ${h} COMMAND "${Python_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/data/generate_header.py" "${TEST_NAME}" "${CMAKE_CURRENT_SOURCE_DIR}/${f}" > ${h} MAIN_DEPENDENCY ${f} DEPENDS "data/generate_header.py" VERBATIM ) list(APPEND HEADERS ${h}) endforeach(f) set(${NAME} "${HEADERS}" PARENT_SCOPE) endfunction() gen_json_header(JSON_HEADERS data/base58_encode_decode.json data/blockfilters.json data/key_io_valid.json data/key_io_invalid.json data/script_tests.json data/sighash.json data/tx_invalid.json data/tx_valid.json ) include(TestSuite) create_test_suite(bitcoin) add_dependencies(check check-bitcoin) # An utility library for bitcoin related test suites. add_library(testutil OBJECT util/blockfilter.cpp util/logging.cpp util/mining.cpp util/net.cpp util/setup_common.cpp util/str.cpp util/transaction_utils.cpp util/wallet.cpp ) target_link_libraries(testutil server) if(BUILD_BITCOIN_WALLET) set(BITCOIN_WALLET_TEST_FIXTURE ../wallet/test/init_test_fixture.cpp ../wallet/test/wallet_test_fixture.cpp ) set(BITCOIN_WALLET_TESTS ../wallet/test/db_tests.cpp ../wallet/test/coinselector_tests.cpp ../wallet/test/init_tests.cpp ../wallet/test/ismine_tests.cpp ../wallet/test/psbt_wallet_tests.cpp ../wallet/test/wallet_tests.cpp ../wallet/test/walletdb_tests.cpp ../wallet/test/wallet_crypto_tests.cpp ) endif() function(gen_asmap_headers HEADERS_VAR) foreach(INPUT_FILE ${ARGN}) set(OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/${INPUT_FILE}.h") add_custom_command( OUTPUT "${OUTPUT_FILE}" COMMENT "Generate ASMAP header from ${INPUT_FILE}" COMMAND "${Python_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/data/generate_asmap.py" "${CMAKE_CURRENT_SOURCE_DIR}/${INPUT_FILE}" "${OUTPUT_FILE}" MAIN_DEPENDENCY "${INPUT_FILE}" DEPENDS "data/generate_header.py" VERBATIM ) list(APPEND ${HEADERS_VAR} "${OUTPUT_FILE}") endforeach() set(${HEADERS_VAR} ${${HEADERS_VAR}} PARENT_SCOPE) endfunction() gen_asmap_headers(ASMAP_HEADERS data/asmap.raw ) add_boost_unit_tests_to_suite(bitcoin test_bitcoin fixture.cpp jsonutil.cpp scriptflags.cpp sigutil.cpp ${ASMAP_HEADERS} # Tests generated from JSON ${JSON_HEADERS} # Wallet test fixture ${BITCOIN_WALLET_TEST_FIXTURE} TESTS activation_tests.cpp addrman_tests.cpp allocator_tests.cpp amount_tests.cpp arith_uint256_tests.cpp base32_tests.cpp base58_tests.cpp base64_tests.cpp bip32_tests.cpp bitmanip_tests.cpp blockchain_tests.cpp blockcheck_tests.cpp blockencodings_tests.cpp blockfilter_tests.cpp blockfilter_index_tests.cpp blockindex_tests.cpp blockstatus_tests.cpp bloom_tests.cpp bswap_tests.cpp cashaddr_tests.cpp cashaddrenc_tests.cpp checkdatasig_tests.cpp checkpoints_tests.cpp checkqueue_tests.cpp coins_tests.cpp compilerbug_tests.cpp compress_tests.cpp config_tests.cpp core_io_tests.cpp crypto_tests.cpp cuckoocache_tests.cpp dbwrapper_tests.cpp denialofservice_tests.cpp descriptor_tests.cpp dstencode_tests.cpp excessiveblock_tests.cpp feerate_tests.cpp finalization_tests.cpp flatfile_tests.cpp fs_tests.cpp getarg_tests.cpp hash_tests.cpp interfaces_tests.cpp inv_tests.cpp key_io_tests.cpp key_tests.cpp lcg_tests.cpp limitedmap_tests.cpp logging_tests.cpp mempool_tests.cpp merkle_tests.cpp merkleblock_tests.cpp miner_tests.cpp monolith_opcodes_tests.cpp multisig_tests.cpp net_tests.cpp netbase_tests.cpp op_reversebytes_tests.cpp pmt_tests.cpp policyestimator_tests.cpp prevector_tests.cpp radix_tests.cpp raii_event_tests.cpp random_tests.cpp rcu_tests.cpp ref_tests.cpp reverselock_tests.cpp rpc_tests.cpp rpc_server_tests.cpp rwcollection_tests.cpp sanity_tests.cpp scheduler_tests.cpp schnorr_tests.cpp script_bitfield_tests.cpp script_commitment_tests.cpp script_p2sh_tests.cpp script_standard_tests.cpp script_tests.cpp scriptnum_tests.cpp serialize_tests.cpp settings_tests.cpp sigcache_tests.cpp sigencoding_tests.cpp sighash_tests.cpp sighashtype_tests.cpp sigcheckcount_tests.cpp skiplist_tests.cpp streams_tests.cpp sync_tests.cpp timedata_tests.cpp torcontrol_tests.cpp transaction_tests.cpp txindex_tests.cpp txvalidation_tests.cpp txvalidationcache_tests.cpp uint256_tests.cpp undo_tests.cpp util_tests.cpp util_threadnames_tests.cpp validation_block_tests.cpp validation_chainstatemanager_tests.cpp + validation_flush_tests.cpp validation_tests.cpp validationinterface_tests.cpp versionbits_tests.cpp work_comparator_tests.cpp # RPC Tests ../rpc/test/server_tests.cpp # Wallet tests ${BITCOIN_WALLET_TESTS} ) function(add_boost_test_runners_with_upgrade_activated SUITE EXECUTABLE) set(SUITE_UPGRADE_ACTIVATED "${SUITE}-upgrade-activated") get_target_from_suite(${SUITE_UPGRADE_ACTIVATED} TARGET_UPGRADE_ACTIVATED) if(NOT TARGET ${TARGET_UPGRADE_ACTIVATED}) create_test_suite_with_parent_targets( ${SUITE_UPGRADE_ACTIVATED} check-upgrade-activated check-upgrade-activated-extended ) add_dependencies(${TARGET_UPGRADE_ACTIVATED} ${EXECUTABLE}) endif() get_target_from_suite(${SUITE} SUITE_TARGET) get_target_property(BOOST_TESTS ${SUITE_TARGET} UNIT_TESTS) get_target_from_suite(${SUITE_UPGRADE_ACTIVATED} SUITE_UPGRADE_ACTIVATED_TARGET) set(HRF_LOGGER "HRF,test_suite") foreach(_test_name ${BOOST_TESTS}) if(ENABLE_JUNIT_REPORT) set(JUNIT_LOGGER ":JUNIT,message,${SUITE_UPGRADE_ACTIVATED}-${_test_name}.xml") endif() add_test_runner( ${SUITE_UPGRADE_ACTIVATED} "${_test_name}" ${EXECUTABLE} JUNIT "--run_test=${_test_name}" "--logger=${HRF_LOGGER}${JUNIT_LOGGER}" "--catch_system_errors=no" # Dec. 1st, 2019 at 00:00:00 -- "-testsuitename=Bitcoin ABC unit tests with next upgrade activated" -axionactivationtime=1575158400 ) endforeach() endfunction() add_boost_test_runners_with_upgrade_activated(bitcoin test_bitcoin) target_link_libraries(test_bitcoin rpcclient testutil) if(TARGET bitcoinconsensus-shared) target_link_libraries(test_bitcoin bitcoinconsensus-shared) else() target_link_libraries(test_bitcoin bitcoinconsensus) endif() add_subdirectory(fuzz) diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp new file mode 100644 index 000000000..436e9d994 --- /dev/null +++ b/src/test/validation_flush_tests.cpp @@ -0,0 +1,178 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include +#include +#include +#include + +#include + +BOOST_FIXTURE_TEST_SUITE(validation_flush_tests, BasicTestingSetup) + +//! Test utilities for detecting when we need to flush the coins cache based +//! on estimated memory usage. +//! +//! @sa CChainState::GetCoinsCacheSizeState() +//! +BOOST_AUTO_TEST_CASE(getcoinscachesizestate) { + BlockManager blockman{}; + CChainState chainstate{blockman}; + chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, + /*should_wipe*/ false); + WITH_LOCK(::cs_main, chainstate.InitCoinsCache()); + CTxMemPool tx_pool{}; + + constexpr bool is_64_bit = sizeof(void *) == 8; + + LOCK(::cs_main); + auto &view = chainstate.CoinsTip(); + + //! Create and add a Coin with DynamicMemoryUsage of 80 bytes to the given + //! view. + auto add_coin = [](CCoinsViewCache &coins_view) -> COutPoint { + ; + auto txid = TxId(InsecureRand256()); + COutPoint outp{txid, 0}; + CScript scriptPubKey; + scriptPubKey.assign(static_cast(62), 1); + Coin newcoin(CTxOut(500 * SATOSHI, std::move(scriptPubKey)), 1, false); + coins_view.AddCoin(std::move(outp), std::move(newcoin), false); + + return outp; + }; + + // The number of bytes consumed by coin's heap data, i.e. CScript + // (prevector<28, uint8_t>) when assigned 56 bytes of data per above. + // + // See also: Coin::DynamicMemoryUsage(). + constexpr int COIN_SIZE = is_64_bit ? 80 : 72; + + auto print_view_mem_usage = [](CCoinsViewCache &_view) { + BOOST_TEST_MESSAGE( + "CCoinsViewCache memory usage: " << _view.DynamicMemoryUsage()); + }; + + constexpr size_t MAX_COINS_CACHE_BYTES = 1024; + + // Without any coins in the cache, we shouldn't need to flush. + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, + /*max_mempool_size_bytes*/ 0), + CoinsCacheSizeState::OK); + + // If the initial memory allocations of cacheCoins don't match these common + // cases, we can't really continue to make assertions about memory usage. + // End the test early. + if (view.DynamicMemoryUsage() != 32 && view.DynamicMemoryUsage() != 16) { + // Add a bunch of coins to see that we at least flip over to CRITICAL. + + for (int i{0}; i < 1000; ++i) { + COutPoint res = add_coin(view); + BOOST_CHECK_EQUAL(view.AccessCoin(res).DynamicMemoryUsage(), + COIN_SIZE); + } + + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, + /*max_mempool_size_bytes*/ 0), + CoinsCacheSizeState::CRITICAL); + + BOOST_TEST_MESSAGE( + "Exiting cache flush tests early due to unsupported arch"); + return; + } + + print_view_mem_usage(view); + BOOST_CHECK_EQUAL(view.DynamicMemoryUsage(), is_64_bit ? 32 : 16); + + // We should be able to add COINS_UNTIL_CRITICAL coins to the cache before + // going CRITICAL. This is contingent not only on the dynamic memory usage + // of the Coins that we're adding (COIN_SIZE bytes per), but also on how + // much memory the cacheCoins (unordered_map) preallocates. + constexpr int COINS_UNTIL_CRITICAL{3}; + + for (int i{0}; i < COINS_UNTIL_CRITICAL; ++i) { + COutPoint res = add_coin(view); + print_view_mem_usage(view); + BOOST_CHECK_EQUAL(view.AccessCoin(res).DynamicMemoryUsage(), COIN_SIZE); + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, + /*max_mempool_size_bytes*/ 0), + CoinsCacheSizeState::OK); + } + + // Adding some additional coins will push us over the edge to CRITICAL. + for (int i{0}; i < 4; ++i) { + add_coin(view); + print_view_mem_usage(view); + if (chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, + /*max_mempool_size_bytes*/ 0) == + CoinsCacheSizeState::CRITICAL) { + break; + } + } + + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, + /*max_mempool_size_bytes*/ 0), + CoinsCacheSizeState::CRITICAL); + + // Passing non-zero max mempool usage should allow us more headroom. + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, + /*max_mempool_size_bytes*/ 1 << 10), + CoinsCacheSizeState::OK); + + for (int i{0}; i < 3; ++i) { + add_coin(view); + print_view_mem_usage(view); + BOOST_CHECK_EQUAL(chainstate.GetCoinsCacheSizeState( + tx_pool, MAX_COINS_CACHE_BYTES, + /*max_mempool_size_bytes*/ 1 << 10), + CoinsCacheSizeState::OK); + } + + // Adding another coin with the additional mempool room will put us >90% + // but not yet critical. + add_coin(view); + print_view_mem_usage(view); + + // Only perform these checks on 64 bit hosts; I haven't done the math + // for 32. + if (is_64_bit) { + float usage_percentage = (float)view.DynamicMemoryUsage() / + (MAX_COINS_CACHE_BYTES + (1 << 10)); + BOOST_TEST_MESSAGE("CoinsTip usage percentage: " << usage_percentage); + BOOST_CHECK(usage_percentage >= 0.9); + BOOST_CHECK(usage_percentage < 1); + BOOST_CHECK_EQUAL(chainstate.GetCoinsCacheSizeState( + tx_pool, MAX_COINS_CACHE_BYTES, 1 << 10), + CoinsCacheSizeState::LARGE); + } + + // Using the default max_* values permits way more coins to be added. + for (int i{0}; i < 1000; ++i) { + add_coin(view); + BOOST_CHECK_EQUAL(chainstate.GetCoinsCacheSizeState(tx_pool), + CoinsCacheSizeState::OK); + } + + // Flushing the view doesn't take us back to OK because cacheCoins has + // preallocated memory that doesn't get reclaimed even after flush. + + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, 0), + CoinsCacheSizeState::CRITICAL); + + view.SetBestBlock(BlockHash(InsecureRand256())); + BOOST_CHECK(view.Flush()); + print_view_mem_usage(view); + + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, 0), + CoinsCacheSizeState::CRITICAL); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txdb.h b/src/txdb.h index f159bcc15..cf4c1f432 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -1,121 +1,119 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_TXDB_H #define BITCOIN_TXDB_H #include #include #include #include #include #include #include #include #include struct BlockHash; class CBlockIndex; class CCoinsViewDBCursor; namespace Consensus { struct Params; } -//! No need to periodic flush if at least this much space still available. -static constexpr int MAX_BLOCK_COINSDB_USAGE = 10; //! min. -dbcache (MiB) static constexpr int64_t MIN_DB_CACHE_MB = 4; //! max. -dbcache (MiB) static constexpr int64_t MAX_DB_CACHE_MB = sizeof(void *) > 4 ? 16384 : 1024; //! -dbcache default (MiB) static constexpr int64_t DEFAULT_DB_CACHE_MB = 1024; //! -dbbatchsize default (bytes) static constexpr int64_t DEFAULT_DB_BATCH_SIZE = 16 << 20; //! Max memory allocated to block tree DB specific cache, if no -txindex (MiB) static constexpr int64_t MAX_BLOCK_DB_CACHE_MB = 2; //! Max memory allocated to block tree DB specific cache, if -txindex (MiB) // Unlike for the UTXO database, for the txindex scenario the leveldb cache make // a meaningful difference: // https://github.com/bitcoin/bitcoin/pull/8273#issuecomment-229601991 static constexpr int64_t MAX_TX_INDEX_CACHE_MB = 1024; //! Max memory allocated to all block filter index caches combined in MiB. static constexpr int64_t MAX_FILTER_INDEX_CACHE_MB = 1024; //! Max memory allocated to coin DB specific cache (MiB) static constexpr int64_t MAX_COINS_DB_CACHE_MB = 8; /** CCoinsView backed by the coin database (chainstate/) */ class CCoinsViewDB final : public CCoinsView { protected: CDBWrapper db; public: /** * @param[in] ldb_path Location in the filesystem where leveldb data will * be stored. */ explicit CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe); bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; bool HaveCoin(const COutPoint &outpoint) const override; BlockHash GetBestBlock() const override; std::vector GetHeadBlocks() const override; bool BatchWrite(CCoinsMap &mapCoins, const BlockHash &hashBlock) override; CCoinsViewCursor *Cursor() const override; //! Attempt to update from an older database format. //! Returns whether an error occurred. bool Upgrade(); size_t EstimateSize() const override; }; /** Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB */ class CCoinsViewDBCursor : public CCoinsViewCursor { public: ~CCoinsViewDBCursor() {} bool GetKey(COutPoint &key) const override; bool GetValue(Coin &coin) const override; unsigned int GetValueSize() const override; bool Valid() const override; void Next() override; private: CCoinsViewDBCursor(CDBIterator *pcursorIn, const BlockHash &hashBlockIn) : CCoinsViewCursor(hashBlockIn), pcursor(pcursorIn) {} std::unique_ptr pcursor; std::pair keyTmp; friend class CCoinsViewDB; }; /** Access to the block database (blocks/index/) */ class CBlockTreeDB : public CDBWrapper { public: explicit CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); bool WriteBatchSync( const std::vector> &fileInfo, int nLastFile, const std::vector &blockinfo); bool ReadBlockFileInfo(int nFile, CBlockFileInfo &info); bool ReadLastBlockFile(int &nFile); bool WriteReindexing(bool fReindexing); bool IsReindexing() const; bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); bool LoadBlockIndexGuts( const Consensus::Params ¶ms, std::function insertBlockIndex); //! Attempt to update from an older database format. //! Returns whether an error occurred. bool Upgrade(const Consensus::Params ¶ms); }; #endif // BITCOIN_TXDB_H diff --git a/src/validation.cpp b/src/validation.cpp index 8fa751a4b..2d443b0cc 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1,6012 +1,6037 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2018 The Bitcoin Core developers // Copyright (c) 2017-2020 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // For ResetASERTAnchorBlockCache #include #include #include #include #include #include