diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 590276e77..845c542df 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,1006 +1,1008 @@ # Copyright (c) 2017 The Bitcoin developers project(bitcoind) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) # Default visibility is hidden on all targets. set(CMAKE_C_VISIBILITY_PRESET hidden) set(CMAKE_CXX_VISIBILITY_PRESET hidden) option(BUILD_BITCOIN_WALLET "Activate the wallet functionality" ON) option(BUILD_BITCOIN_ZMQ "Activate the ZeroMQ functionalities" ON) option(BUILD_BITCOIN_CLI "Build bitcoin-cli" ON) option(BUILD_BITCOIN_TX "Build bitcoin-tx" ON) option(BUILD_BITCOIN_QT "Build bitcoin-qt" ON) option(BUILD_BITCOIN_SEEDER "Build bitcoin-seeder" ON) option(BUILD_LIBBITCOINCONSENSUS "Build the bitcoinconsenus shared library" ON) option(BUILD_BITCOIN_CHAINSTATE "Build bitcoin-chainstate" OFF) option(ENABLE_BIP70 "Enable BIP70 (payment protocol) support in GUI" ON) option(ENABLE_HARDENING "Harden the executables" ON) option(ENABLE_REDUCE_EXPORTS "Reduce the amount of exported symbols" OFF) option(ENABLE_STATIC_LIBSTDCXX "Statically link libstdc++" OFF) option(ENABLE_GLIBC_BACK_COMPAT "Enable Glibc compatibility features" OFF) option(ENABLE_QRCODE "Enable QR code display" ON) option(ENABLE_UPNP "Enable UPnP support" ON) option(START_WITH_UPNP "Make UPnP the default to map ports" OFF) option(ENABLE_NATPMP "Enable NAT-PMP support" ON) option(START_WITH_NATPMP "Make NAT-PMP the default to map ports" OFF) option(ENABLE_CLANG_TIDY "Enable clang-tidy checks for Bitcoin ABC" OFF) option(ENABLE_PROFILING "Select the profiling tool to use" OFF) option(ENABLE_TRACING "Enable eBPF user static defined tracepoints" OFF) # Linker option if(CMAKE_CROSSCOMPILING) set(DEFAULT_LINKER "") elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") set(DEFAULT_LINKER lld) elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(DEFAULT_LINKER gold) else() set(DEFAULT_LINKER "") endif() set(USE_LINKER "${DEFAULT_LINKER}" CACHE STRING "Linker to be used (default: ${DEFAULT_LINKER}). Set to empty string to use the system's default.") set(OS_WITH_JEMALLOC_AS_SYSTEM_DEFAULT "Android" "FreeBSD" "NetBSD" ) if(NOT CMAKE_SYSTEM_NAME IN_LIST OS_WITH_JEMALLOC_AS_SYSTEM_DEFAULT) set(USE_JEMALLOC_DEFAULT ON) endif() # FIXME: Building against jemalloc causes the software to segfault on OSX. # See https://github.com/Bitcoin-ABC/bitcoin-abc/issues/401 if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin" AND NOT CMAKE_CROSSCOMPILING) set(USE_JEMALLOC_DEFAULT OFF) endif() option(USE_JEMALLOC "Use jemalloc as an allocation library" ${USE_JEMALLOC_DEFAULT}) if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") set(DEFAULT_ENABLE_DBUS_NOTIFICATIONS ON) endif() option(ENABLE_DBUS_NOTIFICATIONS "Enable DBus desktop notifications. Linux only." ${DEFAULT_ENABLE_DBUS_NOTIFICATIONS}) # If ccache is available, then use it. find_program(CCACHE ccache) if(CCACHE) message(STATUS "Using ccache: ${CCACHE}") set(CMAKE_C_COMPILER_LAUNCHER ${CCACHE}) set(CMAKE_CXX_COMPILER_LAUNCHER ${CCACHE}) endif(CCACHE) # Disable what we do not need for the native build. include(NativeExecutable) native_add_cmake_flags( "-DBUILD_BITCOIN_WALLET=OFF" "-DBUILD_BITCOIN_CHRONIK=OFF" "-DBUILD_BITCOIN_QT=OFF" "-DBUILD_BITCOIN_ZMQ=OFF" "-DENABLE_QRCODE=OFF" "-DENABLE_NATPMP=OFF" "-DENABLE_UPNP=OFF" "-DUSE_JEMALLOC=OFF" "-DENABLE_CLANG_TIDY=OFF" "-DENABLE_BIP70=OFF" "-DUSE_LINKER=" ) if(ENABLE_CLANG_TIDY) include(ClangTidy) endif() if(ENABLE_SANITIZERS) include(Sanitizers) enable_sanitizers(${ENABLE_SANITIZERS}) endif() include(AddCompilerFlags) if(USE_LINKER) set(LINKER_FLAG "-fuse-ld=${USE_LINKER}") custom_check_linker_flag(IS_LINKER_SUPPORTED ${LINKER_FLAG}) if(NOT IS_LINKER_SUPPORTED) message(FATAL_ERROR "The ${USE_LINKER} linker is not supported, make sure ${USE_LINKER} is properly installed or use -DUSE_LINKER= to use the system's linker") endif() add_linker_flags(${LINKER_FLAG}) # Remember the selected linker, it will be used for the subsequent # custom_check_linker_flag calls set(GLOBAL_LINKER_FLAGS ${LINKER_FLAG} CACHE INTERNAL "Additional linker flags for flag support checking") endif() # Prefer -g3, defaults to -g if unavailable foreach(LANGUAGE C CXX) set(COMPILER_DEBUG_LEVEL -g) check_compiler_flags(G3_IS_SUPPORTED ${LANGUAGE} -g3) if(${G3_IS_SUPPORTED}) set(COMPILER_DEBUG_LEVEL -g3) endif() add_compile_options_to_configuration_for_language(Debug ${LANGUAGE} ${COMPILER_DEBUG_LEVEL}) endforeach() # Define some debugging symbols when the Debug build type is selected. add_compile_definitions_to_configuration(Debug DEBUG DEBUG_LOCKORDER ABORT_ON_FAILED_ASSUME) # Add -ftrapv when building in Debug add_compile_options_to_configuration(Debug -ftrapv) # All versions of gcc that we commonly use for building are subject to bug # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90348. To work around that, set # -fstack-reuse=none for all gcc builds. (Only gcc understands this flag) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") add_compiler_flags(-fstack-reuse=none) endif() if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") # Ensure that WINDRES_PREPROC is enabled when using windres. list(APPEND CMAKE_RC_FLAGS "-DWINDRES_PREPROC") # Build all static so there is no dll file to distribute. add_linker_flags(-static) add_compile_definitions( # Windows 7 _WIN32_WINNT=0x0601 # Internet Explorer 5.01 (!) _WIN32_IE=0x0501 # Define WIN32_LEAN_AND_MEAN to exclude APIs such as Cryptography, DDE, # RPC, Shell, and Windows Sockets. WIN32_LEAN_AND_MEAN ) # We require Windows 7 (NT 6.1) or later add_linker_flags(-Wl,--major-subsystem-version,6 -Wl,--minor-subsystem-version,1) endif() if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") add_compile_definitions(MAC_OSX OBJC_OLD_DISPATCH_PROTOTYPES=0) add_linker_flags(-Wl,-dead_strip_dylibs) endif() if(ENABLE_REDUCE_EXPORTS) # Default visibility is set by CMAKE__VISIBILITY_PRESET, but this # doesn't tell if the visibility set is effective. # Check if the flag -fvisibility=hidden is supported, as using the hidden # visibility is a requirement to reduce exports. check_compiler_flags(HAS_CXX_FVISIBILITY CXX -fvisibility=hidden) if(NOT HAS_CXX_FVISIBILITY) message(FATAL_ERROR "Cannot set default symbol visibility. Use -DENABLE_REDUCE_EXPORTS=OFF.") endif() # Also hide symbols from static libraries add_linker_flags(-Wl,--exclude-libs,libstdc++) endif() # Enable statically linking libstdc++ if(ENABLE_STATIC_LIBSTDCXX) add_linker_flags(-static-libstdc++) endif() set(CMAKE_POSITION_INDEPENDENT_CODE ON) if(ENABLE_HARDENING) # Enable stack protection add_cxx_compiler_flags(-fstack-protector-all -Wstack-protector) # Enable control-flow enforcement add_cxx_compiler_flags(-fcf-protection=full) if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") # stack-clash-protection does not work properly when building for Windows. # See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90458 add_cxx_compiler_flags(-fstack-clash-protection) endif() add_linker_flags(-Wl,-z,noexecstack) # Enable some buffer overflow checking, except in -O0 builds which # do not support them add_compiler_flags(-U_FORTIFY_SOURCE) add_compile_options($<$>:-D_FORTIFY_SOURCE=2>) # Enable ASLR (these flags are primarily targeting MinGw) add_linker_flags(-Wl,--enable-reloc-section -Wl,--dynamicbase -Wl,--nxcompat -Wl,--high-entropy-va) # Make the relocated sections read-only add_linker_flags(-Wl,-z,relro -Wl,-z,now) # Avoids mixing code pages with data to improve cache performance as well # as security add_linker_flags(-Wl,-z,separate-code) # CMake provides the POSITION_INDEPENDENT_CODE property to set PIC/PIE. cmake_policy(SET CMP0083 NEW) include(CheckPIESupported) check_pie_supported() if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") # MinGw provides its own libssp for stack smashing protection link_libraries(ssp) endif() endif() if(ENABLE_PROFILING MATCHES "gprof") message(STATUS "Enable profiling with gprof") # -pg is incompatible with -pie. Since hardening and profiling together # doesn't make sense, we simply make them mutually exclusive here. # Additionally, hardened toolchains may force -pie by default, in which # case it needs to be turned off with -no-pie. if(ENABLE_HARDENING) message(FATAL_ERROR "Profiling with gprof requires disabling hardening with -DENABLE_HARDENING=OFF.") endif() add_linker_flags(-no-pie) add_compiler_flags(-pg) add_linker_flags(-pg) endif() # Enable warning add_c_compiler_flags(-Wnested-externs -Wstrict-prototypes) add_compiler_flags( -Wall -Wextra -Wformat -Wgnu -Wvla -Wcast-align -Wunused-parameter -Wmissing-braces -Wthread-safety -Wrange-loop-analysis -Wredundant-decls -Wunreachable-code-loop-increment -Wsign-compare -Wconditional-uninitialized -Wduplicated-branches -Wduplicated-cond -Wlogical-op -Wdocumentation ) add_compiler_flag_group(-Wformat -Wformat-security) add_cxx_compiler_flags( -Wredundant-move -Woverloaded-virtual ) if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") # GCC has no flag variant which is granular enough to avoid raising the clang # -Wshadow-uncaptured-local equivalent. This is causing a lot of warnings # on serialize.h which cannot be disabled locally, so drop the flag. add_compiler_flags( -Wshadow -Wshadow-field ) endif() option(EXTRA_WARNINGS "Enable extra warnings" OFF) if(EXTRA_WARNINGS) add_cxx_compiler_flags(-Wsuggest-override) else() add_compiler_flags( -Wno-unused-parameter -Wno-implicit-fallthrough -Wno-psabi ) endif() # libtool style configure add_subdirectory(config) # Enable LFS (Large File Support) on targets that don't have it natively. # This should be defined before the libraries are included as leveldb need the # definition to be set. if(NOT HAVE_LARGE_FILE_SUPPORT) add_compile_definitions(_FILE_OFFSET_BITS=64) add_linker_flags(-Wl,--large-address-aware) endif() if(ENABLE_GLIBC_BACK_COMPAT) # Wrap some glibc functions with ours add_linker_flags(-Wl,--wrap=exp) add_linker_flags(-Wl,--wrap=log) add_linker_flags(-Wl,--wrap=log2) add_linker_flags(-Wl,--wrap=pow) if(NOT HAVE_LARGE_FILE_SUPPORT) add_linker_flags(-Wl,--wrap=fcntl -Wl,--wrap=fcntl64) endif() endif() if(USE_JEMALLOC) # Most of the sanitizers require their instrumented allocation functions to # be fully functional. This is obviously the case for all the memory related # sanitizers (asan, lsan, msan) but not only. if(ENABLE_SANITIZERS) message(WARNING "Jemalloc is incompatible with the sanitizers and has been disabled.") else() find_package(Jemalloc 3.6.0 REQUIRED) link_libraries(Jemalloc::jemalloc) endif() endif() # These flags are needed for using std::filesystem with GCC < 9.1 & Clang < 9.0 # Since these are optional libraries they need to be placed accordingly on the # command line. add_linker_flags(-lstdc++fs -lc++fs) custom_check_linker_flag(LINKER_HAS_STDCXXFS "-lstdc++fs") if(LINKER_HAS_STDCXXFS) link_libraries(stdc++fs) endif() custom_check_linker_flag(LINKER_HAS_CXXFS "-lc++fs") if(LINKER_HAS_CXXFS) link_libraries(c++fs) endif() # Make sure that all the global compiler and linker flags are set BEFORE # including the libraries so they apply as needed. # libraries add_subdirectory(crypto) add_subdirectory(leveldb) add_subdirectory(secp256k1) add_subdirectory(univalue) # Find the git root, and returns the full path to the .git/logs/HEAD file if # it exists. function(find_git_head_logs_file RESULT) find_package(Git) if(GIT_FOUND) execute_process( COMMAND "${GIT_EXECUTABLE}" "rev-parse" "--show-toplevel" WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_ROOT RESULT_VARIABLE GIT_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET ) if(GIT_RESULT EQUAL 0) set(GIT_LOGS_DIR "${GIT_ROOT}/.git/logs") set(GIT_HEAD_LOGS_FILE "${GIT_LOGS_DIR}/HEAD") # If the .git/logs/HEAD does not exist, create it if(NOT EXISTS "${GIT_HEAD_LOGS_FILE}") file(MAKE_DIRECTORY "${GIT_LOGS_DIR}") file(TOUCH "${GIT_HEAD_LOGS_FILE}") endif() set(${RESULT} "${GIT_HEAD_LOGS_FILE}" PARENT_SCOPE) endif() endif() endfunction() find_git_head_logs_file(GIT_HEAD_LOGS_FILE) set(OBJ_DIR "${CMAKE_CURRENT_BINARY_DIR}/obj") file(MAKE_DIRECTORY "${OBJ_DIR}") set(BUILD_HEADER "${OBJ_DIR}/build.h") set(BUILD_HEADER_TMP "${BUILD_HEADER}.tmp") add_custom_command( DEPENDS "${GIT_HEAD_LOGS_FILE}" "${CMAKE_SOURCE_DIR}/share/genbuild.sh" OUTPUT "${BUILD_HEADER}" COMMAND "${CMAKE_SOURCE_DIR}/share/genbuild.sh" "${BUILD_HEADER_TMP}" "${CMAKE_SOURCE_DIR}" COMMAND ${CMAKE_COMMAND} -E copy_if_different "${BUILD_HEADER_TMP}" "${BUILD_HEADER}" COMMAND ${CMAKE_COMMAND} -E remove "${BUILD_HEADER_TMP}" ) # Because the Bitcoin ABc source code is disorganised, we # end up with a bunch of libraries without any apparent # cohesive structure. This is inherited from Bitcoin Core # and reflecting this. # TODO: Improve the structure once cmake is rocking. # Various completely unrelated features shared by all executables. add_library(util chainparamsbase.cpp clientversion.cpp compat/glibcxx_sanity.cpp compat/strnlen.cpp currencyunit.cpp fs.cpp interfaces/handler.cpp logging.cpp random.cpp randomenv.cpp rcu.cpp rpc/request.cpp support/cleanse.cpp support/lockedpool.cpp sync.cpp threadinterrupt.cpp uint256.cpp util/asmap.cpp util/bip32.cpp util/bytevectorhash.cpp util/check.cpp util/hasher.cpp util/error.cpp util/getuniquepath.cpp util/message.cpp util/moneystr.cpp util/readwritefile.cpp util/settings.cpp util/string.cpp util/sock.cpp util/spanparsing.cpp util/strencodings.cpp util/string.cpp util/system.cpp util/thread.cpp util/threadnames.cpp util/time.cpp util/tokenpipe.cpp util/url.cpp # obj/build.h "${BUILD_HEADER}" ) target_compile_definitions(util PUBLIC HAVE_CONFIG_H HAVE_BUILD_INFO) target_include_directories(util PUBLIC . # To access the config/ and obj/ directories ${CMAKE_CURRENT_BINARY_DIR} ) if(ENABLE_GLIBC_BACK_COMPAT) target_sources(util PRIVATE compat/glibc_compat.cpp) endif() # Work around jemalloc printing harmless warnings to stderr with qemu. # We match the emulator against qemu but not ^qemu so we are compatible with the # use of absolute paths. if(CMAKE_CROSSCOMPILING AND USE_JEMALLOC AND CMAKE_CROSSCOMPILING_EMULATOR MATCHES "qemu-.+") message(WARNING "Overriding Jemalloc malloc_message() function for use with QEMU.") target_sources(util PRIVATE jemalloc_message.cpp) endif() set(Boost_USE_STATIC_LIBS ON) macro(link_windows_dependencies TARGET) find_package(SHLWAPI REQUIRED) target_link_libraries(${TARGET} SHLWAPI::shlwapi) find_library(WS2_32_LIBRARY NAMES ws2_32) target_link_libraries(${TARGET} ${WS2_32_LIBRARY}) endmacro() # Target specific configs if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") set(Boost_USE_STATIC_RUNTIME ON) set(Boost_THREADAPI win32) link_windows_dependencies(util) endif() target_link_libraries(util univalue crypto) macro(link_event TARGET) non_native_target_link_libraries(${TARGET} Event 2.0.22 ${ARGN}) endmacro() link_event(util event) macro(link_boost_headers_only TARGET) non_native_target_link_headers_only(${TARGET} Boost 1.64 ${ARGN}) endmacro() link_boost_headers_only(util headers) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) target_link_libraries(util Threads::Threads) function(add_network_sources NETWORK_SOURCES) set(NETWORK_DIR abc) list(TRANSFORM ARGN PREPEND "networks/${NETWORK_DIR}/" OUTPUT_VARIABLE NETWORK_SOURCES ) set(NETWORK_SOURCES ${NETWORK_SOURCES} PARENT_SCOPE) endfunction() add_network_sources(NETWORK_SOURCES checkpoints.cpp chainparamsconstants.cpp ) # More completely unrelated features shared by all executables. # Because nothing says this is different from util than "common" add_library(common base58.cpp bloom.cpp cashaddr.cpp cashaddrenc.cpp chainparams.cpp config.cpp consensus/merkle.cpp coins.cpp compressor.cpp eventloop.cpp feerate.cpp core_read.cpp core_write.cpp key.cpp key_io.cpp merkleblock.cpp net_permissions.cpp netaddress.cpp netbase.cpp outputtype.cpp policy/policy.cpp primitives/block.cpp protocol.cpp psbt.cpp rpc/rawtransaction_util.cpp rpc/util.cpp scheduler.cpp warnings.cpp ${NETWORK_SOURCES} ) target_link_libraries(common bitcoinconsensus util secp256k1 script) # script library add_library(script script/bitfield.cpp script/descriptor.cpp script/interpreter.cpp script/script.cpp script/script_error.cpp script/sigencoding.cpp script/sign.cpp script/signingprovider.cpp script/standard.cpp ) target_link_libraries(script common) # libbitcoinconsensus add_library(bitcoinconsensus arith_uint256.cpp hash.cpp primitives/transaction.cpp pubkey.cpp uint256.cpp util/strencodings.cpp consensus/amount.cpp consensus/tx_check.cpp ) target_link_libraries(bitcoinconsensus script) include(InstallationHelper) if(BUILD_LIBBITCOINCONSENSUS) target_compile_definitions(bitcoinconsensus PUBLIC BUILD_BITCOIN_INTERNAL HAVE_CONSENSUS_LIB ) install_shared_library(bitcoinconsensus script/bitcoinconsensus.cpp PUBLIC_HEADER script/bitcoinconsensus.h ) endif() # Bitcoin server facilities add_library(server addrdb.cpp addrman.cpp avalanche/avalanche.cpp avalanche/compactproofs.cpp avalanche/delegation.cpp avalanche/delegationbuilder.cpp avalanche/peermanager.cpp avalanche/processor.cpp avalanche/proof.cpp avalanche/proofid.cpp avalanche/proofbuilder.cpp avalanche/proofpool.cpp avalanche/voterecord.cpp banman.cpp blockencodings.cpp blockfileinfo.cpp blockfilter.cpp blockindex.cpp chain.cpp checkpoints.cpp config.cpp consensus/activation.cpp consensus/tx_verify.cpp dbwrapper.cpp deploymentstatus.cpp dnsseeds.cpp flatfile.cpp headerssync.cpp httprpc.cpp httpserver.cpp i2p.cpp index/base.cpp index/blockfilterindex.cpp index/coinstatsindex.cpp index/txindex.cpp init.cpp init/common.cpp invrequest.cpp + kernel/disconnected_transactions.cpp kernel/mempool_persist.cpp mapport.cpp mempool_args.cpp minerfund.cpp net.cpp net_processing.cpp node/blockstorage.cpp node/caches.cpp node/chainstate.cpp node/coin.cpp node/coinstats.cpp node/context.cpp node/interfaces.cpp node/mempool_persist_args.cpp node/miner.cpp node/psbt.cpp node/transaction.cpp node/ui_interface.cpp node/utxo_snapshot.cpp node/validation_cache_args.cpp noui.cpp policy/block/minerfund.cpp policy/block/preconsensus.cpp policy/block/stakingrewards.cpp policy/fees.cpp policy/packages.cpp policy/settings.cpp pow/aserti32d.cpp pow/daa.cpp pow/eda.cpp pow/grasberg.cpp pow/pow.cpp rest.cpp rpc/abc.cpp rpc/avalanche.cpp rpc/blockchain.cpp rpc/command.cpp rpc/mempool.cpp rpc/mining.cpp rpc/misc.cpp rpc/net.cpp rpc/rawtransaction.cpp rpc/server.cpp rpc/server_util.cpp rpc/txoutproof.cpp script/scriptcache.cpp script/sigcache.cpp shutdown.cpp timedata.cpp torcontrol.cpp txdb.cpp txmempool.cpp txorphanage.cpp validation.cpp validationinterface.cpp versionbits.cpp ) target_include_directories(server PRIVATE leveldb/helpers/memenv) target_link_libraries(server bitcoinconsensus leveldb memenv ) link_event(server event) if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") link_event(server pthreads) endif() if(ENABLE_UPNP) find_package(MiniUPnPc 1.9 REQUIRED) target_link_libraries(server MiniUPnPc::miniupnpc) if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") # TODO: check if we are really using a static library. Assume this is # the one from the depends for now since the native windows build is not # supported. target_compile_definitions(server PUBLIC -DSTATICLIB PUBLIC -DMINIUPNP_STATICLIB ) endif() endif() if(ENABLE_NATPMP) find_package(NATPMP REQUIRED) target_link_libraries(server NATPMP::natpmp) if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") target_compile_definitions(server PUBLIC -DSTATICLIB PUBLIC -DNATPMP_STATICLIB ) endif() endif() # Test suites. add_subdirectory(test) add_subdirectory(avalanche/test) add_subdirectory(pow/test) # Benchmark suite. add_subdirectory(bench) include(BinaryTest) include(WindowsVersionInfo) # Wallet if(BUILD_BITCOIN_WALLET) add_subdirectory(wallet) target_link_libraries(server wallet) # bitcoin-wallet add_executable(bitcoin-wallet bitcoin-wallet.cpp) generate_windows_version_info(bitcoin-wallet DESCRIPTION "CLI tool for ${PACKAGE_NAME} wallets" ) target_link_libraries(bitcoin-wallet wallet-tool common util) add_to_symbols_check(bitcoin-wallet) add_to_security_check(bitcoin-wallet) install_target(bitcoin-wallet) install_manpages(bitcoin-wallet) else() target_sources(server PRIVATE dummywallet.cpp) endif() # ZeroMQ if(BUILD_BITCOIN_ZMQ) add_subdirectory(zmq) target_link_libraries(server zmq) # FIXME: This is needed because of an unwanted dependency: # zmqpublishnotifier.cpp -> blockstorage.h -> txdb.h -> dbwrapper.h -> leveldb/db.h target_link_libraries(zmq leveldb) endif() # RPC client support add_library(rpcclient compat/stdin.cpp rpc/client.cpp ) target_link_libraries(rpcclient univalue util) # bitcoin-seeder if(BUILD_BITCOIN_SEEDER) add_subdirectory(seeder) endif() # bitcoin-cli if(BUILD_BITCOIN_CLI) add_executable(bitcoin-cli bitcoin-cli.cpp) generate_windows_version_info(bitcoin-cli DESCRIPTION "JSON-RPC client for ${PACKAGE_NAME}" ) target_link_libraries(bitcoin-cli common rpcclient) link_event(bitcoin-cli event) add_to_symbols_check(bitcoin-cli) add_to_security_check(bitcoin-cli) install_target(bitcoin-cli) install_manpages(bitcoin-cli) endif() # bitcoin-tx if(BUILD_BITCOIN_TX) add_executable(bitcoin-tx bitcoin-tx.cpp) generate_windows_version_info(bitcoin-tx DESCRIPTION "CLI Bitcoin transaction editor utility" ) target_link_libraries(bitcoin-tx bitcoinconsensus) add_to_symbols_check(bitcoin-tx) add_to_security_check(bitcoin-tx) install_target(bitcoin-tx) install_manpages(bitcoin-tx) endif() # bitcoin-chainstate if(BUILD_BITCOIN_CHAINSTATE) # TODO: libbitcoinkernel is a work in progress consensus engine library, as more # and more modules are decoupled from the consensus engine, this list will # shrink to only those which are absolutely necessary. For example, things # like index/*.cpp will be removed. add_library(bitcoinkernel kernel/bitcoinkernel.cpp + kernel/disconnected_transactions.cpp kernel/mempool_persist.cpp arith_uint256.cpp blockfileinfo.cpp blockfilter.cpp blockindex.cpp bloom.cpp chain.cpp chainparamsbase.cpp chainparams.cpp checkpoints.cpp clientversion.cpp coins.cpp compat/glibcxx_sanity.cpp compressor.cpp config.cpp consensus/activation.cpp consensus/amount.cpp consensus/merkle.cpp consensus/tx_check.cpp consensus/tx_verify.cpp core_read.cpp dbwrapper.cpp deploymentstatus.cpp eventloop.cpp feerate.cpp flatfile.cpp fs.cpp hash.cpp index/base.cpp index/blockfilterindex.cpp index/coinstatsindex.cpp init/common.cpp key.cpp logging.cpp networks/abc/chainparamsconstants.cpp networks/abc/checkpoints.cpp node/blockstorage.cpp node/chainstate.cpp node/coinstats.cpp node/ui_interface.cpp node/utxo_snapshot.cpp policy/fees.cpp policy/packages.cpp policy/policy.cpp policy/settings.cpp pow/aserti32d.cpp pow/daa.cpp pow/eda.cpp pow/pow.cpp primitives/block.cpp primitives/transaction.cpp pubkey.cpp random.cpp randomenv.cpp rcu.cpp scheduler.cpp script/bitfield.cpp script/interpreter.cpp script/script.cpp script/scriptcache.cpp script/script_error.cpp script/sigcache.cpp script/sigencoding.cpp script/standard.cpp shutdown.cpp support/cleanse.cpp support/lockedpool.cpp sync.cpp threadinterrupt.cpp txdb.cpp txmempool.cpp uint256.cpp util/bytevectorhash.cpp util/check.cpp util/getuniquepath.cpp util/hasher.cpp util/moneystr.cpp util/settings.cpp util/strencodings.cpp util/string.cpp util/system.cpp util/thread.cpp util/threadnames.cpp util/time.cpp util/tokenpipe.cpp validation.cpp validationinterface.cpp versionbits.cpp warnings.cpp # Bitcoin ABC specific dependencies that will not go away with Core backports addrdb.cpp # via banman.cpp addrman.cpp # via net.cpp banman.cpp # via net.cpp base58.cpp # via key_io.cpp avalanche/avalanche.cpp avalanche/delegation.cpp avalanche/delegationbuilder.cpp avalanche/peermanager.cpp avalanche/processor.cpp avalanche/proof.cpp avalanche/proofid.cpp avalanche/proofpool.cpp avalanche/voterecord.cpp cashaddr.cpp # via cashaddrenc.cpp cashaddrenc.cpp # via key_io.cpp dnsseeds.cpp # via net.cpp (GetRandomizedDNSSeeds) i2p.cpp # via net.cpp key_io.cpp # avalanche/processor.cpp uses DecodeSecret minerfund.cpp # via policy/block/minerfund.cpp net.cpp # avalanche uses CConnman netaddress.cpp # via net.cpp netbase.cpp # via net.cpp net_permissions.cpp # via net.cpp policy/block/minerfund.cpp policy/block/preconsensus.cpp policy/block/stakingrewards.cpp protocol.cpp # avalanche/processor.cpp uses NetMsgType timedata.cpp # via net.cpp util/asmap.cpp # via netaddress.cpp util/error.cpp # via net_permissions.cpp (ResolveErrMsg) util/readwritefile.cpp # via i2p.cpp util/sock.cpp # via net.cpp ) target_include_directories(bitcoinkernel PUBLIC . leveldb/helpers/memenv # To access the config/ and obj/ directories ${CMAKE_CURRENT_BINARY_DIR} ) target_link_libraries(bitcoinkernel crypto univalue secp256k1 leveldb memenv) link_boost_headers_only(bitcoinkernel headers) if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") link_windows_dependencies(bitcoinkernel) endif() add_executable(bitcoin-chainstate bitcoin-chainstate.cpp) target_link_libraries(bitcoin-chainstate bitcoinkernel) generate_windows_version_info(bitcoin-chainstate DESCRIPTION "CLI Datadir information utility (experimental)" ) add_to_symbols_check(bitcoin-chainstate) add_to_security_check(bitcoin-chainstate) endif() # bitcoind add_executable(bitcoind bitcoind.cpp) target_link_libraries(bitcoind server) generate_windows_version_info(bitcoind DESCRIPTION "Bitcoin node with a JSON-RPC server" ) add_to_symbols_check(bitcoind) add_to_security_check(bitcoind) install_target(bitcoind) install_manpages(bitcoind) # Bitcoin-qt if(BUILD_BITCOIN_QT) add_subdirectory(qt) endif() diff --git a/src/kernel/disconnected_transactions.cpp b/src/kernel/disconnected_transactions.cpp new file mode 100644 index 000000000..1794d8400 --- /dev/null +++ b/src/kernel/disconnected_transactions.cpp @@ -0,0 +1,222 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Copyright (c) 2024 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +/** Maximum bytes for transactions to store for processing during reorg */ +static const size_t MAX_DISCONNECTED_TX_POOL_SIZE = 20 * DEFAULT_MAX_BLOCK_SIZE; + +const DisconnectedBlockTransactions::TxInfo * +DisconnectedBlockTransactions::getTxInfo(const CTransactionRef &tx) const { + if (auto it = txInfo.find(tx->GetId()); it != txInfo.end()) { + return &it->second; + } + + return nullptr; +} + +void DisconnectedBlockTransactions::importMempool(CTxMemPool &pool) { + AssertLockHeld(pool.cs); + // addForBlock's algorithm sorts a vector of transactions back into + // topological order. We use it in a separate object to create a valid + // ordering of all mempool transactions, which we then splice in front of + // the current queuedTx. This results in a valid sequence of transactions to + // be reprocessed in updateMempoolForReorg. + + // We create vtx in order of the entry_id index to facilitate for + // addForBlocks (which iterates in reverse order), as vtx probably end in + // the correct ordering for queuedTx. + std::vector vtx; + + vtx.reserve(pool.mapTx.size()); + txInfo.reserve(pool.mapTx.size()); + for (const CTxMemPoolEntryRef &e : pool.mapTx.get()) { + vtx.push_back(e->GetSharedTx()); + // save entry time, feeDelta, and height for use in + // updateMempoolForReorg() + txInfo.try_emplace(e->GetTx().GetId(), e->GetTime(), + e->GetModifiedFee() - e->GetFee(), e->GetHeight()); + } + for (const CTxMemPoolEntryRef &e : + reverse_iterate(pool.mapTx.get())) { + // Notify all observers of this (possibly temporary) removal. This is + // necessary for tracking the transactions that are removed from the + // mempool during a reorg and can't be added back due to missing parent. + // Transactions that are added back to the mempool will trigger another + // notification. Make sure to notify in reverse topological order, + // children first. + GetMainSignals().TransactionRemovedFromMempool( + e->GetSharedTx(), MemPoolRemovalReason::REORG, + pool.GetAndIncrementSequence()); + } + pool.clear(); + + // Use addForBlocks to sort the transactions and then splice them in front + // of queuedTx + DisconnectedBlockTransactions orderedTxnPool; + orderedTxnPool.addForBlock(vtx, pool); + cachedInnerUsage += orderedTxnPool.cachedInnerUsage; + queuedTx.get().splice( + queuedTx.get().begin(), + orderedTxnPool.queuedTx.get()); + + // We limit memory usage because we can't know if more blocks will be + // disconnected + while (DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE) { + // Drop the earliest entry which, by definition, has no children + removeEntry(queuedTx.get().begin()); + } +} + +void DisconnectedBlockTransactions::addForBlock( + const std::vector &vtx, CTxMemPool &pool) { + AssertLockHeld(pool.cs); + for (const auto &tx : reverse_iterate(vtx)) { + // If we already added it, just skip. + auto it = queuedTx.find(tx->GetId()); + if (it != queuedTx.end()) { + continue; + } + + // Insert the transaction into the pool. + addTransaction(tx); + + // Fill in the set of parents. + std::unordered_set parents; + for (const CTxIn &in : tx->vin) { + parents.insert(in.prevout.GetTxId()); + } + + // In order to make sure we keep things in topological order, we check + // if we already know of the parent of the current transaction. If so, + // we remove them from the set and then add them back. + while (parents.size() > 0) { + std::unordered_set worklist( + std::move(parents)); + parents.clear(); + + for (const TxId &txid : worklist) { + // If we do not have that txid in the set, nothing needs to be + // done. + auto pit = queuedTx.find(txid); + if (pit == queuedTx.end()) { + continue; + } + + // We have parent in our set, we reinsert them at the right + // position. + const CTransactionRef ptx = *pit; + queuedTx.erase(pit); + queuedTx.insert(ptx); + + // And we make sure ancestors are covered. + for (const CTxIn &in : ptx->vin) { + parents.insert(in.prevout.GetTxId()); + } + } + } + } + + // Keep the size under control. + while (DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE) { + // Drop the earliest entry, and remove its children from the + // mempool. + auto it = queuedTx.get().begin(); + pool.removeRecursive(**it, MemPoolRemovalReason::REORG); + removeEntry(it); + } +} + +void DisconnectedBlockTransactions::removeForBlock( + const std::vector &vtx, CTxMemPool &pool) { + AssertLockHeld(pool.cs); + + if (pool.mapTx.empty() && pool.mapDeltas.empty()) { + // fast-path for IBD and/or when mempool is empty; there is no need to + // do any of the set-up work below which eats precious cycles. + // Note that this also skips updating the rolling fee udpate, which is + // fine: it is only recomputed when the mempool has to be trimmed down + // because it is full which is contradictory with this condition. + return; + } + + addForBlock(vtx, pool); + + for (const CTransactionRef &tx : + reverse_iterate(queuedTx.get())) { + CTxMemPool::txiter it = pool.mapTx.find(tx->GetId()); + if (it != pool.mapTx.end()) { + CTxMemPool::setEntries stage; + stage.insert(it); + pool.RemoveStaged(stage, MemPoolRemovalReason::BLOCK); + } else { + // Conflicting txs can only exist if the tx was not in the mempool + pool.removeConflicts(*tx); + } + pool.ClearPrioritisation(tx->GetId()); + } + + pool.updateFeeForBlock(); + + removeForBlock(vtx); +} + +void DisconnectedBlockTransactions::updateMempoolForReorg( + const Config &config, Chainstate &active_chainstate, bool fAddToMempool, + CTxMemPool &pool) { + AssertLockHeld(cs_main); + AssertLockHeld(pool.cs); + + if (fAddToMempool) { + // disconnectpool's insertion_order index sorts the entries from oldest + // to newest, but the oldest entry will be the last tx from the latest + // mined block that was disconnected. + // Iterate disconnectpool in reverse, so that we add transactions back + // to the mempool starting with the earliest transaction that had been + // previously seen in a block. + for (const CTransactionRef &tx : + reverse_iterate(queuedTx.get())) { + if (tx->IsCoinBase()) { + continue; + } + // restore saved PrioritiseTransaction state and nAcceptTime + const auto ptxInfo = getTxInfo(tx); + bool hasFeeDelta = false; + if (ptxInfo && ptxInfo->feeDelta != Amount::zero()) { + // manipulate mapDeltas directly (faster than calling + // PrioritiseTransaction) + pool.mapDeltas[tx->GetId()] = ptxInfo->feeDelta; + hasFeeDelta = true; + } + // ignore validation errors in resurrected transactions + auto result = AcceptToMemoryPool( + config, active_chainstate, tx, + /*accept_time=*/ptxInfo ? ptxInfo->time.count() : GetTime(), + /*bypass_limits=*/true, /*test_accept=*/false, + /*heightOverride=*/ptxInfo ? ptxInfo->height : 0); + if (result.m_result_type != + MempoolAcceptResult::ResultType::VALID && + hasFeeDelta) { + // tx not accepted: undo mapDelta insertion from above + pool.mapDeltas.erase(tx->GetId()); + } + } + } + + queuedTx.clear(); + txInfo.clear(); + + // Re-limit mempool size, in case we added any transactions + pool.LimitSize(active_chainstate.CoinsTip()); +} diff --git a/src/kernel/disconnected_transactions.h b/src/kernel/disconnected_transactions.h new file mode 100644 index 000000000..5c4cd37d5 --- /dev/null +++ b/src/kernel/disconnected_transactions.h @@ -0,0 +1,172 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Copyright (c) 2024 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_KERNEL_DISCONNECTED_TRANSACTIONS_H +#define BITCOIN_KERNEL_DISCONNECTED_TRANSACTIONS_H + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +/** + * DisconnectedBlockTransactions + * + * During the reorg, it's desirable to re-add previously confirmed transactions + * to the mempool, so that anything not re-confirmed in the new chain is + * available to be mined. However, it's more efficient to wait until the reorg + * is complete and process all still-unconfirmed transactions at that time, + * since we expect most confirmed transactions to (typically) still be + * confirmed in the new chain, and re-accepting to the memory pool is expensive + * (and therefore better to not do in the middle of reorg-processing). + * Instead, store the disconnected transactions (in order!) as we go, remove any + * that are included in blocks in the new chain, and then process the remaining + * still-unconfirmed transactions at the end. + * + * It also enables efficient reprocessing of current mempool entries, useful + * when (de)activating forks that result in in-mempool transactions becoming + * invalid + */ +// multi_index tag names +struct txid_index {}; +struct insertion_order {}; + +class DisconnectedBlockTransactions { +private: + typedef boost::multi_index_container< + CTransactionRef, boost::multi_index::indexed_by< + // hashed by txid + boost::multi_index::hashed_unique< + boost::multi_index::tag, + mempoolentry_txid, SaltedTxIdHasher>, + // sorted by order in the blockchain + boost::multi_index::sequenced< + boost::multi_index::tag>>> + indexed_disconnected_transactions; + + indexed_disconnected_transactions queuedTx; + uint64_t cachedInnerUsage = 0; + + struct TxInfo { + const std::chrono::seconds time; + const Amount feeDelta; + const unsigned height; + TxInfo(const std::chrono::seconds &time_, Amount feeDelta_, + unsigned height_) noexcept + : time(time_), feeDelta(feeDelta_), height(height_) {} + }; + + using TxInfoMap = std::unordered_map; + /// populated by importMempool(); the original tx entry times and feeDeltas + TxInfoMap txInfo; + + void addTransaction(const CTransactionRef &tx) { + queuedTx.insert(tx); + cachedInnerUsage += RecursiveDynamicUsage(tx); + } + + /// @returns a pointer into the txInfo map if tx->GetId() exists in the map, + /// or nullptr otherwise. Note that the returned pointer is only valid for + /// as long as its underlying map node is valid. + const TxInfo *getTxInfo(const CTransactionRef &tx) const; + +public: + // It's almost certainly a logic bug if we don't clear out queuedTx before + // destruction, as we add to it while disconnecting blocks, and then we + // need to re-process remaining transactions to ensure mempool consistency. + // For now, assert() that we've emptied out this object on destruction. + // This assert() can always be removed if the reorg-processing code were + // to be refactored such that this assumption is no longer true (for + // instance if there was some other way we cleaned up the mempool after a + // reorg, besides draining this object). + ~DisconnectedBlockTransactions() { assert(queuedTx.empty()); } + + // Estimate the overhead of queuedTx to be 6 pointers + an allocation, as + // no exact formula for boost::multi_index_contained is implemented. + size_t DynamicMemoryUsage() const { + return memusage::MallocUsage(sizeof(CTransactionRef) + + 6 * sizeof(void *)) * + queuedTx.size() + + memusage::DynamicUsage(txInfo) + cachedInnerUsage; + } + + const indexed_disconnected_transactions &GetQueuedTx() const { + return queuedTx; + } + + // Import mempool entries in topological order into queuedTx and clear the + // mempool. Caller should call updateMempoolForReorg to reprocess these + // transactions + void importMempool(CTxMemPool &pool) EXCLUSIVE_LOCKS_REQUIRED(pool.cs); + + // Add entries for a block while reconstructing the topological ordering so + // they can be added back to the mempool simply. + void addForBlock(const std::vector &vtx, CTxMemPool &pool) + EXCLUSIVE_LOCKS_REQUIRED(pool.cs); + + // Remove entries based on txid_index, and update memory usage. + void removeForBlock(const std::vector &vtx) { + // Short-circuit in the common case of a block being added to the tip + if (queuedTx.empty()) { + return; + } + for (auto const &tx : vtx) { + auto it = queuedTx.find(tx->GetId()); + if (it != queuedTx.end()) { + cachedInnerUsage -= RecursiveDynamicUsage(*it); + queuedTx.erase(it); + txInfo.erase(tx->GetId()); + } + } + } + + void removeForBlock(const std::vector &vtx, + CTxMemPool &pool) EXCLUSIVE_LOCKS_REQUIRED(pool.cs); + + // Remove an entry by insertion_order index, and update memory usage. + void removeEntry(indexed_disconnected_transactions::index< + insertion_order>::type::iterator entry) { + cachedInnerUsage -= RecursiveDynamicUsage(*entry); + txInfo.erase((*entry)->GetId()); + queuedTx.get().erase(entry); + } + + bool isEmpty() const { return queuedTx.empty(); } + + void clear() { + cachedInnerUsage = 0; + queuedTx.clear(); + txInfo.clear(); + } + + /** + * Make mempool consistent after a reorg, by re-adding or recursively + * erasing disconnected block transactions from the mempool, and also + * removing any other transactions from the mempool that are no longer valid + * given the new tip/height. + * + * Note: we assume that disconnectpool only contains transactions that are + * NOT confirmed in the current chain nor already in the mempool (otherwise, + * in-mempool descendants of such transactions would be removed). + * + * Passing fAddToMempool=false will skip trying to add the transactions + * back, and instead just erase from the mempool as needed. + */ + void updateMempoolForReorg(const Config &config, + Chainstate &active_chainstate, + bool fAddToMempool, CTxMemPool &pool) + EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs); +}; + +#endif // BITCOIN_KERNEL_DISCONNECTED_TRANSACTIONS_H diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index e5dcd32d4..fc066bd29 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -1,582 +1,583 @@ // Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include #include #include #include #include #include #include #include BOOST_FIXTURE_TEST_SUITE(mempool_tests, TestingSetup) static constexpr auto REMOVAL_REASON_DUMMY = MemPoolRemovalReason::REPLACED; BOOST_AUTO_TEST_CASE(MempoolRemoveTest) { // Test CTxMemPool::remove functionality TestMemPoolEntryHelper entry; // Parent transaction with three children, and three grand-children: CMutableTransaction txParent; txParent.vin.resize(1); txParent.vin[0].scriptSig = CScript() << OP_11; txParent.vout.resize(3); for (int i = 0; i < 3; i++) { txParent.vout[i].scriptPubKey = CScript() << OP_11 << OP_EQUAL; txParent.vout[i].nValue = 33000 * SATOSHI; } CMutableTransaction txChild[3]; for (int i = 0; i < 3; i++) { txChild[i].vin.resize(1); txChild[i].vin[0].scriptSig = CScript() << OP_11; txChild[i].vin[0].prevout = COutPoint(txParent.GetId(), i); txChild[i].vout.resize(1); txChild[i].vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; txChild[i].vout[0].nValue = 11000 * SATOSHI; } CMutableTransaction txGrandChild[3]; for (int i = 0; i < 3; i++) { txGrandChild[i].vin.resize(1); txGrandChild[i].vin[0].scriptSig = CScript() << OP_11; txGrandChild[i].vin[0].prevout = COutPoint(txChild[i].GetId(), 0); txGrandChild[i].vout.resize(1); txGrandChild[i].vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; txGrandChild[i].vout[0].nValue = 11000 * SATOSHI; } CTxMemPool &testPool = *Assert(m_node.mempool); LOCK2(cs_main, testPool.cs); // Nothing in pool, remove should do nothing: unsigned int poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txParent), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize); // Just the parent: testPool.addUnchecked(entry.FromTx(txParent)); poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txParent), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize - 1); // Parent, children, grandchildren: testPool.addUnchecked(entry.FromTx(txParent)); for (int i = 0; i < 3; i++) { testPool.addUnchecked(entry.FromTx(txChild[i])); testPool.addUnchecked(entry.FromTx(txGrandChild[i])); } // Remove Child[0], GrandChild[0] should be removed: poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txChild[0]), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize - 2); // ... make sure grandchild and child are gone: poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txGrandChild[0]), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize); poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txChild[0]), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize); // Remove parent, all children/grandchildren should go: poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txParent), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize - 5); BOOST_CHECK_EQUAL(testPool.size(), 0UL); // Add children and grandchildren, but NOT the parent (simulate the parent // being in a block) for (int i = 0; i < 3; i++) { testPool.addUnchecked(entry.FromTx(txChild[i])); testPool.addUnchecked(entry.FromTx(txGrandChild[i])); } // Now remove the parent, as might happen if a block-re-org occurs but the // parent cannot be put into the mempool (maybe because it is non-standard): poolSize = testPool.size(); testPool.removeRecursive(CTransaction(txParent), REMOVAL_REASON_DUMMY); BOOST_CHECK_EQUAL(testPool.size(), poolSize - 6); BOOST_CHECK_EQUAL(testPool.size(), 0UL); } BOOST_AUTO_TEST_CASE(MempoolClearTest) { // Test CTxMemPool::clear functionality TestMemPoolEntryHelper entry; // Create a transaction CMutableTransaction txParent; txParent.vin.resize(1); txParent.vin[0].scriptSig = CScript() << OP_11; txParent.vout.resize(3); for (int i = 0; i < 3; i++) { txParent.vout[i].scriptPubKey = CScript() << OP_11 << OP_EQUAL; txParent.vout[i].nValue = 33000 * SATOSHI; } CTxMemPool &testPool = *Assert(m_node.mempool); LOCK2(cs_main, testPool.cs); // Nothing in pool, clear should do nothing: testPool.clear(); BOOST_CHECK_EQUAL(testPool.size(), 0UL); // Add the transaction testPool.addUnchecked(entry.FromTx(txParent)); BOOST_CHECK_EQUAL(testPool.size(), 1UL); BOOST_CHECK_EQUAL(testPool.mapTx.size(), 1UL); BOOST_CHECK_EQUAL(testPool.mapNextTx.size(), 1UL); // CTxMemPool's members should be empty after a clear testPool.clear(); BOOST_CHECK_EQUAL(testPool.size(), 0UL); BOOST_CHECK_EQUAL(testPool.mapTx.size(), 0UL); BOOST_CHECK_EQUAL(testPool.mapNextTx.size(), 0UL); } template static void CheckSort(CTxMemPool &pool, std::vector &sortedOrder, const std::string &testcase) EXCLUSIVE_LOCKS_REQUIRED(pool.cs) { BOOST_CHECK_EQUAL(pool.size(), sortedOrder.size()); typename CTxMemPool::indexed_transaction_set::index::type::iterator it = pool.mapTx.get().begin(); int count = 0; for (; it != pool.mapTx.get().end(); ++it, ++count) { BOOST_CHECK_MESSAGE((*it)->GetTx().GetId().ToString() == sortedOrder[count], (*it)->GetTx().GetId().ToString() << " != " << sortedOrder[count] << " in test " << testcase << ":" << count); } } BOOST_AUTO_TEST_CASE(MempoolIndexingTest) { CTxMemPool &pool = *Assert(m_node.mempool); LOCK2(cs_main, pool.cs); TestMemPoolEntryHelper entry; /** * Remove the default nonzero sigChecks, since the below tests are * focussing on fee-based ordering and involve some artificially very tiny * 21-byte transactions without any inputs. */ entry.SigChecks(0); /* 3rd highest fee */ CMutableTransaction tx1 = CMutableTransaction(); tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; /* highest fee */ CMutableTransaction tx2 = CMutableTransaction(); tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx2.vout[0].nValue = 2 * COIN; pool.addUnchecked(entry.Fee(20000 * SATOSHI).FromTx(tx2)); /* lowest fee */ CMutableTransaction tx3 = CMutableTransaction(); tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx3.vout[0].nValue = 5 * COIN; pool.addUnchecked(entry.Fee(Amount::zero()).FromTx(tx3)); /* 2nd highest fee */ CMutableTransaction tx4 = CMutableTransaction(); tx4.vout.resize(1); tx4.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx4.vout[0].nValue = 6 * COIN; pool.addUnchecked(entry.Fee(15000 * SATOSHI).FromTx(tx4)); /* equal fee rate to tx1, but arrived later to the mempool */ CMutableTransaction tx5 = CMutableTransaction(); tx5.vout.resize(1); tx5.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx5.vout[0].nValue = 11 * COIN; pool.addUnchecked(entry.Fee(10000 * SATOSHI).Time(100).FromTx(tx1)); pool.addUnchecked(entry.Fee(10000 * SATOSHI).Time(200).FromTx(tx5)); BOOST_CHECK_EQUAL(pool.size(), 5UL); std::vector sortedOrder; sortedOrder.resize(5); sortedOrder[0] = tx2.GetId().ToString(); // 20000 sortedOrder[1] = tx4.GetId().ToString(); // 15000 sortedOrder[3] = tx5.GetId().ToString(); // 10000 sortedOrder[2] = tx1.GetId().ToString(); // 10000 sortedOrder[4] = tx3.GetId().ToString(); // 0 CheckSort(pool, sortedOrder, "MempoolIndexingTest1"); } BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) { CTxMemPool &pool = *Assert(m_node.mempool); LOCK2(cs_main, pool.cs); TestMemPoolEntryHelper entry; Amount feeIncrement = MEMPOOL_FULL_FEE_INCREMENT.GetFeePerK(); CMutableTransaction tx1 = CMutableTransaction(); tx1.vin.resize(1); tx1.vin[0].scriptSig = CScript() << OP_1; tx1.vout.resize(1); tx1.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; tx1.vout[0].nValue = 10 * COIN; pool.addUnchecked(entry.Fee(20000 * SATOSHI).FromTx(tx1)); CMutableTransaction tx2 = CMutableTransaction(); tx2.vin.resize(1); tx2.vin[0].scriptSig = CScript() << OP_2; tx2.vout.resize(1); tx2.vout[0].scriptPubKey = CScript() << OP_2 << OP_EQUAL; tx2.vout[0].nValue = 10 * COIN; pool.addUnchecked(entry.Fee(4000 * SATOSHI).FromTx(tx2)); // should do nothing pool.TrimToSize(pool.DynamicMemoryUsage()); BOOST_CHECK(pool.exists(tx1.GetId())); BOOST_CHECK(pool.exists(tx2.GetId())); // should remove the lower-feerate transaction pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); BOOST_CHECK(pool.exists(tx1.GetId())); BOOST_CHECK(!pool.exists(tx2.GetId())); pool.addUnchecked(entry.FromTx(tx2)); CMutableTransaction tx3 = CMutableTransaction(); tx3.vin.resize(1); tx3.vin[0].prevout = COutPoint(tx2.GetId(), 0); tx3.vin[0].scriptSig = CScript() << OP_2; tx3.vout.resize(1); tx3.vout[0].scriptPubKey = CScript() << OP_3 << OP_EQUAL; tx3.vout[0].nValue = 10 * COIN; pool.addUnchecked(entry.Fee(16000 * SATOSHI).FromTx(tx3)); // tx2 should be removed, tx3 is a child of tx2, so it should be removed // even though it has highest fee. pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); BOOST_CHECK(pool.exists(tx1.GetId())); BOOST_CHECK(!pool.exists(tx2.GetId())); BOOST_CHECK(!pool.exists(tx3.GetId())); // mempool is limited to tx1's size in memory usage, so nothing fits std::vector vNoSpendsRemaining; pool.TrimToSize(CTransaction(tx1).GetTotalSize(), &vNoSpendsRemaining); BOOST_CHECK(!pool.exists(tx1.GetId())); BOOST_CHECK(!pool.exists(tx2.GetId())); BOOST_CHECK(!pool.exists(tx3.GetId())); // This vector should only contain 'root' (not unconfirmed) outpoints // Though both tx2 and tx3 were removed, tx3's input came from tx2. BOOST_CHECK_EQUAL(vNoSpendsRemaining.size(), 1); BOOST_CHECK(vNoSpendsRemaining == std::vector{COutPoint()}); // maxFeeRateRemoved was set by the transaction with the highest fee, // that was not removed because it was a child of another tx. CFeeRate maxFeeRateRemoved(20000 * SATOSHI, CTransaction(tx1).GetTotalSize()); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + feeIncrement); CMutableTransaction tx4 = CMutableTransaction(); tx4.vin.resize(2); tx4.vin[0].prevout = COutPoint(); tx4.vin[0].scriptSig = CScript() << OP_4; tx4.vin[1].prevout = COutPoint(); tx4.vin[1].scriptSig = CScript() << OP_4; tx4.vout.resize(2); tx4.vout[0].scriptPubKey = CScript() << OP_4 << OP_EQUAL; tx4.vout[0].nValue = 10 * COIN; tx4.vout[1].scriptPubKey = CScript() << OP_4 << OP_EQUAL; tx4.vout[1].nValue = 10 * COIN; CMutableTransaction tx5 = CMutableTransaction(); tx5.vin.resize(2); tx5.vin[0].prevout = COutPoint(tx4.GetId(), 0); tx5.vin[0].scriptSig = CScript() << OP_4; tx5.vin[1].prevout = COutPoint(); tx5.vin[1].scriptSig = CScript() << OP_5; tx5.vout.resize(2); tx5.vout[0].scriptPubKey = CScript() << OP_5 << OP_EQUAL; tx5.vout[0].nValue = 10 * COIN; tx5.vout[1].scriptPubKey = CScript() << OP_5 << OP_EQUAL; tx5.vout[1].nValue = 10 * COIN; CMutableTransaction tx6 = CMutableTransaction(); tx6.vin.resize(2); tx6.vin[0].prevout = COutPoint(tx4.GetId(), 1); tx6.vin[0].scriptSig = CScript() << OP_4; tx6.vin[1].prevout = COutPoint(); tx6.vin[1].scriptSig = CScript() << OP_6; tx6.vout.resize(2); tx6.vout[0].scriptPubKey = CScript() << OP_6 << OP_EQUAL; tx6.vout[0].nValue = 10 * COIN; tx6.vout[1].scriptPubKey = CScript() << OP_6 << OP_EQUAL; tx6.vout[1].nValue = 10 * COIN; CMutableTransaction tx7 = CMutableTransaction(); tx7.vin.resize(2); tx7.vin[0].prevout = COutPoint(tx5.GetId(), 0); tx7.vin[0].scriptSig = CScript() << OP_5; tx7.vin[1].prevout = COutPoint(tx6.GetId(), 0); tx7.vin[1].scriptSig = CScript() << OP_6; tx7.vout.resize(2); tx7.vout[0].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[0].nValue = 10 * COIN; tx7.vout[1].scriptPubKey = CScript() << OP_7 << OP_EQUAL; tx7.vout[1].nValue = 10 * COIN; pool.addUnchecked(entry.Fee(7000 * SATOSHI).FromTx(tx4)); pool.addUnchecked(entry.Fee(1000 * SATOSHI).FromTx(tx5)); pool.addUnchecked(entry.Fee(1100 * SATOSHI).FromTx(tx6)); pool.addUnchecked(entry.Fee(9000 * SATOSHI).FromTx(tx7)); // we only require this to remove, at max, 2 txn, because it's not clear // what we're really optimizing for aside from that pool.TrimToSize(pool.DynamicMemoryUsage() - 1); BOOST_CHECK(pool.exists(tx4.GetId())); BOOST_CHECK(pool.exists(tx6.GetId())); BOOST_CHECK(!pool.exists(tx7.GetId())); if (!pool.exists(tx5.GetId())) { pool.addUnchecked(entry.Fee(1000 * SATOSHI).FromTx(tx5)); } pool.addUnchecked(entry.Fee(9000 * SATOSHI).FromTx(tx7)); // should maximize mempool size by only removing 5/7 pool.TrimToSize(pool.DynamicMemoryUsage() / 2); BOOST_CHECK(pool.exists(tx4.GetId())); BOOST_CHECK(!pool.exists(tx5.GetId())); BOOST_CHECK(pool.exists(tx6.GetId())); BOOST_CHECK(!pool.exists(tx7.GetId())); pool.addUnchecked(entry.Fee(1000 * SATOSHI).FromTx(tx5)); pool.addUnchecked(entry.Fee(9000 * SATOSHI).FromTx(tx7)); std::vector vtx; SetMockTime(42); SetMockTime(42 + CTxMemPool::ROLLING_FEE_HALFLIFE); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + feeIncrement); // ... we should keep the same min fee until we get a block DisconnectedBlockTransactions disconnectedBlockTxs; disconnectedBlockTxs.removeForBlock(vtx, pool); SetMockTime(42 + 2 * CTxMemPool::ROLLING_FEE_HALFLIFE); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + feeIncrement) / 2); // ... then feerate should drop 1/2 each halflife SetMockTime(42 + 2 * CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE / 2); BOOST_CHECK_EQUAL( pool.GetMinFee(pool.DynamicMemoryUsage() * 5 / 2).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + feeIncrement) / 4); // ... with a 1/2 halflife when mempool is < 1/2 its target size SetMockTime(42 + 2 * CTxMemPool::ROLLING_FEE_HALFLIFE + CTxMemPool::ROLLING_FEE_HALFLIFE / 2 + CTxMemPool::ROLLING_FEE_HALFLIFE / 4); BOOST_CHECK_EQUAL( pool.GetMinFee(pool.DynamicMemoryUsage() * 9 / 2).GetFeePerK(), (maxFeeRateRemoved.GetFeePerK() + feeIncrement) / 8 + SATOSHI); // ... with a 1/4 halflife when mempool is < 1/4 its target size } // expectedSize can be smaller than correctlyOrderedIds.size(), since we // might be testing intermediary states. Just avoiding some slice operations, void CheckDisconnectPoolOrder(DisconnectedBlockTransactions &disconnectPool, std::vector correctlyOrderedIds, unsigned int expectedSize) { int i = 0; BOOST_CHECK_EQUAL(disconnectPool.GetQueuedTx().size(), expectedSize); // Txns in queuedTx's insertion_order index are sorted from children to // parent txn for (const CTransactionRef &tx : reverse_iterate(disconnectPool.GetQueuedTx().get())) { BOOST_CHECK(tx->GetId() == correctlyOrderedIds[i]); i++; } } typedef std::vector vecptx; BOOST_AUTO_TEST_CASE(TestImportMempool) { CMutableTransaction chainedTxn[5]; std::vector correctlyOrderedIds; COutPoint lastOutpoint; // Construct a chain of 5 transactions for (int i = 0; i < 5; i++) { chainedTxn[i].vin.emplace_back(lastOutpoint); chainedTxn[i].vout.emplace_back(10 * SATOSHI, CScript() << OP_TRUE); correctlyOrderedIds.push_back(chainedTxn[i].GetId()); lastOutpoint = COutPoint(correctlyOrderedIds[i], 0); } // The first 3 txns simulate once confirmed transactions that have been // disconnected. We test 3 different orders: in order, one case of mixed // order and inverted order. vecptx disconnectedTxnsInOrder = {&chainedTxn[0], &chainedTxn[1], &chainedTxn[2]}; vecptx disconnectedTxnsMixedOrder = {&chainedTxn[1], &chainedTxn[2], &chainedTxn[0]}; vecptx disconnectedTxnsInvertedOrder = {&chainedTxn[2], &chainedTxn[1], &chainedTxn[0]}; // The last 2 txns simulate a chain of unconfirmed transactions in the // mempool. We test 2 different orders: in and out of order. vecptx unconfTxnsInOrder = {&chainedTxn[3], &chainedTxn[4]}; vecptx unconfTxnsOutOfOrder = {&chainedTxn[4], &chainedTxn[3]}; // Now we test all combinations of the previously defined orders for // disconnected and unconfirmed txns. The expected outcome is to have these // transactions in the correct order in queuedTx, as defined in // correctlyOrderedIds. for (auto &disconnectedTxns : {disconnectedTxnsInOrder, disconnectedTxnsMixedOrder, disconnectedTxnsInvertedOrder}) { for (auto &unconfTxns : {unconfTxnsInOrder, unconfTxnsOutOfOrder}) { CTxMemPool &testPool = *Assert(m_node.mempool); // addForBlock inserts disconnectTxns in disconnectPool. They // simulate transactions that were once confirmed in a block std::vector vtx; for (auto tx : disconnectedTxns) { vtx.push_back(MakeTransactionRef(*tx)); } DisconnectedBlockTransactions disconnectPool; LOCK2(cs_main, testPool.cs); { disconnectPool.addForBlock(vtx, testPool); CheckDisconnectPoolOrder(disconnectPool, correctlyOrderedIds, disconnectedTxns.size()); // If the mempool is empty, importMempool doesn't change // disconnectPool disconnectPool.importMempool(testPool); CheckDisconnectPoolOrder(disconnectPool, correctlyOrderedIds, disconnectedTxns.size()); // Add all unconfirmed transactions in testPool for (auto tx : unconfTxns) { TestMemPoolEntryHelper entry; testPool.addUnchecked(entry.FromTx(*tx)); } // Now we test importMempool with a non empty mempool disconnectPool.importMempool(testPool); } CheckDisconnectPoolOrder(disconnectPool, correctlyOrderedIds, disconnectedTxns.size() + unconfTxns.size()); // We must clear disconnectPool to not trigger the assert in its // destructor disconnectPool.clear(); } } } inline CTransactionRef make_tx(std::vector &&output_values, std::vector &&inputs = std::vector(), std::vector &&input_indices = std::vector()) { CMutableTransaction tx = CMutableTransaction(); tx.vin.resize(inputs.size()); tx.vout.resize(output_values.size()); for (size_t i = 0; i < inputs.size(); ++i) { tx.vin[i].prevout = COutPoint(inputs[i]->GetId(), input_indices.size() > i ? input_indices[i] : 0); } for (size_t i = 0; i < output_values.size(); ++i) { tx.vout[i].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx.vout[i].nValue = output_values[i]; } return MakeTransactionRef(tx); } BOOST_AUTO_TEST_CASE(GetModifiedFeeRateTest) { CMutableTransaction tx = CMutableTransaction(); tx.vin.resize(1); // Make tx exactly 1000 bytes. const size_t dummyDataSize = 1000 - (GetSerializeSize(tx, PROTOCOL_VERSION) + 5 /* OP_PUSHDATA2 and ?? */); tx.vin[0].scriptSig << std::vector(dummyDataSize); assert(GetSerializeSize(tx, PROTOCOL_VERSION) == 1000); TestMemPoolEntryHelper entry; auto entryNormal = entry.Fee(1000 * SATOSHI).FromTx(tx); BOOST_CHECK_EQUAL(1000 * SATOSHI, entryNormal->GetModifiedFeeRate().GetFee(1000)); // Add modified fee CTxMemPoolEntryRef entryFeeModified = entry.Fee(1000 * SATOSHI).FromTx(tx); entryFeeModified->UpdateFeeDelta(1000 * SATOSHI); BOOST_CHECK_EQUAL(2000 * SATOSHI, entryFeeModified->GetModifiedFeeRate().GetFee(1000)); // Excessive sigop count "modifies" size CTxMemPoolEntryRef entrySizeModified = entry.Fee(1000 * SATOSHI) .SigChecks(2000 / DEFAULT_BYTES_PER_SIGCHECK) .FromTx(tx); BOOST_CHECK_EQUAL(500 * SATOSHI, entrySizeModified->GetModifiedFeeRate().GetFee(1000)); } BOOST_AUTO_TEST_CASE(CompareTxMemPoolEntryByModifiedFeeRateTest) { CTransactionRef a = make_tx(/* output_values */ {1 * COIN}); CTransactionRef b = make_tx(/* output_values */ {2 * COIN}); // For this test, we want b to have lower txid. if (a->GetId() < b->GetId()) { std::swap(a, b); } BOOST_CHECK_GT(a->GetId(), b->GetId()); TestMemPoolEntryHelper entry; CompareTxMemPoolEntryByModifiedFeeRate compare; auto checkOrdering = [&compare](const auto &a, const auto &b) { BOOST_CHECK(compare(a, b)); BOOST_CHECK(!compare(b, a)); }; // If the fees and entryId are the same, lower TxId should sort before checkOrdering(entry.Fee(100 * SATOSHI).FromTx(b), entry.Fee(100 * SATOSHI).FromTx(a)); // Earlier entryId, same fee, should sort before checkOrdering(entry.Fee(100 * SATOSHI).EntryId(1).FromTx(a), entry.Fee(100 * SATOSHI).EntryId(2).FromTx(b)); // Higher fee, earlier entryId should sort before checkOrdering(entry.Fee(101 * SATOSHI).EntryId(1).FromTx(a), entry.Fee(100 * SATOSHI).EntryId(2).FromTx(b)); // Higher fee, same entryId should sort before checkOrdering(entry.Fee(101 * SATOSHI).FromTx(a), entry.Fee(100 * SATOSHI).FromTx(b)); // Same with fee delta. { CTxMemPoolEntryRef entryA = entry.Fee(100 * SATOSHI).FromTx(a); CTxMemPoolEntryRef entryB = entry.Fee(200 * SATOSHI).FromTx(b); // .. A and B have same modified fee, ordering is by lowest txid entryA->UpdateFeeDelta(100 * SATOSHI); checkOrdering(entryB, entryA); } // .. A is first entering the mempool CTxMemPoolEntryRef entryA = entry.Fee(100 * SATOSHI).EntryId(1).FromTx(a); CTxMemPoolEntryRef entryB = entry.Fee(100 * SATOSHI).EntryId(2).FromTx(b); checkOrdering(entryA, entryB); // .. B has higher modified fee. entryB->UpdateFeeDelta(1 * SATOSHI); checkOrdering(entryB, entryA); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 614b56422..22c7427ff 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -1,99 +1,100 @@ // Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include +#include #include #include #include #include #include BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, TestingSetup) BOOST_AUTO_TEST_CASE(MempoolMinimumFeeEstimate) { CTxMemPool &mpool = *Assert(m_node.mempool); LOCK2(cs_main, mpool.cs); TestMemPoolEntryHelper entry; // Create a transaction template CScript garbage; for (unsigned int i = 0; i < 128; i++) { garbage.push_back('X'); } CMutableTransaction tx; tx.vin.resize(1); tx.vin[0].scriptSig = garbage; tx.vout.resize(1); tx.vout[0].nValue = Amount::zero(); // Create a fake block std::vector block; int blocknum = 0; // Loop through 200 blocks adding transactions so we have a estimateFee // that is calculable. while (blocknum < 200) { for (int64_t j = 0; j < 100; j++) { // make transaction unique tx.vin[0].nSequence = 10000 * blocknum + j; TxId txid = tx.GetId(); mpool.addUnchecked( entry.Fee((j + 1) * DEFAULT_BLOCK_MIN_TX_FEE_PER_KB) .Time(GetTime()) .Height(blocknum++) .FromTx(tx)); CTransactionRef ptx = mpool.get(txid); block.push_back(ptx); } DisconnectedBlockTransactions disconnectedBlocktxs; disconnectedBlocktxs.removeForBlock(block, mpool); block.clear(); } // Check that the estimate is above the rolling minimum fee. This should be // true since we have not trimmed the mempool. BOOST_CHECK(mpool.GetMinFee(1) <= mpool.estimateFee()); // Check that estimateFee returns the minimum rolling fee even when the // mempool grows very quickly and no blocks have been mined. // Add a bunch of low fee transactions which are not in the mempool // And have zero fees. CMutableTransaction mtx; tx.vin.resize(1); tx.vin[0].scriptSig = garbage; tx.vout.resize(1); block.clear(); // Add tons of transactions to the mempool, // but don't mine them. for (int64_t i = 0; i < 10000; i++) { // Mutate the hash tx.vin[0].nSequence = 10000 * blocknum + i; // Add new transaction to the mempool with a increasing fee // The average should end up as 1/2 * 100 * // DEFAULT_BLOCK_MIN_TX_FEE_PER_KB mpool.addUnchecked(entry.Fee((i + 1) * DEFAULT_BLOCK_MIN_TX_FEE_PER_KB) .Time(GetTime()) .Height(blocknum) .FromTx(tx)); } // Trim to size. GetMinFee should be more than 10000 * // DEFAULT_BLOCK_MIN_TX_FEE_PER_KB, but the estimateFee should remain // unchanged. mpool.TrimToSize(1); BOOST_CHECK(mpool.GetMinFee(1) >= CFeeRate(10000 * DEFAULT_BLOCK_MIN_TX_FEE_PER_KB, CTransaction(tx).GetTotalSize())); BOOST_CHECK_MESSAGE(mpool.estimateFee() == mpool.GetMinFee(1), "Confirm blocks has failed"); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index ea6a4a4c1..463874246 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -1,735 +1,736 @@ // Copyright (c) 2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // #include #include #include +#include #include #include #include #include #include #include #include #include #include #include #include #include using node::SnapshotMetadata; BOOST_FIXTURE_TEST_SUITE(validation_chainstatemanager_tests, ChainTestingSetup) //! Basic tests for ChainstateManager. //! //! First create a legacy (IBD) chainstate, then create a snapshot chainstate. BOOST_AUTO_TEST_CASE(chainstatemanager) { ChainstateManager &manager = *m_node.chainman; CTxMemPool &mempool = *m_node.mempool; std::vector chainstates; BOOST_CHECK(!manager.SnapshotBlockhash().has_value()); // Create a legacy (IBD) chainstate. // Chainstate &c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(&mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); BOOST_CHECK(!manager.IsSnapshotActive()); BOOST_CHECK(WITH_LOCK(::cs_main, return !manager.IsSnapshotValidated())); auto all = manager.GetAll(); BOOST_CHECK_EQUAL_COLLECTIONS(all.begin(), all.end(), chainstates.begin(), chainstates.end()); auto &active_chain = WITH_LOCK(manager.GetMutex(), return manager.ActiveChain()); BOOST_CHECK_EQUAL(&active_chain, &c1.m_chain); BOOST_CHECK_EQUAL( WITH_LOCK(manager.GetMutex(), return manager.ActiveHeight()), -1); auto active_tip = WITH_LOCK(manager.GetMutex(), return manager.ActiveTip()); auto exp_tip = c1.m_chain.Tip(); BOOST_CHECK_EQUAL(active_tip, exp_tip); BOOST_CHECK(!manager.SnapshotBlockhash().has_value()); // Create a snapshot-based chainstate. // const BlockHash snapshot_blockhash{GetRandHash()}; Chainstate &c2 = WITH_LOCK( ::cs_main, return manager.ActivateExistingSnapshot(&mempool, snapshot_blockhash)); chainstates.push_back(&c2); BOOST_CHECK_EQUAL(manager.SnapshotBlockhash().value(), snapshot_blockhash); c2.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); WITH_LOCK(::cs_main, c2.InitCoinsCache(1 << 23)); // Unlike c1, which doesn't have any blocks. Gets us different tip, height. c2.LoadGenesisBlock(); BlockValidationState _; BOOST_CHECK(c2.ActivateBestChain(GetConfig(), _, nullptr)); BOOST_CHECK(manager.IsSnapshotActive()); BOOST_CHECK(WITH_LOCK(::cs_main, return !manager.IsSnapshotValidated())); BOOST_CHECK_EQUAL(&c2, &manager.ActiveChainstate()); BOOST_CHECK(&c1 != &manager.ActiveChainstate()); auto all2 = manager.GetAll(); BOOST_CHECK_EQUAL_COLLECTIONS(all2.begin(), all2.end(), chainstates.begin(), chainstates.end()); auto &active_chain2 = WITH_LOCK(manager.GetMutex(), return manager.ActiveChain()); BOOST_CHECK_EQUAL(&active_chain2, &c2.m_chain); BOOST_CHECK_EQUAL( WITH_LOCK(manager.GetMutex(), return manager.ActiveHeight()), 0); auto active_tip2 = WITH_LOCK(manager.GetMutex(), return manager.ActiveTip()); auto exp_tip2 = c2.m_chain.Tip(); BOOST_CHECK_EQUAL(active_tip2, exp_tip2); // Ensure that these pointers actually correspond to different // CCoinsViewCache instances. BOOST_CHECK(exp_tip != exp_tip2); // Let scheduler events finish running to avoid accessing memory that is // going to be unloaded SyncWithValidationInterfaceQueue(); } //! Test rebalancing the caches associated with each chainstate. BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) { ChainstateManager &manager = *m_node.chainman; CTxMemPool &mempool = *m_node.mempool; size_t max_cache = 10000; manager.m_total_coinsdb_cache = max_cache; manager.m_total_coinstip_cache = max_cache; std::vector chainstates; // Create a legacy (IBD) chainstate. // Chainstate &c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); { LOCK(::cs_main); c1.InitCoinsCache(1 << 23); BOOST_REQUIRE(c1.LoadGenesisBlock()); c1.CoinsTip().SetBestBlock(BlockHash{InsecureRand256()}); manager.MaybeRebalanceCaches(); } BOOST_CHECK_EQUAL(c1.m_coinstip_cache_size_bytes, max_cache); BOOST_CHECK_EQUAL(c1.m_coinsdb_cache_size_bytes, max_cache); // Create a snapshot-based chainstate. // Chainstate &c2 = WITH_LOCK(cs_main, return manager.ActivateExistingSnapshot( &mempool, BlockHash{GetRandHash()})); chainstates.push_back(&c2); c2.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); { LOCK(::cs_main); c2.InitCoinsCache(1 << 23); BOOST_REQUIRE(c2.LoadGenesisBlock()); c2.CoinsTip().SetBestBlock(BlockHash{InsecureRand256()}); manager.MaybeRebalanceCaches(); } // Since both chainstates are considered to be in initial block download, // the snapshot chainstate should take priority. BOOST_CHECK_CLOSE(c1.m_coinstip_cache_size_bytes, max_cache * 0.05, 1); BOOST_CHECK_CLOSE(c1.m_coinsdb_cache_size_bytes, max_cache * 0.05, 1); BOOST_CHECK_CLOSE(c2.m_coinstip_cache_size_bytes, max_cache * 0.95, 1); BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1); } struct SnapshotTestSetup : TestChain100Setup { // Run with coinsdb on the filesystem to support, e.g., moving invalidated // chainstate dirs to "*_invalid". // // Note that this means the tests run considerably slower than in-memory DB // tests, but we can't otherwise test this functionality since it relies on // destructive filesystem operations. SnapshotTestSetup() : TestChain100Setup{ {}, {}, /*coins_db_in_memory=*/false, /*block_tree_db_in_memory=*/false, } {} std::tuple SetupSnapshot() { ChainstateManager &chainman = *Assert(m_node.chainman); BOOST_CHECK(!chainman.IsSnapshotActive()); { LOCK(::cs_main); BOOST_CHECK(!chainman.IsSnapshotValidated()); BOOST_CHECK(!node::FindSnapshotChainstateDir()); } size_t initial_size; size_t initial_total_coins{100}; // Make some initial assertions about the contents of the chainstate. { LOCK(::cs_main); CCoinsViewCache &ibd_coinscache = chainman.ActiveChainstate().CoinsTip(); initial_size = ibd_coinscache.GetCacheSize(); size_t total_coins{0}; for (CTransactionRef &txn : m_coinbase_txns) { COutPoint op{txn->GetId(), 0}; BOOST_CHECK(ibd_coinscache.HaveCoin(op)); total_coins++; } BOOST_CHECK_EQUAL(total_coins, initial_total_coins); BOOST_CHECK_EQUAL(initial_size, initial_total_coins); } Chainstate &validation_chainstate = chainman.ActiveChainstate(); // Snapshot should refuse to load at this height. BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(this)); BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash); BOOST_CHECK(!chainman.SnapshotBlockhash()); // Mine 10 more blocks, putting at us height 110 where a valid // assumeutxo value can be found. constexpr int snapshot_height = 110; mineBlocks(10); initial_size += 10; initial_total_coins += 10; // Should not load malleated snapshots BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( this, [](AutoFile &auto_infile, SnapshotMetadata &metadata) { // A UTXO is missing but count is correct metadata.m_coins_count -= 1; COutPoint outpoint; Coin coin; auto_infile >> outpoint; auto_infile >> coin; })); BOOST_CHECK(!node::FindSnapshotChainstateDir()); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( this, [](AutoFile &auto_infile, SnapshotMetadata &metadata) { // Coins count is larger than coins in file metadata.m_coins_count += 1; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( this, [](AutoFile &auto_infile, SnapshotMetadata &metadata) { // Coins count is smaller than coins in file metadata.m_coins_count -= 1; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( this, [](AutoFile &auto_infile, SnapshotMetadata &metadata) { // Wrong hash metadata.m_base_blockhash = BlockHash{uint256::ZERO}; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( this, [](AutoFile &auto_infile, SnapshotMetadata &metadata) { // Wrong hash metadata.m_base_blockhash = BlockHash{uint256::ONE}; })); BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(this)); BOOST_CHECK(fs::exists(*node::FindSnapshotChainstateDir())); // Ensure our active chain is the snapshot chainstate. BOOST_CHECK( !chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull()); BOOST_CHECK_EQUAL( *chainman.ActiveChainstate().m_from_snapshot_blockhash, *chainman.SnapshotBlockhash()); Chainstate &snapshot_chainstate = chainman.ActiveChainstate(); { LOCK(::cs_main); fs::path found = *node::FindSnapshotChainstateDir(); // Note: WriteSnapshotBaseBlockhash() is implicitly tested above. BOOST_CHECK_EQUAL(*node::ReadSnapshotBaseBlockhash(found), *chainman.SnapshotBlockhash()); // Ensure that the genesis block was not marked assumed-valid. BOOST_CHECK(!chainman.ActiveChain().Genesis()->IsAssumedValid()); } const AssumeutxoData &au_data = *ExpectedAssumeutxo( snapshot_height, ::GetConfig().GetChainParams()); const CBlockIndex *tip = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip()); BOOST_CHECK_EQUAL(tip->nChainTx, au_data.nChainTx); // To be checked against later when we try loading a subsequent // snapshot. BlockHash loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()}; // Make some assertions about the both chainstates. These checks ensure // the legacy chainstate hasn't changed and that the newly created // chainstate reflects the expected content. { LOCK(::cs_main); int chains_tested{0}; for (Chainstate *chainstate : chainman.GetAll()) { BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); CCoinsViewCache &coinscache = chainstate->CoinsTip(); // Both caches will be empty initially. BOOST_CHECK_EQUAL((unsigned int)0, coinscache.GetCacheSize()); size_t total_coins{0}; for (CTransactionRef &txn : m_coinbase_txns) { COutPoint op{txn->GetId(), 0}; BOOST_CHECK(coinscache.HaveCoin(op)); total_coins++; } BOOST_CHECK_EQUAL(initial_size, coinscache.GetCacheSize()); BOOST_CHECK_EQUAL(total_coins, initial_total_coins); chains_tested++; } BOOST_CHECK_EQUAL(chains_tested, 2); } // Mine some new blocks on top of the activated snapshot chainstate. constexpr size_t new_coins{100}; mineBlocks(new_coins); // Defined in TestChain100Setup. { LOCK(::cs_main); size_t coins_in_active{0}; size_t coins_in_background{0}; size_t coins_missing_from_background{0}; for (Chainstate *chainstate : chainman.GetAll()) { BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); CCoinsViewCache &coinscache = chainstate->CoinsTip(); bool is_background = chainstate != &chainman.ActiveChainstate(); for (CTransactionRef &txn : m_coinbase_txns) { COutPoint op{txn->GetId(), 0}; if (coinscache.HaveCoin(op)) { (is_background ? coins_in_background : coins_in_active)++; } else if (is_background) { coins_missing_from_background++; } } } BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins); BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins); BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins); } // Snapshot should refuse to load after one has already loaded. BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(this)); // Snapshot blockhash should be unchanged. BOOST_CHECK_EQUAL( *chainman.ActiveChainstate().m_from_snapshot_blockhash, loaded_snapshot_blockhash); return std::make_tuple(&validation_chainstate, &snapshot_chainstate); } //! Simulate a restart of the node by flushing all state to disk, clearing //! the existing ChainstateManager, and unloading the block index. //! //! @returns a reference to the "restarted" ChainstateManager ChainstateManager &SimulateNodeRestart() { ChainstateManager &chainman = *Assert(m_node.chainman); BOOST_TEST_MESSAGE("Simulating node restart"); { for (Chainstate *cs : chainman.GetAll()) { LOCK(::cs_main); cs->ForceFlushStateToDisk(); } // Process all callbacks referring to the old manager before wiping // it. SyncWithValidationInterfaceQueue(); LOCK(::cs_main); chainman.ResetChainstates(); BOOST_CHECK_EQUAL(chainman.GetAll().size(), 0); const ChainstateManager::Options chainman_opts{ .config = ::GetConfig(), .adjusted_time_callback = GetAdjustedTime, }; // For robustness, ensure the old manager is destroyed before // creating a new one. m_node.chainman.reset(); m_node.chainman.reset(new ChainstateManager(chainman_opts)); } return *Assert(m_node.chainman); } }; //! Test basic snapshot activation. BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, SnapshotTestSetup) { this->SetupSnapshot(); } //! Test LoadBlockIndex behavior when multiple chainstates are in use. //! //! - First, verfiy that setBlockIndexCandidates is as expected when using a //! single, fully-validating chainstate. //! //! - Then mark a region of the chain BLOCK_ASSUMED_VALID and introduce a second //! chainstate that will tolerate assumed-valid blocks. Run LoadBlockIndex() //! and ensure that the first chainstate only contains fully validated blocks //! and the other chainstate contains all blocks, even those assumed-valid. //! BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) { ChainstateManager &chainman = *Assert(m_node.chainman); CBlockIndex *assumed_tip{ WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip())}; auto reload_all_block_indexes = [&]() { for (Chainstate *cs : chainman.GetAll()) { LOCK(::cs_main); cs->UnloadBlockIndex(); BOOST_CHECK(cs->setBlockIndexCandidates.empty()); } WITH_LOCK(::cs_main, chainman.LoadBlockIndex()); }; // Ensure that without any assumed-valid BlockIndex entries, all entries are // considered tip candidates. reload_all_block_indexes(); Chainstate &cs1 = chainman.ActiveChainstate(); BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), cs1.m_chain.Height() + 1); // Mark some region of the chain assumed-valid. int num_indexes{0}; int num_assumed_valid{0}; const int expected_assumed_valid{20}; const int last_assumed_valid_idx{40}; const int assumed_valid_start_idx = last_assumed_valid_idx - expected_assumed_valid; CBlockIndex *validated_tip{nullptr}; for (int i = 0; i <= cs1.m_chain.Height(); ++i) { LOCK(::cs_main); auto index = cs1.m_chain[i]; if (i < last_assumed_valid_idx && i >= assumed_valid_start_idx) { index->nStatus = BlockStatus() .withValidity(BlockValidity::TREE) .withAssumedValid(); } ++num_indexes; if (index->IsAssumedValid()) { ++num_assumed_valid; } // Note the last fully-validated block as the expected validated tip. if (i == (assumed_valid_start_idx - 1)) { validated_tip = index; BOOST_CHECK(!index->IsAssumedValid()); } } BOOST_CHECK_EQUAL(expected_assumed_valid, num_assumed_valid); Chainstate &cs2 = *WITH_LOCK(::cs_main, return &chainman.ActivateExistingSnapshot( &*m_node.mempool, BlockHash{GetRandHash()})); reload_all_block_indexes(); // The fully validated chain only has candidates up to the start of the // assumed-valid blocks. BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.count(validated_tip), 1); BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.count(assumed_tip), 0); BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), assumed_valid_start_idx); // The assumed-valid tolerant chain has all blocks as candidates. BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(validated_tip), 1); BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(assumed_tip), 1); BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.size(), num_indexes); } //! Ensure that snapshot chainstates initialize properly when found on disk. BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_init, SnapshotTestSetup) { ChainstateManager &chainman = *Assert(m_node.chainman); Chainstate &bg_chainstate = chainman.ActiveChainstate(); this->SetupSnapshot(); fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(); BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot"); BOOST_CHECK(chainman.IsSnapshotActive()); const BlockHash snapshot_tip_hash = WITH_LOCK( chainman.GetMutex(), return chainman.ActiveTip()->GetBlockHash()); auto all_chainstates = chainman.GetAll(); BOOST_CHECK_EQUAL(all_chainstates.size(), 2); // "Rewind" the background chainstate so that its tip is not at the // base block of the snapshot - this is so after simulating a node restart, // it will initialize instead of attempting to complete validation. // // Note that this is not a realistic use of DisconnectTip(). DisconnectedBlockTransactions unused_pool; BlockValidationState unused_state; { LOCK2(::cs_main, bg_chainstate.MempoolMutex()); BOOST_CHECK(bg_chainstate.DisconnectTip(unused_state, &unused_pool)); // to avoid queuedTx assertion errors on teardown unused_pool.clear(); } BOOST_CHECK_EQUAL(bg_chainstate.m_chain.Height(), 109); // Test that simulating a shutdown (resetting ChainstateManager) and then // performing chainstate reinitializing successfully cleans up the // background-validation chainstate data, and we end up with a single // chainstate that is at tip. ChainstateManager &chainman_restarted = this->SimulateNodeRestart(); BOOST_TEST_MESSAGE("Performing Load/Verify/Activate of chainstate"); // This call reinitializes the chainstates. this->LoadVerifyActivateChainstate(::GetConfig()); { LOCK(chainman_restarted.GetMutex()); BOOST_CHECK_EQUAL(chainman_restarted.GetAll().size(), 2); BOOST_CHECK(chainman_restarted.IsSnapshotActive()); BOOST_CHECK(!chainman_restarted.IsSnapshotValidated()); BOOST_CHECK_EQUAL(chainman_restarted.ActiveTip()->GetBlockHash(), snapshot_tip_hash); BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 210); } BOOST_TEST_MESSAGE("Ensure we can mine blocks on top of the initialized " "snapshot chainstate"); mineBlocks(10); { LOCK(chainman_restarted.GetMutex()); BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 220); // Background chainstate should be unaware of new blocks on the snapshot // chainstate. for (const Chainstate *cs : chainman_restarted.GetAll()) { if (cs != &chainman_restarted.ActiveChainstate()) { BOOST_CHECK_EQUAL(cs->m_chain.Height(), 109); } } } } BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_completion, SnapshotTestSetup) { this->SetupSnapshot(); ChainstateManager &chainman = *Assert(m_node.chainman); Chainstate &active_cs = chainman.ActiveChainstate(); auto tip_cache_before_complete = active_cs.m_coinstip_cache_size_bytes; auto db_cache_before_complete = active_cs.m_coinsdb_cache_size_bytes; SnapshotCompletionResult res; auto mock_shutdown = [](bilingual_str msg) {}; fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(); BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot"); BOOST_CHECK(chainman.IsSnapshotActive()); const BlockHash snapshot_tip_hash = WITH_LOCK( chainman.GetMutex(), return chainman.ActiveTip()->GetBlockHash()); res = WITH_LOCK(::cs_main, return chainman.MaybeCompleteSnapshotValidation( mock_shutdown)); BOOST_CHECK_EQUAL(res, SnapshotCompletionResult::SUCCESS); WITH_LOCK(::cs_main, BOOST_CHECK(chainman.IsSnapshotValidated())); BOOST_CHECK(chainman.IsSnapshotActive()); // Cache should have been rebalanced and reallocated to the "only" remaining // chainstate. BOOST_CHECK(active_cs.m_coinstip_cache_size_bytes > tip_cache_before_complete); BOOST_CHECK(active_cs.m_coinsdb_cache_size_bytes > db_cache_before_complete); auto all_chainstates = chainman.GetAll(); BOOST_CHECK_EQUAL(all_chainstates.size(), 1); BOOST_CHECK_EQUAL(all_chainstates[0], &active_cs); // Trying completion again should return false. res = WITH_LOCK(::cs_main, return chainman.MaybeCompleteSnapshotValidation( mock_shutdown)); BOOST_CHECK_EQUAL(res, SnapshotCompletionResult::SKIPPED); // The invalid snapshot path should not have been used. fs::path snapshot_invalid_dir = gArgs.GetDataDirNet() / "chainstate_snapshot_INVALID"; BOOST_CHECK(!fs::exists(snapshot_invalid_dir)); // chainstate_snapshot should still exist. BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); // Test that simulating a shutdown (reseting ChainstateManager) and then // performing chainstate reinitializing successfully cleans up the // background-validation chainstate data, and we end up with a single // chainstate that is at tip. ChainstateManager &chainman_restarted = this->SimulateNodeRestart(); BOOST_TEST_MESSAGE("Performing Load/Verify/Activate of chainstate"); // This call reinitializes the chainstates, and should clean up the now // unnecessary background-validation leveldb contents. this->LoadVerifyActivateChainstate(::GetConfig()); BOOST_CHECK(!fs::exists(snapshot_invalid_dir)); // chainstate_snapshot should now *not* exist. BOOST_CHECK(!fs::exists(snapshot_chainstate_dir)); const Chainstate &active_cs2 = chainman_restarted.ActiveChainstate(); { LOCK(chainman_restarted.GetMutex()); BOOST_CHECK_EQUAL(chainman_restarted.GetAll().size(), 1); BOOST_CHECK(!chainman_restarted.IsSnapshotActive()); BOOST_CHECK(!chainman_restarted.IsSnapshotValidated()); BOOST_CHECK(active_cs2.m_coinstip_cache_size_bytes > tip_cache_before_complete); BOOST_CHECK(active_cs2.m_coinsdb_cache_size_bytes > db_cache_before_complete); BOOST_CHECK_EQUAL(chainman_restarted.ActiveTip()->GetBlockHash(), snapshot_tip_hash); BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 210); } BOOST_TEST_MESSAGE( "Ensure we can mine blocks on top of the \"new\" IBD chainstate"); mineBlocks(10); { LOCK(chainman_restarted.GetMutex()); BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 220); } } BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_completion_hash_mismatch, SnapshotTestSetup) { auto chainstates = this->SetupSnapshot(); Chainstate &validation_chainstate = *std::get<0>(chainstates); ChainstateManager &chainman = *Assert(m_node.chainman); SnapshotCompletionResult res; auto mock_shutdown = [](bilingual_str msg) {}; // Test tampering with the IBD UTXO set with an extra coin to ensure it // causes snapshot completion to fail. CCoinsViewCache &ibd_coins = WITH_LOCK(::cs_main, return validation_chainstate.CoinsTip()); CScript script; script.assign(InsecureRandBits(6), 0); Coin badcoin{CTxOut{int64_t(InsecureRand32()) * SATOSHI, script}, /*nHeightIn=*/1, /*IsCoinbase=*/false}; TxId txid{InsecureRand256()}; ibd_coins.AddCoin(COutPoint(txid, 0), std::move(badcoin), false); fs::path snapshot_chainstate_dir = gArgs.GetDataDirNet() / "chainstate_snapshot"; BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); res = WITH_LOCK(::cs_main, return chainman.MaybeCompleteSnapshotValidation( mock_shutdown)); BOOST_CHECK_EQUAL(res, SnapshotCompletionResult::HASH_MISMATCH); auto all_chainstates = chainman.GetAll(); BOOST_CHECK_EQUAL(all_chainstates.size(), 1); BOOST_CHECK_EQUAL(all_chainstates[0], &validation_chainstate); BOOST_CHECK_EQUAL(&chainman.ActiveChainstate(), &validation_chainstate); fs::path snapshot_invalid_dir = gArgs.GetDataDirNet() / "chainstate_snapshot_INVALID"; BOOST_CHECK(fs::exists(snapshot_invalid_dir)); // Test that simulating a shutdown (reseting ChainstateManager) and then // performing chainstate reinitializing successfully loads only the // fully-validated chainstate data, and we end up with a single chainstate // that is at tip. ChainstateManager &chainman_restarted = this->SimulateNodeRestart(); BOOST_TEST_MESSAGE("Performing Load/Verify/Activate of chainstate"); // This call reinitializes the chainstates, and should clean up the now // unnecessary background-validation leveldb contents. this->LoadVerifyActivateChainstate(::GetConfig()); BOOST_CHECK(fs::exists(snapshot_invalid_dir)); BOOST_CHECK(!fs::exists(snapshot_chainstate_dir)); { LOCK(::cs_main); BOOST_CHECK_EQUAL(chainman_restarted.GetAll().size(), 1); BOOST_CHECK(!chainman_restarted.IsSnapshotActive()); BOOST_CHECK(!chainman_restarted.IsSnapshotValidated()); BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 210); } BOOST_TEST_MESSAGE( "Ensure we can mine blocks on top of the \"new\" IBD chainstate"); mineBlocks(10); { LOCK(::cs_main); BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 220); } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 29845938f..0b7d4da4b 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1,1047 +1,837 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include -#include -#include #include #include // for GetConsensus. #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#include #include #include #include #include #include bool TestLockPointValidity(const CChain &active_chain, const LockPoints &lp) { AssertLockHeld(cs_main); // If there are relative lock times then the maxInputBlock will be set // If there are no relative lock times, the LockPoints don't depend on the // chain if (lp.maxInputBlock) { // Check whether active_chain is an extension of the block at which the // LockPoints calculation was valid. If not LockPoints are no longer // valid if (!active_chain.Contains(lp.maxInputBlock)) { return false; } } // LockPoints still valid return true; } CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef &_tx, const Amount fee, int64_t time, unsigned int entry_height, bool spends_coinbase, int64_t _sigChecks, LockPoints lp) : tx{_tx}, nFee{fee}, nTxSize(tx->GetTotalSize()), nUsageSize{RecursiveDynamicUsage(tx)}, nTime(time), entryHeight{entry_height}, spendsCoinbase(spends_coinbase), sigChecks(_sigChecks), lockPoints(lp) {} size_t CTxMemPoolEntry::GetTxVirtualSize() const { return GetVirtualTransactionSize(nTxSize, sigChecks, ::nBytesPerSigCheck); } void CTxMemPoolEntry::UpdateFeeDelta(Amount newFeeDelta) { feeDelta = newFeeDelta; } void CTxMemPoolEntry::UpdateLockPoints(const LockPoints &lp) { lockPoints = lp; } bool CTxMemPool::CalculateAncestors( setEntries &setAncestors, CTxMemPoolEntry::Parents &staged_ancestors) const { while (!staged_ancestors.empty()) { const auto stage = staged_ancestors.begin()->get(); txiter stageit = mapTx.find(stage->GetTx().GetId()); assert(stageit != mapTx.end()); setAncestors.insert(stageit); staged_ancestors.erase(staged_ancestors.begin()); const CTxMemPoolEntry::Parents &parents = (*stageit)->GetMemPoolParentsConst(); for (const auto &parent : parents) { txiter parent_it = mapTx.find(parent.get()->GetTx().GetId()); assert(parent_it != mapTx.end()); // If this is a new ancestor, add it. if (setAncestors.count(parent_it) == 0) { staged_ancestors.insert(parent); } } } return true; } bool CTxMemPool::CalculateMemPoolAncestors( const CTxMemPoolEntryRef &entry, setEntries &setAncestors, bool fSearchForParents /* = true */) const { CTxMemPoolEntry::Parents staged_ancestors; const CTransaction &tx = entry->GetTx(); if (fSearchForParents) { // Get parents of this transaction that are in the mempool // GetMemPoolParents() is only valid for entries in the mempool, so we // iterate mapTx to find parents. for (const CTxIn &in : tx.vin) { std::optional piter = GetIter(in.prevout.GetTxId()); if (!piter) { continue; } staged_ancestors.insert(**piter); } } else { // If we're not searching for parents, we require this to be an entry in // the mempool already. staged_ancestors = entry->GetMemPoolParentsConst(); } return CalculateAncestors(setAncestors, staged_ancestors); } void CTxMemPool::UpdateParentsOf(bool add, txiter it) { // add or remove this tx as a child of each parent for (const auto &parent : (*it)->GetMemPoolParentsConst()) { auto parent_it = mapTx.find(parent.get()->GetTx().GetId()); assert(parent_it != mapTx.end()); UpdateChild(parent_it, it, add); } } void CTxMemPool::UpdateChildrenForRemoval(txiter it) { const CTxMemPoolEntry::Children &children = (*it)->GetMemPoolChildrenConst(); for (const auto &child : children) { auto updateIt = mapTx.find(child.get()->GetTx().GetId()); assert(updateIt != mapTx.end()); UpdateParent(updateIt, it, false); } } void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove) { for (txiter removeIt : entriesToRemove) { // Note that UpdateParentsOf severs the child links that point to // removeIt in the entries for the parents of removeIt. UpdateParentsOf(false, removeIt); } // After updating all the parent links, we can now sever the link between // each transaction being removed and any mempool children (ie, update // CTxMemPoolEntry::m_parents for each direct child of a transaction being // removed). for (txiter removeIt : entriesToRemove) { UpdateChildrenForRemoval(removeIt); } } CTxMemPool::CTxMemPool(const Options &opts) : m_check_ratio(opts.check_ratio), m_max_size_bytes{opts.max_size_bytes}, m_expiry{opts.expiry}, m_min_relay_feerate{opts.min_relay_feerate}, m_dust_relay_feerate{opts.dust_relay_feerate}, m_permit_bare_multisig{opts.permit_bare_multisig}, m_max_datacarrier_bytes{opts.max_datacarrier_bytes}, m_require_standard{opts.require_standard} { // lock free clear _clear(); } CTxMemPool::~CTxMemPool() {} bool CTxMemPool::isSpent(const COutPoint &outpoint) const { LOCK(cs); return mapNextTx.count(outpoint); } unsigned int CTxMemPool::GetTransactionsUpdated() const { return nTransactionsUpdated; } void CTxMemPool::AddTransactionsUpdated(unsigned int n) { nTransactionsUpdated += n; } void CTxMemPool::addUnchecked(CTxMemPoolEntryRef entry) { // get a guaranteed unique id (in case tests re-use the same object) entry->SetEntryId(nextEntryId++); // Update transaction for any feeDelta created by PrioritiseTransaction { Amount feeDelta = Amount::zero(); ApplyDelta(entry->GetTx().GetId(), feeDelta); entry->UpdateFeeDelta(feeDelta); } // Add to memory pool without checking anything. // Used by AcceptToMemoryPool(), which DOES do all the appropriate checks. auto [newit, inserted] = mapTx.insert(entry); // Sanity check: It is a programming error if insertion fails (uniqueness // invariants in mapTx are violated, etc) assert(inserted); // Sanity check: We should always end up inserting at the end of the // entry_id index assert(&*mapTx.get().rbegin() == &*newit); // Update cachedInnerUsage to include contained transaction's usage. // (When we update the entry for in-mempool parents, memory usage will be // further updated.) cachedInnerUsage += entry->DynamicMemoryUsage(); const CTransaction &tx = entry->GetTx(); std::set setParentTransactions; for (const CTxIn &in : tx.vin) { mapNextTx.insert(std::make_pair(&in.prevout, &tx)); setParentTransactions.insert(in.prevout.GetTxId()); } // Don't bother worrying about child transactions of this one. It is // guaranteed that a new transaction arriving will not have any children, // because such children would be orphans. // Update ancestors with information about this tx for (const auto &pit : GetIterSet(setParentTransactions)) { UpdateParent(newit, pit, true); } UpdateParentsOf(true, newit); nTransactionsUpdated++; totalTxSize += entry->GetTxSize(); m_total_fee += entry->GetFee(); } void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) { // We increment mempool sequence value no matter removal reason // even if not directly reported below. uint64_t mempool_sequence = GetAndIncrementSequence(); if (reason != MemPoolRemovalReason::BLOCK) { // Notify clients that a transaction has been removed from the mempool // for any reason except being included in a block. Clients interested // in transactions included in blocks can subscribe to the // BlockConnected notification. GetMainSignals().TransactionRemovedFromMempool( (*it)->GetSharedTx(), reason, mempool_sequence); } for (const CTxIn &txin : (*it)->GetTx().vin) { mapNextTx.erase(txin.prevout); } /* add logging because unchecked */ const TxId &txid = (*it)->GetTx().GetId(); RemoveUnbroadcastTx(txid, true); finalizedTxs.remove(txid); totalTxSize -= (*it)->GetTxSize(); m_total_fee -= (*it)->GetFee(); cachedInnerUsage -= (*it)->DynamicMemoryUsage(); cachedInnerUsage -= memusage::DynamicUsage((*it)->GetMemPoolParentsConst()) + memusage::DynamicUsage((*it)->GetMemPoolChildrenConst()); mapTx.erase(it); nTransactionsUpdated++; } // Calculates descendants of entry that are not already in setDescendants, and // adds to setDescendants. Assumes entryit is already a tx in the mempool and // CTxMemPoolEntry::m_children is correct for tx and all descendants. Also // assumes that if an entry is in setDescendants already, then all in-mempool // descendants of it are already in setDescendants as well, so that we can save // time by not iterating over those entries. void CTxMemPool::CalculateDescendants(txiter entryit, setEntries &setDescendants) const { setEntries stage; if (setDescendants.count(entryit) == 0) { stage.insert(entryit); } // Traverse down the children of entry, only adding children that are not // accounted for in setDescendants already (because those children have // either already been walked, or will be walked in this iteration). while (!stage.empty()) { txiter it = *stage.begin(); setDescendants.insert(it); stage.erase(stage.begin()); const CTxMemPoolEntry::Children &children = (*it)->GetMemPoolChildrenConst(); for (const auto &child : children) { txiter childiter = mapTx.find(child.get()->GetTx().GetId()); assert(childiter != mapTx.end()); if (!setDescendants.count(childiter)) { stage.insert(childiter); } } } } void CTxMemPool::removeRecursive(const CTransaction &origTx, MemPoolRemovalReason reason) { // Remove transaction from memory pool. AssertLockHeld(cs); setEntries txToRemove; txiter origit = mapTx.find(origTx.GetId()); if (origit != mapTx.end()) { txToRemove.insert(origit); } else { // When recursively removing but origTx isn't in the mempool be sure to // remove any children that are in the pool. This can happen during // chain re-orgs if origTx isn't re-accepted into the mempool for any // reason. auto it = mapNextTx.lower_bound(COutPoint(origTx.GetId(), 0)); while (it != mapNextTx.end() && it->first->GetTxId() == origTx.GetId()) { txiter nextit = mapTx.find(it->second->GetId()); assert(nextit != mapTx.end()); txToRemove.insert(nextit); ++it; } } setEntries setAllRemoves; for (txiter it : txToRemove) { CalculateDescendants(it, setAllRemoves); } RemoveStaged(setAllRemoves, reason); } void CTxMemPool::removeConflicts(const CTransaction &tx) { // Remove transactions which depend on inputs of tx, recursively AssertLockHeld(cs); for (const CTxIn &txin : tx.vin) { auto it = mapNextTx.find(txin.prevout); if (it != mapNextTx.end()) { const CTransaction &txConflict = *it->second; if (txConflict != tx) { ClearPrioritisation(txConflict.GetId()); removeRecursive(txConflict, MemPoolRemovalReason::CONFLICT); } } } } /** * Called when a block is connected. Updates the miner fee estimator. */ void CTxMemPool::updateFeeForBlock() { AssertLockHeld(cs); lastRollingFeeUpdate = GetTime(); blockSinceLastRollingFeeBump = true; } -void DisconnectedBlockTransactions::removeForBlock( - const std::vector &vtx, CTxMemPool &pool) { - AssertLockHeld(pool.cs); - - if (pool.mapTx.empty() && pool.mapDeltas.empty()) { - // fast-path for IBD and/or when mempool is empty; there is no need to - // do any of the set-up work below which eats precious cycles. - // Note that this also skips updating the rolling fee udpate, which is - // fine: it is only recomputed when the mempool has to be trimmed down - // because it is full which is contradictory with this condition. - return; - } - - addForBlock(vtx, pool); - - for (const CTransactionRef &tx : - reverse_iterate(queuedTx.get())) { - CTxMemPool::txiter it = pool.mapTx.find(tx->GetId()); - if (it != pool.mapTx.end()) { - CTxMemPool::setEntries stage; - stage.insert(it); - pool.RemoveStaged(stage, MemPoolRemovalReason::BLOCK); - } else { - // Conflicting txs can only exist if the tx was not in the mempool - pool.removeConflicts(*tx); - } - pool.ClearPrioritisation(tx->GetId()); - } - - pool.updateFeeForBlock(); - - removeForBlock(vtx); -} - void CTxMemPool::_clear() { mapTx.clear(); mapNextTx.clear(); totalTxSize = 0; m_total_fee = Amount::zero(); cachedInnerUsage = 0; lastRollingFeeUpdate = GetTime(); blockSinceLastRollingFeeBump = false; rollingMinimumFeeRate = 0; ++nTransactionsUpdated; } void CTxMemPool::clear() { LOCK(cs); _clear(); } void CTxMemPool::check(const CCoinsViewCache &active_coins_tip, int64_t spendheight) const { if (m_check_ratio == 0) { return; } if (GetRand(m_check_ratio) >= 1) { return; } AssertLockHeld(::cs_main); LOCK(cs); LogPrint(BCLog::MEMPOOL, "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); uint64_t checkTotal = 0; Amount check_total_fee{Amount::zero()}; uint64_t innerUsage = 0; CCoinsViewCache mempoolDuplicate( const_cast(&active_coins_tip)); for (const CTxMemPoolEntryRef &entry : mapTx.get()) { checkTotal += entry->GetTxSize(); check_total_fee += entry->GetFee(); innerUsage += entry->DynamicMemoryUsage(); const CTransaction &tx = entry->GetTx(); innerUsage += memusage::DynamicUsage(entry->GetMemPoolParentsConst()) + memusage::DynamicUsage(entry->GetMemPoolChildrenConst()); CTxMemPoolEntry::Parents setParentCheck; for (const CTxIn &txin : tx.vin) { // Check that every mempool transaction's inputs refer to available // coins, or other mempool tx's. txiter parentIt = mapTx.find(txin.prevout.GetTxId()); if (parentIt != mapTx.end()) { const CTransaction &parentTx = (*parentIt)->GetTx(); assert(parentTx.vout.size() > txin.prevout.GetN() && !parentTx.vout[txin.prevout.GetN()].IsNull()); setParentCheck.insert(*parentIt); // also check that parents have a topological ordering before // their children assert((*parentIt)->GetEntryId() < entry->GetEntryId()); } // We are iterating through the mempool entries sorted // topologically. // All parents must have been checked before their children and // their coins added to the mempoolDuplicate coins cache. assert(mempoolDuplicate.HaveCoin(txin.prevout)); // Check whether its inputs are marked in mapNextTx. auto prevoutNextIt = mapNextTx.find(txin.prevout); assert(prevoutNextIt != mapNextTx.end()); assert(prevoutNextIt->first == &txin.prevout); assert(prevoutNextIt->second == &tx); } auto comp = [](const auto &a, const auto &b) -> bool { return a.get()->GetTx().GetId() == b.get()->GetTx().GetId(); }; assert(setParentCheck.size() == entry->GetMemPoolParentsConst().size()); assert(std::equal(setParentCheck.begin(), setParentCheck.end(), entry->GetMemPoolParentsConst().begin(), comp)); // Verify ancestor state is correct. setEntries setAncestors; std::string dummy; const bool ok = CalculateMemPoolAncestors(entry, setAncestors); assert(ok); // all ancestors should have entryId < this tx's entryId for (const auto &ancestor : setAncestors) { assert((*ancestor)->GetEntryId() < entry->GetEntryId()); } // Check children against mapNextTx CTxMemPoolEntry::Children setChildrenCheck; auto iter = mapNextTx.lower_bound(COutPoint(entry->GetTx().GetId(), 0)); for (; iter != mapNextTx.end() && iter->first->GetTxId() == entry->GetTx().GetId(); ++iter) { txiter childIt = mapTx.find(iter->second->GetId()); // mapNextTx points to in-mempool transactions assert(childIt != mapTx.end()); setChildrenCheck.insert(*childIt); } assert(setChildrenCheck.size() == entry->GetMemPoolChildrenConst().size()); assert(std::equal(setChildrenCheck.begin(), setChildrenCheck.end(), entry->GetMemPoolChildrenConst().begin(), comp)); // Not used. CheckTxInputs() should always pass TxValidationState dummy_state; Amount txfee{Amount::zero()}; assert(!tx.IsCoinBase()); assert(Consensus::CheckTxInputs(tx, dummy_state, mempoolDuplicate, spendheight, txfee)); for (const auto &input : tx.vin) { mempoolDuplicate.SpendCoin(input.prevout); } AddCoins(mempoolDuplicate, tx, std::numeric_limits::max()); } for (auto &[_, nextTx] : mapNextTx) { txiter it = mapTx.find(nextTx->GetId()); assert(it != mapTx.end()); assert(&(*it)->GetTx() == nextTx); } assert(totalTxSize == checkTotal); assert(m_total_fee == check_total_fee); assert(innerUsage == cachedInnerUsage); } bool CTxMemPool::CompareTopologically(const TxId &txida, const TxId &txidb) const { LOCK(cs); auto it1 = mapTx.find(txida); if (it1 == mapTx.end()) { return false; } auto it2 = mapTx.find(txidb); if (it2 == mapTx.end()) { return true; } return (*it1)->GetEntryId() < (*it2)->GetEntryId(); } void CTxMemPool::getAllTxIds(std::vector &vtxid) const { LOCK(cs); vtxid.clear(); vtxid.reserve(mapTx.size()); for (const auto &entry : mapTx.get()) { vtxid.push_back(entry->GetTx().GetId()); } } static TxMempoolInfo GetInfo(CTxMemPool::indexed_transaction_set::const_iterator it) { return TxMempoolInfo{(*it)->GetSharedTx(), (*it)->GetTime(), (*it)->GetFee(), (*it)->GetTxSize(), (*it)->GetModifiedFee() - (*it)->GetFee()}; } std::vector CTxMemPool::infoAll() const { LOCK(cs); std::vector ret; ret.reserve(mapTx.size()); const auto &index = mapTx.get(); for (auto it = index.begin(); it != index.end(); ++it) { ret.push_back(GetInfo(mapTx.project<0>(it))); } return ret; } CTransactionRef CTxMemPool::get(const TxId &txid) const { LOCK(cs); indexed_transaction_set::const_iterator i = mapTx.find(txid); if (i == mapTx.end()) { return nullptr; } return (*i)->GetSharedTx(); } TxMempoolInfo CTxMemPool::info(const TxId &txid) const { LOCK(cs); indexed_transaction_set::const_iterator i = mapTx.find(txid); if (i == mapTx.end()) { return TxMempoolInfo(); } return GetInfo(i); } CFeeRate CTxMemPool::estimateFee() const { LOCK(cs); // minerPolicy uses recent blocks to figure out a reasonable fee. This // may disagree with the rollingMinimumFeerate under certain scenarios // where the mempool increases rapidly, or blocks are being mined which // do not contain propagated transactions. return std::max(m_min_relay_feerate, GetMinFee()); } void CTxMemPool::PrioritiseTransaction(const TxId &txid, const Amount nFeeDelta) { { LOCK(cs); Amount &delta = mapDeltas[txid]; delta += nFeeDelta; txiter it = mapTx.find(txid); if (it != mapTx.end()) { mapTx.modify(it, [&delta](CTxMemPoolEntryRef &e) { e->UpdateFeeDelta(delta); }); ++nTransactionsUpdated; } } LogPrintf("PrioritiseTransaction: %s fee += %s\n", txid.ToString(), FormatMoney(nFeeDelta)); } void CTxMemPool::ApplyDelta(const TxId &txid, Amount &nFeeDelta) const { AssertLockHeld(cs); std::map::const_iterator pos = mapDeltas.find(txid); if (pos == mapDeltas.end()) { return; } nFeeDelta += pos->second; } void CTxMemPool::ClearPrioritisation(const TxId &txid) { AssertLockHeld(cs); mapDeltas.erase(txid); } const CTransaction *CTxMemPool::GetConflictTx(const COutPoint &prevout) const { const auto it = mapNextTx.find(prevout); return it == mapNextTx.end() ? nullptr : it->second; } std::optional CTxMemPool::GetIter(const TxId &txid) const { auto it = mapTx.find(txid); if (it != mapTx.end()) { return it; } return std::nullopt; } CTxMemPool::setEntries CTxMemPool::GetIterSet(const std::set &txids) const { CTxMemPool::setEntries ret; for (const auto &txid : txids) { const auto mi = GetIter(txid); if (mi) { ret.insert(*mi); } } return ret; } bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const { for (const CTxIn &in : tx.vin) { if (exists(in.prevout.GetTxId())) { return false; } } return true; } CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView *baseIn, const CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) {} bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { // Check to see if the inputs are made available by another tx in the // package. These Coins would not be available in the underlying CoinsView. if (auto it = m_temp_added.find(outpoint); it != m_temp_added.end()) { coin = it->second; return true; } // If an entry in the mempool exists, always return that one, as it's // guaranteed to never conflict with the underlying cache, and it cannot // have pruned entries (as it contains full) transactions. First checking // the underlying cache risks returning a pruned entry instead. CTransactionRef ptx = mempool.get(outpoint.GetTxId()); if (ptx) { if (outpoint.GetN() < ptx->vout.size()) { coin = Coin(ptx->vout[outpoint.GetN()], MEMPOOL_HEIGHT, false); return true; } return false; } return base->GetCoin(outpoint, coin); } void CCoinsViewMemPool::PackageAddTransaction(const CTransactionRef &tx) { for (uint32_t n = 0; n < tx->vout.size(); ++n) { m_temp_added.emplace(COutPoint(tx->GetId(), n), Coin(tx->vout[n], MEMPOOL_HEIGHT, false)); } } size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); // Estimate the overhead of mapTx to be 12 pointers + an allocation, as no // exact formula for boost::multi_index_contained is implemented. return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 12 * sizeof(void *)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + cachedInnerUsage; } void CTxMemPool::RemoveUnbroadcastTx(const TxId &txid, const bool unchecked) { LOCK(cs); if (m_unbroadcast_txids.erase(txid)) { LogPrint( BCLog::MEMPOOL, "Removed %i from set of unbroadcast txns%s\n", txid.GetHex(), (unchecked ? " before confirmation that txn was sent out" : "")); } } void CTxMemPool::RemoveStaged(const setEntries &stage, MemPoolRemovalReason reason) { AssertLockHeld(cs); UpdateForRemoveFromMempool(stage); // Remove txs in reverse-topological order const setRevTopoEntries stageRevTopo(stage.begin(), stage.end()); for (txiter it : stageRevTopo) { removeUnchecked(it, reason); } } int CTxMemPool::Expire(std::chrono::seconds time) { AssertLockHeld(cs); indexed_transaction_set::index::type::iterator it = mapTx.get().begin(); setEntries toremove; while (it != mapTx.get().end() && (*it)->GetTime() < time) { toremove.insert(mapTx.project<0>(it)); it++; } setEntries stage; for (txiter removeit : toremove) { CalculateDescendants(removeit, stage); } RemoveStaged(stage, MemPoolRemovalReason::EXPIRY); return stage.size(); } void CTxMemPool::LimitSize(CCoinsViewCache &coins_cache) { AssertLockHeld(::cs_main); AssertLockHeld(cs); int expired = Expire(GetTime() - m_expiry); if (expired != 0) { LogPrint(BCLog::MEMPOOL, "Expired %i transactions from the memory pool\n", expired); } std::vector vNoSpendsRemaining; TrimToSize(m_max_size_bytes, &vNoSpendsRemaining); for (const COutPoint &removed : vNoSpendsRemaining) { coins_cache.Uncache(removed); } } void CTxMemPool::UpdateChild(txiter entry, txiter child, bool add) { AssertLockHeld(cs); CTxMemPoolEntry::Children s; if (add && (*entry)->GetMemPoolChildren().insert(*child).second) { cachedInnerUsage += memusage::IncrementalDynamicUsage(s); } else if (!add && (*entry)->GetMemPoolChildren().erase(*child)) { cachedInnerUsage -= memusage::IncrementalDynamicUsage(s); } } void CTxMemPool::UpdateParent(txiter entry, txiter parent, bool add) { AssertLockHeld(cs); CTxMemPoolEntry::Parents s; if (add && (*entry)->GetMemPoolParents().insert(*parent).second) { cachedInnerUsage += memusage::IncrementalDynamicUsage(s); } else if (!add && (*entry)->GetMemPoolParents().erase(*parent)) { cachedInnerUsage -= memusage::IncrementalDynamicUsage(s); } } CFeeRate CTxMemPool::GetMinFee(size_t sizelimit) const { LOCK(cs); if (!blockSinceLastRollingFeeBump || rollingMinimumFeeRate == 0) { return CFeeRate(int64_t(ceill(rollingMinimumFeeRate)) * SATOSHI); } int64_t time = GetTime(); if (time > lastRollingFeeUpdate + 10) { double halflife = ROLLING_FEE_HALFLIFE; if (DynamicMemoryUsage() < sizelimit / 4) { halflife /= 4; } else if (DynamicMemoryUsage() < sizelimit / 2) { halflife /= 2; } rollingMinimumFeeRate = rollingMinimumFeeRate / pow(2.0, (time - lastRollingFeeUpdate) / halflife); lastRollingFeeUpdate = time; } return CFeeRate(int64_t(ceill(rollingMinimumFeeRate)) * SATOSHI); } void CTxMemPool::trackPackageRemoved(const CFeeRate &rate) { AssertLockHeld(cs); if ((rate.GetFeePerK() / SATOSHI) > rollingMinimumFeeRate) { rollingMinimumFeeRate = rate.GetFeePerK() / SATOSHI; blockSinceLastRollingFeeBump = false; } } void CTxMemPool::TrimToSize(size_t sizelimit, std::vector *pvNoSpendsRemaining) { AssertLockHeld(cs); unsigned nTxnRemoved = 0; CFeeRate maxFeeRateRemoved(Amount::zero()); while (!mapTx.empty() && DynamicMemoryUsage() > sizelimit) { auto it = mapTx.get().end(); --it; // We set the new mempool min fee to the feerate of the removed // transaction, plus the "minimum reasonable fee rate" (ie some value // under which we consider txn to have 0 fee). This way, we don't allow // txn to enter mempool with feerate equal to txn which were removed // with no block in between. CFeeRate removed = (*it)->GetModifiedFeeRate(); removed += MEMPOOL_FULL_FEE_INCREMENT; trackPackageRemoved(removed); maxFeeRateRemoved = std::max(maxFeeRateRemoved, removed); setEntries stage; CalculateDescendants(mapTx.project<0>(it), stage); nTxnRemoved += stage.size(); if (pvNoSpendsRemaining) { for (const txiter &iter : stage) { for (const CTxIn &txin : (*iter)->GetTx().vin) { if (!exists(txin.prevout.GetTxId())) { pvNoSpendsRemaining->push_back(txin.prevout); } } } } RemoveStaged(stage, MemPoolRemovalReason::SIZELIMIT); } if (maxFeeRateRemoved > CFeeRate(Amount::zero())) { LogPrint(BCLog::MEMPOOL, "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); } } bool CTxMemPool::GetLoadTried() const { LOCK(cs); return m_load_tried; } void CTxMemPool::SetLoadTried(bool load_tried) { LOCK(cs); m_load_tried = load_tried; } - -/** Maximum bytes for transactions to store for processing during reorg */ -static const size_t MAX_DISCONNECTED_TX_POOL_SIZE = 20 * DEFAULT_MAX_BLOCK_SIZE; - -void DisconnectedBlockTransactions::addForBlock( - const std::vector &vtx, CTxMemPool &pool) { - AssertLockHeld(pool.cs); - for (const auto &tx : reverse_iterate(vtx)) { - // If we already added it, just skip. - auto it = queuedTx.find(tx->GetId()); - if (it != queuedTx.end()) { - continue; - } - - // Insert the transaction into the pool. - addTransaction(tx); - - // Fill in the set of parents. - std::unordered_set parents; - for (const CTxIn &in : tx->vin) { - parents.insert(in.prevout.GetTxId()); - } - - // In order to make sure we keep things in topological order, we check - // if we already know of the parent of the current transaction. If so, - // we remove them from the set and then add them back. - while (parents.size() > 0) { - std::unordered_set worklist( - std::move(parents)); - parents.clear(); - - for (const TxId &txid : worklist) { - // If we do not have that txid in the set, nothing needs to be - // done. - auto pit = queuedTx.find(txid); - if (pit == queuedTx.end()) { - continue; - } - - // We have parent in our set, we reinsert them at the right - // position. - const CTransactionRef ptx = *pit; - queuedTx.erase(pit); - queuedTx.insert(ptx); - - // And we make sure ancestors are covered. - for (const CTxIn &in : ptx->vin) { - parents.insert(in.prevout.GetTxId()); - } - } - } - } - - // Keep the size under control. - while (DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE) { - // Drop the earliest entry, and remove its children from the - // mempool. - auto it = queuedTx.get().begin(); - pool.removeRecursive(**it, MemPoolRemovalReason::REORG); - removeEntry(it); - } -} - -void DisconnectedBlockTransactions::importMempool(CTxMemPool &pool) { - AssertLockHeld(pool.cs); - // addForBlock's algorithm sorts a vector of transactions back into - // topological order. We use it in a separate object to create a valid - // ordering of all mempool transactions, which we then splice in front of - // the current queuedTx. This results in a valid sequence of transactions to - // be reprocessed in updateMempoolForReorg. - - // We create vtx in order of the entry_id index to facilitate for - // addForBlocks (which iterates in reverse order), as vtx probably end in - // the correct ordering for queuedTx. - std::vector vtx; - - vtx.reserve(pool.mapTx.size()); - txInfo.reserve(pool.mapTx.size()); - for (const CTxMemPoolEntryRef &e : pool.mapTx.get()) { - vtx.push_back(e->GetSharedTx()); - // save entry time, feeDelta, and height for use in - // updateMempoolForReorg() - txInfo.try_emplace(e->GetTx().GetId(), e->GetTime(), - e->GetModifiedFee() - e->GetFee(), e->GetHeight()); - } - for (const CTxMemPoolEntryRef &e : - reverse_iterate(pool.mapTx.get())) { - // Notify all observers of this (possibly temporary) removal. This is - // necessary for tracking the transactions that are removed from the - // mempool during a reorg and can't be added back due to missing parent. - // Transactions that are added back to the mempool will trigger another - // notification. Make sure to notify in reverse topological order, - // children first. - GetMainSignals().TransactionRemovedFromMempool( - e->GetSharedTx(), MemPoolRemovalReason::REORG, - pool.GetAndIncrementSequence()); - } - pool.clear(); - - // Use addForBlocks to sort the transactions and then splice them in front - // of queuedTx - DisconnectedBlockTransactions orderedTxnPool; - orderedTxnPool.addForBlock(vtx, pool); - cachedInnerUsage += orderedTxnPool.cachedInnerUsage; - queuedTx.get().splice( - queuedTx.get().begin(), - orderedTxnPool.queuedTx.get()); - - // We limit memory usage because we can't know if more blocks will be - // disconnected - while (DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE) { - // Drop the earliest entry which, by definition, has no children - removeEntry(queuedTx.get().begin()); - } -} - -auto DisconnectedBlockTransactions::getTxInfo(const CTransactionRef &tx) const - -> const TxInfo * { - if (auto it = txInfo.find(tx->GetId()); it != txInfo.end()) { - return &it->second; - } - - return nullptr; -} - -void DisconnectedBlockTransactions::updateMempoolForReorg( - const Config &config, Chainstate &active_chainstate, bool fAddToMempool, - CTxMemPool &pool) { - AssertLockHeld(cs_main); - AssertLockHeld(pool.cs); - - if (fAddToMempool) { - // disconnectpool's insertion_order index sorts the entries from oldest - // to newest, but the oldest entry will be the last tx from the latest - // mined block that was disconnected. - // Iterate disconnectpool in reverse, so that we add transactions back - // to the mempool starting with the earliest transaction that had been - // previously seen in a block. - for (const CTransactionRef &tx : - reverse_iterate(queuedTx.get())) { - if (tx->IsCoinBase()) { - continue; - } - // restore saved PrioritiseTransaction state and nAcceptTime - const auto ptxInfo = getTxInfo(tx); - bool hasFeeDelta = false; - if (ptxInfo && ptxInfo->feeDelta != Amount::zero()) { - // manipulate mapDeltas directly (faster than calling - // PrioritiseTransaction) - pool.mapDeltas[tx->GetId()] = ptxInfo->feeDelta; - hasFeeDelta = true; - } - // ignore validation errors in resurrected transactions - auto result = AcceptToMemoryPool( - config, active_chainstate, tx, - /*accept_time=*/ptxInfo ? ptxInfo->time.count() : GetTime(), - /*bypass_limits=*/true, /*test_accept=*/false, - /*heightOverride=*/ptxInfo ? ptxInfo->height : 0); - if (result.m_result_type != - MempoolAcceptResult::ResultType::VALID && - hasFeeDelta) { - // tx not accepted: undo mapDelta insertion from above - pool.mapDeltas.erase(tx->GetId()); - } - } - } - - queuedTx.clear(); - txInfo.clear(); - - // Re-limit mempool size, in case we added any transactions - pool.LimitSize(active_chainstate.CoinsTip()); -} diff --git a/src/txmempool.h b/src/txmempool.h index 619880cd2..5772db163 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -1,905 +1,756 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_TXMEMPOOL_H #define BITCOIN_TXMEMPOOL_H #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include class CBlockIndex; class CChain; class Chainstate; class Config; extern RecursiveMutex cs_main; /** * Fake height value used in Coins to signify they are only in the memory * pool(since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; struct LockPoints { // Will be set to the blockchain height and median time past values that // would be necessary to satisfy all relative locktime constraints (BIP68) // of this tx given our view of block chain history int height{0}; int64_t time{0}; // As long as the current chain descends from the highest height block // containing one of the inputs used in the calculation, then the cached // values are still valid even after a reorg. CBlockIndex *maxInputBlock{nullptr}; }; /** * Test whether the LockPoints height and time are still valid on the current * chain. */ bool TestLockPointValidity(const CChain &active_chain, const LockPoints &lp) EXCLUSIVE_LOCKS_REQUIRED(cs_main); struct CompareIteratorById { // SFINAE for T where T is either a std::reference_wrapper (e.g. a // CTxMemPoolEntryRef) or an iterator to a pointer type (e.g., a txiter) template bool operator()(const std::reference_wrapper &a, const std::reference_wrapper &b) const { return a.get()->GetTx().GetId() < b.get()->GetTx().GetId(); } template bool operator()(const T &a, const T &b) const { return (*a)->GetTx().GetId() < (*b)->GetTx().GetId(); } }; /** Iterate txs in reverse-topological order */ struct CompareIteratorByRevEntryId { template bool operator()(const std::reference_wrapper &a, const std::reference_wrapper &b) const { return a.get()->GetEntryId() > b.get()->GetEntryId(); } template bool operator()(const T &a, const T &b) const { return (*a)->GetEntryId() > (*b)->GetEntryId(); } }; class CTxMemPoolEntry; using CTxMemPoolEntryRef = RCUPtr; /** \class CTxMemPoolEntry * * CTxMemPoolEntry stores data about the corresponding transaction, as well as * data about all in-mempool transactions that depend on the transaction * ("descendant" transactions). */ class CTxMemPoolEntry { public: // two aliases, should the types ever diverge typedef std::set, CompareIteratorById> Parents; typedef std::set, CompareIteratorById> Children; private: //! Unique identifier -- used for topological sorting uint64_t entryId = 0; const CTransactionRef tx; mutable Parents m_parents; mutable Children m_children; //! Cached to avoid expensive parent-transaction lookups const Amount nFee; //! ... and avoid recomputing tx size const size_t nTxSize; //! ... and total memory usage const size_t nUsageSize; //! Local time when entering the mempool const int64_t nTime; //! Chain height when entering the mempool const unsigned int entryHeight; //! keep track of transactions that spend a coinbase const bool spendsCoinbase; //! Total sigChecks const int64_t sigChecks; //! Used for determining the priority of the transaction for mining in a //! block Amount feeDelta{Amount::zero()}; //! Track the height and time at which tx was final LockPoints lockPoints; IMPLEMENT_RCU_REFCOUNT(uint64_t); public: CTxMemPoolEntry(const CTransactionRef &_tx, const Amount fee, int64_t time, unsigned int entry_height, bool spends_coinbase, int64_t sigchecks, LockPoints lp); CTxMemPoolEntry(const CTxMemPoolEntry &other) = delete; CTxMemPoolEntry(CTxMemPoolEntry &&other) : entryId(other.entryId), tx(std::move(other.tx)), m_parents(std::move(other.m_parents)), m_children(std::move(other.m_children)), nFee(other.nFee), nTxSize(other.nTxSize), nUsageSize(other.nUsageSize), nTime(other.nTime), entryHeight(other.entryHeight), spendsCoinbase(other.spendsCoinbase), sigChecks(other.sigChecks), feeDelta(other.feeDelta), lockPoints(std::move(other.lockPoints)), refcount(other.refcount.load()){}; uint64_t GetEntryId() const { return entryId; } //! This should only be set by addUnchecked() before entry insertion into //! mempool void SetEntryId(uint64_t eid) { entryId = eid; } const CTransaction &GetTx() const { return *this->tx; } CTransactionRef GetSharedTx() const { return this->tx; } Amount GetFee() const { return nFee; } size_t GetTxSize() const { return nTxSize; } size_t GetTxVirtualSize() const; std::chrono::seconds GetTime() const { return std::chrono::seconds{nTime}; } unsigned int GetHeight() const { return entryHeight; } int64_t GetSigChecks() const { return sigChecks; } Amount GetModifiedFee() const { return nFee + feeDelta; } CFeeRate GetModifiedFeeRate() const { return CFeeRate(GetModifiedFee(), GetTxVirtualSize()); } size_t DynamicMemoryUsage() const { return nUsageSize; } const LockPoints &GetLockPoints() const { return lockPoints; } // Updates the fee delta used for mining priority score void UpdateFeeDelta(Amount feeDelta); // Update the LockPoints after a reorg void UpdateLockPoints(const LockPoints &lp); bool GetSpendsCoinbase() const { return spendsCoinbase; } const Parents &GetMemPoolParentsConst() const { return m_parents; } const Children &GetMemPoolChildrenConst() const { return m_children; } Parents &GetMemPoolParents() const { return m_parents; } Children &GetMemPoolChildren() const { return m_children; } }; // extracts a transaction id from CTxMemPoolEntry or CTransactionRef struct mempoolentry_txid { typedef TxId result_type; result_type operator()(const CTxMemPoolEntryRef &entry) const { return entry->GetTx().GetId(); } result_type operator()(const CTransactionRef &tx) const { return tx->GetId(); } }; /** * Radix tree adapter for storing a CTxMemPoolEntry as a tree element. */ struct MemPoolEntryRadixTreeAdapter { Uint256RadixKey getId(const CTxMemPoolEntry &entry) const { return entry.GetSharedTx()->GetId(); } }; // used by the entry_time index struct CompareTxMemPoolEntryByEntryTime { bool operator()(const CTxMemPoolEntryRef &a, const CTxMemPoolEntryRef &b) const { return a->GetTime() < b->GetTime(); } }; // used by the entry_id index struct CompareTxMemPoolEntryByEntryId { bool operator()(const CTxMemPoolEntryRef &a, const CTxMemPoolEntryRef &b) const { return a->GetEntryId() < b->GetEntryId(); } }; /** * \class CompareTxMemPoolEntryByModifiedFeeRate * * Sort by feerate of entry (modfee/vsize) in descending order. * This is used by the block assembler (mining). */ struct CompareTxMemPoolEntryByModifiedFeeRate { // Used in tests bool operator()(const CTxMemPoolEntry &a, const CTxMemPoolEntry &b) const { const CFeeRate frA = a.GetModifiedFeeRate(); const CFeeRate frB = b.GetModifiedFeeRate(); // Sort by modified fee rate first if (frA != frB) { return frA > frB; } // Ties are broken by whichever is topologically earlier // (this helps mining code avoid some backtracking). if (a.GetEntryId() != b.GetEntryId()) { return a.GetEntryId() < b.GetEntryId(); } // If nothing else, sort by txid (this should never happen as entryID is // expected to be unique). return a.GetSharedTx()->GetId() < b.GetSharedTx()->GetId(); } bool operator()(const CTxMemPoolEntryRef &a, const CTxMemPoolEntryRef &b) const { return operator()(*a, *b); } }; // Multi_index tag names struct entry_time {}; struct modified_feerate {}; struct entry_id {}; /** * Information about a mempool transaction. */ struct TxMempoolInfo { /** The transaction itself */ CTransactionRef tx; /** Time the transaction entered the mempool. */ std::chrono::seconds m_time; /** Fee of the transaction. */ Amount fee; /** Virtual size of the transaction. */ size_t vsize; /** The fee delta. */ Amount nFeeDelta; }; /** * Reason why a transaction was removed from the mempool, this is passed to the * notification signal. */ enum class MemPoolRemovalReason { //! Expired from mempool EXPIRY, //! Removed in size limiting SIZELIMIT, //! Removed for reorganization REORG, //! Removed for block BLOCK, //! Removed for conflict with in-block transaction CONFLICT, //! Removed for replacement REPLACED, //! Removed by avalanche vote AVALANCHE, }; /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions that * may be included in the next block. * * Transactions are added when they are seen on the network (or created by the * local node), but not all transactions seen are added to the pool. For * example, the following new transactions will not be added to the mempool: * - a transaction which doesn't meet the minimum fee requirements. * - a new transaction that double-spends an input of a transaction already in * the pool. * - a non-standard transaction. * * CTxMemPool::mapTx, and CTxMemPoolEntry bookkeeping: * * mapTx is a boost::multi_index that sorts the mempool on 3 criteria: * - transaction hash * - time in mempool * - entry id (this is a topological index) * * Note: the term "descendant" refers to in-mempool transactions that depend on * this one, while "ancestor" refers to in-mempool transactions that a given * transaction depends on. * * When a new transaction is added to the mempool, it has no in-mempool children * (because any such children would be an orphan). So in addUnchecked(), we: * - update a new entry's setMemPoolParents to include all in-mempool parents * - update the new entry's direct parents to include the new tx as a child * * When a transaction is removed from the mempool, we must: * - update all in-mempool parents to not track the tx in setMemPoolChildren * - update all in-mempool children to not include it as a parent * * These happen in UpdateForRemoveFromMempool(). (Note that when removing a * transaction along with its descendants, we must calculate that set of * transactions to be removed before doing the removal, or else the mempool can * be in an inconsistent state where it's impossible to walk the ancestors of a * transaction.) * * In the event of a reorg, the invariant that all newly-added tx's have no * in-mempool children must be maintained. On top of this, we use a topological * index (GetEntryId). As such, we always dump mempool tx's into a disconnect * pool on reorg, and simply add them one by one, along with tx's from * disconnected blocks, when the reorg is complete. */ class CTxMemPool { private: //! Value n means that 1 times in n we check. const int m_check_ratio; //! Used by getblocktemplate to trigger CreateNewBlock() invocation std::atomic nTransactionsUpdated{0}; //! sum of all mempool tx's sizes. uint64_t totalTxSize GUARDED_BY(cs); //! sum of all mempool tx's fees (NOT modified fee) Amount m_total_fee GUARDED_BY(cs); //! sum of dynamic memory usage of all the map elements (NOT the maps //! themselves) uint64_t cachedInnerUsage GUARDED_BY(cs); mutable int64_t lastRollingFeeUpdate GUARDED_BY(cs); mutable bool blockSinceLastRollingFeeBump GUARDED_BY(cs); //! minimum fee to get into the pool, decreases exponentially mutable double rollingMinimumFeeRate GUARDED_BY(cs); // In-memory counter for external mempool tracking purposes. // This number is incremented once every time a transaction // is added or removed from the mempool for any reason. mutable uint64_t m_sequence_number GUARDED_BY(cs){1}; void trackPackageRemoved(const CFeeRate &rate) EXCLUSIVE_LOCKS_REQUIRED(cs); bool m_load_tried GUARDED_BY(cs){false}; //! Used by addUnchecked to generate ever-increasing //! CTxMemPoolEntry::entryId's uint64_t nextEntryId GUARDED_BY(cs) = 1; public: // public only for testing static const int ROLLING_FEE_HALFLIFE = 60 * 60 * 12; typedef boost::multi_index_container< CTxMemPoolEntryRef, boost::multi_index::indexed_by< // indexed by txid boost::multi_index::hashed_unique, // sorted by fee rate boost::multi_index::ordered_non_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByModifiedFeeRate>, // sorted by entry time boost::multi_index::ordered_non_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByEntryTime>, // sorted topologically (insertion order) boost::multi_index::ordered_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByEntryId>>> indexed_transaction_set; /** * This mutex needs to be locked when accessing `mapTx` or other members * that are guarded by it. * * @par Consistency guarantees * * By design, it is guaranteed that: * * 1. Locking both `cs_main` and `mempool.cs` will give a view of mempool * that is consistent with current chain tip (`ActiveChain()` and * `CoinsTip()`) and is fully populated. Fully populated means that if * the current active chain is missing transactions that were present in * a previously active chain, all the missing transactions will have been * re-added to the mempool and should be present if they meet size and * consistency constraints. * * 2. Locking `mempool.cs` without `cs_main` will give a view of a mempool * consistent with some chain that was active since `cs_main` was last * locked, and that is fully populated as described above. It is ok for * code that only needs to query or remove transactions from the mempool * to lock just `mempool.cs` without `cs_main`. * * To provide these guarantees, it is necessary to lock both `cs_main` and * `mempool.cs` whenever adding transactions to the mempool and whenever * changing the chain tip. It's necessary to keep both mutexes locked until * the mempool is consistent with the new chain tip and fully populated. */ mutable RecursiveMutex cs; indexed_transaction_set mapTx GUARDED_BY(cs); using txiter = indexed_transaction_set::nth_index<0>::type::const_iterator; typedef std::set setEntries; typedef std::set setRevTopoEntries; RadixTree finalizedTxs; private: void UpdateParent(txiter entry, txiter parent, bool add) EXCLUSIVE_LOCKS_REQUIRED(cs); void UpdateChild(txiter entry, txiter child, bool add) EXCLUSIVE_LOCKS_REQUIRED(cs); /** * Track locally submitted transactions to periodically retry initial * broadcast */ std::set m_unbroadcast_txids GUARDED_BY(cs); /** * Helper function to calculate all in-mempool ancestors of staged_ancestors * param@[in] staged_ancestors Should contain entries in the mempool. * param@[out] setAncestors Will be populated with all mempool * ancestors. */ bool CalculateAncestors(setEntries &setAncestors, CTxMemPoolEntry::Parents &staged_ancestors) const EXCLUSIVE_LOCKS_REQUIRED(cs); public: indirectmap mapNextTx GUARDED_BY(cs); std::map mapDeltas GUARDED_BY(cs); using Options = kernel::MemPoolOptions; const int64_t m_max_size_bytes; const std::chrono::seconds m_expiry; const CFeeRate m_min_relay_feerate; const CFeeRate m_dust_relay_feerate; const bool m_permit_bare_multisig; const std::optional m_max_datacarrier_bytes; const bool m_require_standard; /** * Create a new CTxMemPool. * Sanity checks will be off by default for performance, because otherwise * accepting transactions becomes O(N^2) where N is the number of * transactions in the pool. */ CTxMemPool(const Options &opts); ~CTxMemPool(); /** * If sanity-checking is turned on, check makes sure the pool is consistent * (does not contain two transactions that spend the same inputs, all inputs * are in the mapNextTx array). If sanity-checking is turned off, check does * nothing. */ void check(const CCoinsViewCache &active_coins_tip, int64_t spendheight) const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); // addUnchecked must update state for all parents of a given transaction, // updating child links as necessary. void addUnchecked(CTxMemPoolEntryRef entry) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); void removeRecursive(const CTransaction &tx, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs); void removeConflicts(const CTransaction &tx) EXCLUSIVE_LOCKS_REQUIRED(cs); void updateFeeForBlock() EXCLUSIVE_LOCKS_REQUIRED(cs); void clear(); // lock free void _clear() EXCLUSIVE_LOCKS_REQUIRED(cs); bool CompareTopologically(const TxId &txida, const TxId &txidb) const; void getAllTxIds(std::vector &vtxid) const; bool isSpent(const COutPoint &outpoint) const; unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); /** * Check that none of this transactions inputs are in the mempool, and thus * the tx is not dependent on other mempool transactions to be included in a * block. */ bool HasNoInputsOf(const CTransaction &tx) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** Affect CreateNewBlock prioritisation of transactions */ void PrioritiseTransaction(const TxId &txid, const Amount nFeeDelta); void ApplyDelta(const TxId &txid, Amount &nFeeDelta) const EXCLUSIVE_LOCKS_REQUIRED(cs); void ClearPrioritisation(const TxId &txid) EXCLUSIVE_LOCKS_REQUIRED(cs); /** Get the transaction in the pool that spends the same prevout */ const CTransaction *GetConflictTx(const COutPoint &prevout) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** Returns an iterator to the given txid, if found */ std::optional GetIter(const TxId &txid) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** * Translate a set of txids into a set of pool iterators to avoid repeated * lookups. */ setEntries GetIterSet(const std::set &txids) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** * Remove a set of transactions from the mempool. If a transaction is in * this set, then all in-mempool descendants must also be in the set, unless * this transaction is being removed for being in a block. */ void RemoveStaged(const setEntries &stage, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs); /** * Try to calculate all in-mempool ancestors of entry. * (these are all calculated including the tx itself) * fSearchForParents = whether to search a tx's vin for in-mempool parents, * or look up parents from m_parents. Must be true for entries not in the * mempool */ bool CalculateMemPoolAncestors(const CTxMemPoolEntryRef &entry, setEntries &setAncestors, bool fSearchForParents = true) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** * Populate setDescendants with all in-mempool descendants of hash. * Assumes that setDescendants includes all in-mempool descendants of * anything already in it. */ void CalculateDescendants(txiter it, setEntries &setDescendants) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** * The minimum fee to get into the mempool, which may itself not be enough * for larger-sized transactions. */ CFeeRate GetMinFee() const { return GetMinFee(m_max_size_bytes); } CFeeRate GetMinFee(size_t sizelimit) const; /** * Remove transactions from the mempool until its dynamic size is <= * sizelimit. pvNoSpendsRemaining, if set, will be populated with the list * of outpoints which are not in mempool which no longer have any spends in * this mempool. */ void TrimToSize(size_t sizelimit, std::vector *pvNoSpendsRemaining = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); /** * Expire all transaction (and their dependencies) in the mempool older than * time. Return the number of removed transactions. */ int Expire(std::chrono::seconds time) EXCLUSIVE_LOCKS_REQUIRED(cs); /** * Reduce the size of the mempool by expiring and then trimming the mempool. */ void LimitSize(CCoinsViewCache &coins_cache) EXCLUSIVE_LOCKS_REQUIRED(cs, ::cs_main); /** * @returns true if we've made an attempt to load the mempool regardless of * whether the attempt was successful or not */ bool GetLoadTried() const; /** * Set whether or not we've made an attempt to load the mempool (regardless * of whether the attempt was successful or not) */ void SetLoadTried(bool load_tried); unsigned long size() const { LOCK(cs); return mapTx.size(); } uint64_t GetTotalTxSize() const EXCLUSIVE_LOCKS_REQUIRED(cs) { AssertLockHeld(cs); return totalTxSize; } Amount GetTotalFee() const EXCLUSIVE_LOCKS_REQUIRED(cs) { AssertLockHeld(cs); return m_total_fee; } bool exists(const TxId &txid) const { LOCK(cs); return mapTx.count(txid) != 0; } bool setAvalancheFinalized(const CTxMemPoolEntryRef &tx) EXCLUSIVE_LOCKS_REQUIRED(cs) { return finalizedTxs.insert(tx); } bool isAvalancheFinalized(const TxId &txid) const { LOCK(cs); return finalizedTxs.get(txid) != nullptr; } CTransactionRef get(const TxId &txid) const; TxMempoolInfo info(const TxId &txid) const; std::vector infoAll() const; CFeeRate estimateFee() const; size_t DynamicMemoryUsage() const; /** Adds a transaction to the unbroadcast set */ void AddUnbroadcastTx(const TxId &txid) { LOCK(cs); // Sanity check the transaction is in the mempool & insert into // unbroadcast set. if (exists(txid)) { m_unbroadcast_txids.insert(txid); } } /** Removes a transaction from the unbroadcast set */ void RemoveUnbroadcastTx(const TxId &txid, const bool unchecked = false); /** Returns transactions in unbroadcast set */ std::set GetUnbroadcastTxs() const { LOCK(cs); return m_unbroadcast_txids; } /** Returns whether a txid is in the unbroadcast set */ bool IsUnbroadcastTx(const TxId &txid) const EXCLUSIVE_LOCKS_REQUIRED(cs) { AssertLockHeld(cs); return (m_unbroadcast_txids.count(txid) != 0); } /** Guards this internal counter for external reporting */ uint64_t GetAndIncrementSequence() const EXCLUSIVE_LOCKS_REQUIRED(cs) { return m_sequence_number++; } uint64_t GetSequence() const EXCLUSIVE_LOCKS_REQUIRED(cs) { return m_sequence_number; } private: /** Set ancestor state for an entry */ void UpdateEntryForAncestors(txiter it, const setEntries *setAncestors) EXCLUSIVE_LOCKS_REQUIRED(cs); /** * Update parents of `it` to add/remove it as a child transaction. */ void UpdateParentsOf(bool add, txiter it) EXCLUSIVE_LOCKS_REQUIRED(cs); /** * For each transaction being removed, update ancestors and any direct * children. */ void UpdateForRemoveFromMempool(const setEntries &entriesToRemove) EXCLUSIVE_LOCKS_REQUIRED(cs); /** Sever link between specified transaction and direct children. */ void UpdateChildrenForRemoval(txiter entry) EXCLUSIVE_LOCKS_REQUIRED(cs); /** * Before calling removeUnchecked for a given transaction, * UpdateForRemoveFromMempool must be called on the entire (dependent) set * of transactions being removed at the same time. We use each * CTxMemPoolEntry's setMemPoolParents in order to walk ancestors of a given * transaction that is removed, so we can't remove intermediate transactions * in a chain before we've updated all the state for the removal. */ void removeUnchecked(txiter entry, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs); }; /** * CCoinsView that brings transactions from a mempool into view. * It does not check for spendings by memory pool transactions. * Instead, it provides access to all Coins which are either unspent in the * base CCoinsView, are outputs from any mempool transaction, or are * tracked temporarily to allow transaction dependencies in package validation. * This allows transaction replacement to work as expected, as you want to * have all inputs "available" to check signatures, and any cycles in the * dependency graph are checked directly in AcceptToMemoryPool. * It also allows you to sign a double-spend directly in * signrawtransactionwithkey and signrawtransactionwithwallet, * as long as the conflicting transaction is not yet confirmed. */ class CCoinsViewMemPool : public CCoinsViewBacked { /** * Coins made available by transactions being validated. Tracking these * allows for package validation, since we can access transaction outputs * without submitting them to mempool. */ std::unordered_map m_temp_added; protected: const CTxMemPool &mempool; public: CCoinsViewMemPool(CCoinsView *baseIn, const CTxMemPool &mempoolIn); bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; /** * Add the coins created by this transaction. These coins are only * temporarily stored in m_temp_added and cannot be flushed to the back end. * Only used for package validation. */ void PackageAddTransaction(const CTransactionRef &tx); }; -/** - * DisconnectedBlockTransactions - * - * During the reorg, it's desirable to re-add previously confirmed transactions - * to the mempool, so that anything not re-confirmed in the new chain is - * available to be mined. However, it's more efficient to wait until the reorg - * is complete and process all still-unconfirmed transactions at that time, - * since we expect most confirmed transactions to (typically) still be - * confirmed in the new chain, and re-accepting to the memory pool is expensive - * (and therefore better to not do in the middle of reorg-processing). - * Instead, store the disconnected transactions (in order!) as we go, remove any - * that are included in blocks in the new chain, and then process the remaining - * still-unconfirmed transactions at the end. - * - * It also enables efficient reprocessing of current mempool entries, useful - * when (de)activating forks that result in in-mempool transactions becoming - * invalid - */ -// multi_index tag names -struct txid_index {}; -struct insertion_order {}; - -class DisconnectedBlockTransactions { -private: - typedef boost::multi_index_container< - CTransactionRef, boost::multi_index::indexed_by< - // hashed by txid - boost::multi_index::hashed_unique< - boost::multi_index::tag, - mempoolentry_txid, SaltedTxIdHasher>, - // sorted by order in the blockchain - boost::multi_index::sequenced< - boost::multi_index::tag>>> - indexed_disconnected_transactions; - - indexed_disconnected_transactions queuedTx; - uint64_t cachedInnerUsage = 0; - - struct TxInfo { - const std::chrono::seconds time; - const Amount feeDelta; - const unsigned height; - TxInfo(const std::chrono::seconds &time_, Amount feeDelta_, - unsigned height_) noexcept - : time(time_), feeDelta(feeDelta_), height(height_) {} - }; - - using TxInfoMap = std::unordered_map; - /// populated by importMempool(); the original tx entry times and feeDeltas - TxInfoMap txInfo; - - void addTransaction(const CTransactionRef &tx) { - queuedTx.insert(tx); - cachedInnerUsage += RecursiveDynamicUsage(tx); - } - - /// @returns a pointer into the txInfo map if tx->GetId() exists in the map, - /// or nullptr otherwise. Note that the returned pointer is only valid for - /// as long as its underlying map node is valid. - const TxInfo *getTxInfo(const CTransactionRef &tx) const; - -public: - // It's almost certainly a logic bug if we don't clear out queuedTx before - // destruction, as we add to it while disconnecting blocks, and then we - // need to re-process remaining transactions to ensure mempool consistency. - // For now, assert() that we've emptied out this object on destruction. - // This assert() can always be removed if the reorg-processing code were - // to be refactored such that this assumption is no longer true (for - // instance if there was some other way we cleaned up the mempool after a - // reorg, besides draining this object). - ~DisconnectedBlockTransactions() { assert(queuedTx.empty()); } - - // Estimate the overhead of queuedTx to be 6 pointers + an allocation, as - // no exact formula for boost::multi_index_contained is implemented. - size_t DynamicMemoryUsage() const { - return memusage::MallocUsage(sizeof(CTransactionRef) + - 6 * sizeof(void *)) * - queuedTx.size() + - memusage::DynamicUsage(txInfo) + cachedInnerUsage; - } - - const indexed_disconnected_transactions &GetQueuedTx() const { - return queuedTx; - } - - // Import mempool entries in topological order into queuedTx and clear the - // mempool. Caller should call updateMempoolForReorg to reprocess these - // transactions - void importMempool(CTxMemPool &pool) EXCLUSIVE_LOCKS_REQUIRED(pool.cs); - - // Add entries for a block while reconstructing the topological ordering so - // they can be added back to the mempool simply. - void addForBlock(const std::vector &vtx, CTxMemPool &pool) - EXCLUSIVE_LOCKS_REQUIRED(pool.cs); - - // Remove entries based on txid_index, and update memory usage. - void removeForBlock(const std::vector &vtx) { - // Short-circuit in the common case of a block being added to the tip - if (queuedTx.empty()) { - return; - } - for (auto const &tx : vtx) { - auto it = queuedTx.find(tx->GetId()); - if (it != queuedTx.end()) { - cachedInnerUsage -= RecursiveDynamicUsage(*it); - queuedTx.erase(it); - txInfo.erase(tx->GetId()); - } - } - } - - void removeForBlock(const std::vector &vtx, - CTxMemPool &pool) EXCLUSIVE_LOCKS_REQUIRED(pool.cs); - - // Remove an entry by insertion_order index, and update memory usage. - void removeEntry(indexed_disconnected_transactions::index< - insertion_order>::type::iterator entry) { - cachedInnerUsage -= RecursiveDynamicUsage(*entry); - txInfo.erase((*entry)->GetId()); - queuedTx.get().erase(entry); - } - - bool isEmpty() const { return queuedTx.empty(); } - - void clear() { - cachedInnerUsage = 0; - queuedTx.clear(); - txInfo.clear(); - } - - /** - * Make mempool consistent after a reorg, by re-adding or recursively - * erasing disconnected block transactions from the mempool, and also - * removing any other transactions from the mempool that are no longer valid - * given the new tip/height. - * - * Note: we assume that disconnectpool only contains transactions that are - * NOT confirmed in the current chain nor already in the mempool (otherwise, - * in-mempool descendants of such transactions would be removed). - * - * Passing fAddToMempool=false will skip trying to add the transactions - * back, and instead just erase from the mempool as needed. - */ - void updateMempoolForReorg(const Config &config, - Chainstate &active_chainstate, - bool fAddToMempool, CTxMemPool &pool) - EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs); -}; - #endif // BITCOIN_TXMEMPOOL_H diff --git a/src/validation.cpp b/src/validation.cpp index ade07dd9b..d1fc3ea03 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1,6595 +1,6596 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2018 The Bitcoin Core developers // Copyright (c) 2017-2020 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include