diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index de53b467b..c4d0f70a1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,716 +1,716 @@ # Copyright (c) 2017 The Bitcoin developers project(bitcoind) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) # Default visibility is hidden on all targets. set(CMAKE_C_VISIBILITY_PRESET hidden) set(CMAKE_CXX_VISIBILITY_PRESET hidden) # Supported Networks set(NETWORK_COMPATIBILITY "ABC" CACHE STRING "Network that will be supported: BCHA or BCHN") option(BUILD_BITCOIN_WALLET "Activate the wallet functionality" ON) option(BUILD_BITCOIN_ZMQ "Activate the ZeroMQ functionalities" ON) option(BUILD_BITCOIN_CLI "Build bitcoin-cli" ON) option(BUILD_BITCOIN_TX "Build bitcoin-tx" ON) option(BUILD_BITCOIN_QT "Build bitcoin-qt" ON) option(BUILD_BITCOIN_SEEDER "Build bitcoin-seeder" ON) option(BUILD_LIBBITCOINCONSENSUS "Build the bitcoinconsenus shared library" ON) option(ENABLE_BIP70 "Enable BIP70 (payment protocol) support in GUI" ON) option(ENABLE_HARDENING "Harden the executables" ON) option(ENABLE_REDUCE_EXPORTS "Reduce the amount of exported symbols" OFF) option(ENABLE_STATIC_LIBSTDCXX "Statically link libstdc++" OFF) option(ENABLE_GLIBC_BACK_COMPAT "Enable Glibc compatibility features" OFF) option(ENABLE_QRCODE "Enable QR code display" ON) option(ENABLE_UPNP "Enable UPnP support" ON) option(START_WITH_UPNP "Make UPnP the default to map ports" OFF) option(ENABLE_CLANG_TIDY "Enable clang-tidy checks for Bitcoin ABC" OFF) option(ENABLE_PROFILING "Select the profiling tool to use" OFF) option(USE_LD_GOLD "Try to use gold as a linker if available" ON) set(OS_WITH_JEMALLOC_AS_SYSTEM_DEFAULT "Android" "FreeBSD" "NetBSD" ) if(NOT CMAKE_SYSTEM_NAME IN_LIST OS_WITH_JEMALLOC_AS_SYSTEM_DEFAULT) set(USE_JEMALLOC_DEFAULT ON) endif() # FIXME: Building against jemalloc causes the software to segfault on OSX. # See https://github.com/Bitcoin-ABC/bitcoin-abc/issues/401 if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin" AND NOT CMAKE_CROSSCOMPILING) set(USE_JEMALLOC_DEFAULT OFF) endif() option(USE_JEMALLOC "Use jemalloc as an allocation library" ${USE_JEMALLOC_DEFAULT}) if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") set(DEFAULT_ENABLE_DBUS_NOTIFICATIONS ON) endif() option(ENABLE_DBUS_NOTIFICATIONS "Enable DBus desktop notifications. Linux only." ${DEFAULT_ENABLE_DBUS_NOTIFICATIONS}) # If ccache is available, then use it. find_program(CCACHE ccache) if(CCACHE) message(STATUS "Using ccache: ${CCACHE}") set(CMAKE_C_COMPILER_LAUNCHER ${CCACHE}) set(CMAKE_CXX_COMPILER_LAUNCHER ${CCACHE}) endif(CCACHE) # Disable what we do not need for the native build. include(NativeExecutable) native_add_cmake_flags( "-DBUILD_BITCOIN_WALLET=OFF" "-DBUILD_BITCOIN_QT=OFF" "-DBUILD_BITCOIN_ZMQ=OFF" "-DENABLE_QRCODE=OFF" "-DENABLE_UPNP=OFF" "-DUSE_JEMALLOC=OFF" "-DENABLE_CLANG_TIDY=OFF" "-DENABLE_BIP70=OFF" ) if(ENABLE_CLANG_TIDY) include(ClangTidy) endif() if(ENABLE_SANITIZERS) include(Sanitizers) enable_sanitizers(${ENABLE_SANITIZERS}) endif() include(AddCompilerFlags) if(USE_LD_GOLD) add_linker_flags(-fuse-ld=gold) endif() # Prefer -g3, defaults to -g if unavailable foreach(LANGUAGE C CXX) set(COMPILER_DEBUG_LEVEL -g) check_compiler_flags(G3_IS_SUPPORTED ${LANGUAGE} -g3) if(${G3_IS_SUPPORTED}) set(COMPILER_DEBUG_LEVEL -g3) endif() add_compile_options_to_configuration_for_language(Debug ${LANGUAGE} ${COMPILER_DEBUG_LEVEL}) endforeach() # Define the debugging symbols DEBUG and DEBUG_LOCKORDER when the Debug build # type is selected. add_compile_definitions_to_configuration(Debug DEBUG DEBUG_LOCKORDER) # Add -ftrapv when building in Debug add_compile_options_to_configuration(Debug -ftrapv) # All versions of gcc that we commonly use for building are subject to bug # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90348. To work around that, set # -fstack-reuse=none for all gcc builds. (Only gcc understands this flag) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") add_compiler_flags(-fstack-reuse=none) endif() if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") # Ensure that WINDRES_PREPROC is enabled when using windres. list(APPEND CMAKE_RC_FLAGS "-DWINDRES_PREPROC") # Build all static so there is no dll file to distribute. add_linker_flags(-static) add_compile_definitions( # Windows 7 _WIN32_WINNT=0x0601 # Internet Explorer 5.01 (!) _WIN32_IE=0x0501 # Define WIN32_LEAN_AND_MEAN to exclude APIs such as Cryptography, DDE, # RPC, Shell, and Windows Sockets. WIN32_LEAN_AND_MEAN ) endif() if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") add_compile_definitions(MAC_OSX OBJC_OLD_DISPATCH_PROTOTYPES=0) add_linker_flags(-Wl,-dead_strip_dylibs) endif() if(ENABLE_REDUCE_EXPORTS) # Default visibility is set by CMAKE__VISIBILITY_PRESET, but this # doesn't tell if the visibility set is effective. # Check if the flag -fvisibility=hidden is supported, as using the hidden # visibility is a requirement to reduce exports. check_compiler_flags(HAS_CXX_FVISIBILITY CXX -fvisibility=hidden) if(NOT HAS_CXX_FVISIBILITY) message(FATAL_ERROR "Cannot set default symbol visibility. Use -DENABLE_REDUCE_EXPORTS=OFF.") endif() # Also hide symbols from static libraries add_linker_flags(-Wl,--exclude-libs,libstdc++) endif() # Enable statically linking libstdc++ if(ENABLE_STATIC_LIBSTDCXX) add_linker_flags(-static-libstdc++) endif() set(CMAKE_POSITION_INDEPENDENT_CODE ON) if(ENABLE_HARDENING) # Enable stack protection add_cxx_compiler_flags(-fstack-protector-all -Wstack-protector) # Enable some buffer overflow checking, except in -O0 builds which # do not support them add_compiler_flags(-U_FORTIFY_SOURCE) add_compile_options($<$>:-D_FORTIFY_SOURCE=2>) # Enable ASLR (these flags are primarily targeting MinGw) add_linker_flags(-Wl,--dynamicbase -Wl,--nxcompat -Wl,--high-entropy-va) # Make the relocated sections read-only add_linker_flags(-Wl,-z,relro -Wl,-z,now) # CMake provides the POSITION_INDEPENDENT_CODE property to set PIC/PIE. cmake_policy(SET CMP0083 NEW) include(CheckPIESupported) check_pie_supported() if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") # MinGw provides its own libssp for stack smashing protection link_libraries(ssp) endif() endif() if(ENABLE_PROFILING MATCHES "gprof") message(STATUS "Enable profiling with gprof") # -pg is incompatible with -pie. Since hardening and profiling together # doesn't make sense, we simply make them mutually exclusive here. # Additionally, hardened toolchains may force -pie by default, in which # case it needs to be turned off with -no-pie. if(ENABLE_HARDENING) message(FATAL_ERROR "Profiling with gprof requires disabling hardening with -DENABLE_HARDENING=OFF.") endif() add_linker_flags(-no-pie) add_compiler_flags(-pg) add_linker_flags(-pg) endif() # Enable warning add_c_compiler_flags(-Wnested-externs -Wstrict-prototypes) add_compiler_flags( -Wall -Wextra -Wformat -Wvla -Wcast-align -Wunused-parameter -Wmissing-braces -Wthread-safety -Wshadow -Wshadow-field -Wrange-loop-analysis -Wredundant-decls -Wunreachable-code-loop-increment ) add_compiler_flag_group(-Wformat -Wformat-security) add_cxx_compiler_flags( -Wredundant-move ) option(EXTRA_WARNINGS "Enable extra warnings" OFF) if(EXTRA_WARNINGS) add_cxx_compiler_flags(-Wsuggest-override) else() add_compiler_flags(-Wno-unused-parameter) add_compiler_flags(-Wno-implicit-fallthrough) endif() # libtool style configure add_subdirectory(config) # Enable LFS (Large File Support) on targets that don't have it natively. # This should be defined before the libraries are included as leveldb need the # definition to be set. if(NOT HAVE_LARGE_FILE_SUPPORT) add_compile_definitions(_FILE_OFFSET_BITS=64) add_linker_flags(-Wl,--large-address-aware) endif() if(ENABLE_GLIBC_BACK_COMPAT) # Wrap some glibc functions with ours add_linker_flags(-Wl,--wrap=__divmoddi4) add_linker_flags(-Wl,--wrap=log2f) if(NOT HAVE_LARGE_FILE_SUPPORT) add_linker_flags(-Wl,--wrap=fcntl -Wl,--wrap=fcntl64) endif() endif() if(USE_JEMALLOC) # Most of the sanitizers require their instrumented allocation functions to # be fully functional. This is obviously the case for all the memory related # sanitizers (asan, lsan, msan) but not only. if(ENABLE_SANITIZERS) message(WARNING "Jemalloc is incompatible with the sanitizers and has been disabled.") else() find_package(Jemalloc 3.6.0 REQUIRED) link_libraries(Jemalloc::jemalloc) endif() endif() # Make sure that all the global compiler and linker flags are set BEFORE # including the libraries so they apply as needed. # libraries add_subdirectory(crypto) add_subdirectory(leveldb) add_subdirectory(secp256k1) add_subdirectory(univalue) # Find the git root, and returns the full path to the .git/logs/HEAD file if # it exists. function(find_git_head_logs_file RESULT) find_package(Git) if(GIT_FOUND) execute_process( COMMAND "${GIT_EXECUTABLE}" "rev-parse" "--show-toplevel" WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_ROOT RESULT_VARIABLE GIT_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET ) if(GIT_RESULT EQUAL 0) set(GIT_LOGS_DIR "${GIT_ROOT}/.git/logs") set(GIT_HEAD_LOGS_FILE "${GIT_LOGS_DIR}/HEAD") # If the .git/logs/HEAD does not exist, create it if(NOT EXISTS "${GIT_HEAD_LOGS_FILE}") file(MAKE_DIRECTORY "${GIT_LOGS_DIR}") file(TOUCH "${GIT_HEAD_LOGS_FILE}") endif() set(${RESULT} "${GIT_HEAD_LOGS_FILE}" PARENT_SCOPE) endif() endif() endfunction() find_git_head_logs_file(GIT_HEAD_LOGS_FILE) set(OBJ_DIR "${CMAKE_CURRENT_BINARY_DIR}/obj") file(MAKE_DIRECTORY "${OBJ_DIR}") set(BUILD_HEADER "${OBJ_DIR}/build.h") set(BUILD_HEADER_TMP "${BUILD_HEADER}.tmp") add_custom_command( DEPENDS "${GIT_HEAD_LOGS_FILE}" "${CMAKE_SOURCE_DIR}/share/genbuild.sh" OUTPUT "${BUILD_HEADER}" COMMAND "${CMAKE_SOURCE_DIR}/share/genbuild.sh" "${BUILD_HEADER_TMP}" "${CMAKE_SOURCE_DIR}" COMMAND ${CMAKE_COMMAND} -E copy_if_different "${BUILD_HEADER_TMP}" "${BUILD_HEADER}" COMMAND ${CMAKE_COMMAND} -E remove "${BUILD_HEADER_TMP}" ) # Because the Bitcoin ABc source code is disorganised, we # end up with a bunch of libraries without any apparent # cohesive structure. This is inherited from Bitcoin Core # and reflecting this. # TODO: Improve the structure once cmake is rocking. # Various completely unrelated features shared by all executables. add_library(util chainparamsbase.cpp clientversion.cpp compat/glibcxx_sanity.cpp compat/strnlen.cpp fs.cpp interfaces/handler.cpp logging.cpp random.cpp randomenv.cpp rcu.cpp rpc/request.cpp blockdb.cpp support/cleanse.cpp support/lockedpool.cpp sync.cpp threadinterrupt.cpp uint256.cpp util/asmap.cpp util/bip32.cpp util/bytevectorhash.cpp util/error.cpp util/message.cpp util/moneystr.cpp util/settings.cpp util/spanparsing.cpp util/strencodings.cpp util/string.cpp util/system.cpp util/threadnames.cpp util/time.cpp util/url.cpp # obj/build.h "${BUILD_HEADER}" ) target_compile_definitions(util PUBLIC HAVE_CONFIG_H HAVE_BUILD_INFO) target_include_directories(util PUBLIC . # To access the config/ and obj/ directories ${CMAKE_CURRENT_BINARY_DIR} ) if(ENABLE_GLIBC_BACK_COMPAT) target_sources(util PRIVATE compat/glibc_compat.cpp) endif() # Target specific configs if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") set(Boost_USE_STATIC_LIBS ON) set(Boost_USE_STATIC_RUNTIME ON) set(Boost_THREADAPI win32) find_package(SHLWAPI REQUIRED) target_link_libraries(util SHLWAPI::shlwapi) find_library(WS2_32_LIBRARY NAMES ws2_32) target_link_libraries(util ${WS2_32_LIBRARY}) target_compile_definitions(util PUBLIC BOOST_THREAD_USE_LIB) endif() target_link_libraries(util univalue crypto) macro(link_event TARGET) non_native_target_link_libraries(${TARGET} Event 2.0.22 ${ARGN}) endmacro() link_event(util event) macro(link_boost TARGET) non_native_target_link_libraries(${TARGET} Boost 1.59 ${ARGN}) endmacro() link_boost(util filesystem thread) # Make sure boost uses std::atomic (it doesn't before 1.63) target_compile_definitions(util PUBLIC BOOST_SP_USE_STD_ATOMIC BOOST_AC_USE_STD_ATOMIC) function(add_network_sources NETWORK_SOURCES) if(${NETWORK_COMPATIBILITY} MATCHES "ABC|BCHA") set(NETWORK_DIR abc) elseif(${NETWORK_COMPATIBILITY} MATCHES "BCHN") set(NETWORK_DIR bchn) else() message(FATAL "${NETWORK_COMPATIBILITY} is not a supported network") endif() list(TRANSFORM ARGN PREPEND "networks/${NETWORK_DIR}/" OUTPUT_VARIABLE NETWORK_SOURCES ) set(NETWORK_SOURCES ${NETWORK_SOURCES} PARENT_SCOPE) endfunction() add_network_sources(NETWORK_SOURCES checkpoints.cpp network.cpp chainparamsconstants.cpp ) # More completely unrelated features shared by all executables. # Because nothing says this is different from util than "common" add_library(common amount.cpp base58.cpp bloom.cpp cashaddr.cpp cashaddrenc.cpp chainparams.cpp chainparamsconstants.cpp config.cpp consensus/merkle.cpp coins.cpp compressor.cpp eventloop.cpp feerate.cpp core_read.cpp core_write.cpp key.cpp key_io.cpp merkleblock.cpp net_permissions.cpp netaddress.cpp netbase.cpp outputtype.cpp policy/policy.cpp primitives/block.cpp protocol.cpp psbt.cpp rpc/rawtransaction_util.cpp rpc/util.cpp scheduler.cpp salteduint256hasher.cpp versionbitsinfo.cpp warnings.cpp ${NETWORK_SOURCES} ) target_link_libraries(common bitcoinconsensus util secp256k1 script) # script library add_library(script script/bitfield.cpp script/descriptor.cpp script/interpreter.cpp script/script.cpp script/script_error.cpp script/sigencoding.cpp script/sign.cpp script/signingprovider.cpp script/standard.cpp ) target_link_libraries(script common) # libbitcoinconsensus add_library(bitcoinconsensus arith_uint256.cpp hash.cpp primitives/transaction.cpp pubkey.cpp uint256.cpp util/strencodings.cpp consensus/tx_check.cpp ) target_link_libraries(bitcoinconsensus script) include(InstallationHelper) if(BUILD_LIBBITCOINCONSENSUS) target_compile_definitions(bitcoinconsensus PUBLIC BUILD_BITCOIN_INTERNAL HAVE_CONSENSUS_LIB ) install_shared_library(bitcoinconsensus script/bitcoinconsensus.cpp PUBLIC_HEADER script/bitcoinconsensus.h ) endif() # Bitcoin server facilities add_library(server addrdb.cpp addrman.cpp avalanche/delegation.cpp avalanche/delegationbuilder.cpp avalanche/peermanager.cpp avalanche/processor.cpp avalanche/proof.cpp avalanche/proofbuilder.cpp banman.cpp blockencodings.cpp blockfilter.cpp blockindex.cpp chain.cpp checkpoints.cpp config.cpp consensus/activation.cpp consensus/tx_verify.cpp dbwrapper.cpp flatfile.cpp httprpc.cpp httpserver.cpp index/base.cpp index/blockfilterindex.cpp index/txindex.cpp init.cpp interfaces/chain.cpp interfaces/node.cpp miner.cpp minerfund.cpp net.cpp net_processing.cpp node/coin.cpp node/coinstats.cpp node/context.cpp node/psbt.cpp node/transaction.cpp + node/ui_interface.cpp noui.cpp policy/fees.cpp policy/settings.cpp pow/aserti32d.cpp pow/daa.cpp pow/eda.cpp pow/grasberg.cpp pow/pow.cpp rest.cpp rpc/abc.cpp rpc/avalanche.cpp rpc/blockchain.cpp rpc/command.cpp rpc/mining.cpp rpc/misc.cpp rpc/net.cpp rpc/rawtransaction.cpp rpc/server.cpp script/scriptcache.cpp script/sigcache.cpp shutdown.cpp timedata.cpp torcontrol.cpp txdb.cpp txmempool.cpp - ui_interface.cpp validation.cpp validationinterface.cpp versionbits.cpp ) target_include_directories(server PRIVATE leveldb/helpers/memenv) target_link_libraries(server bitcoinconsensus leveldb memenv ) link_event(server event) if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") link_event(server pthreads) endif() if(ENABLE_UPNP) find_package(MiniUPnPc 1.9 REQUIRED) target_link_libraries(server MiniUPnPc::miniupnpc) if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") # TODO: check if we are really using a static library. Assume this is # the one from the depends for now since the native windows build is not # supported. target_compile_definitions(server PUBLIC -DSTATICLIB PUBLIC -DMINIUPNP_STATICLIB ) endif() endif() # Test suites. add_subdirectory(test) add_subdirectory(avalanche/test) add_subdirectory(pow/test) # Benchmark suite. add_subdirectory(bench) include(BinaryTest) include(WindowsVersionInfo) # Wallet if(BUILD_BITCOIN_WALLET) add_subdirectory(wallet) target_link_libraries(server wallet) # bitcoin-wallet add_executable(bitcoin-wallet bitcoin-wallet.cpp) generate_windows_version_info(bitcoin-wallet DESCRIPTION "CLI tool for ${PACKAGE_NAME} wallets" ) target_link_libraries(bitcoin-wallet wallet-tool common util) add_to_symbols_check(bitcoin-wallet) add_to_security_check(bitcoin-wallet) install_target(bitcoin-wallet) install_manpages(bitcoin-wallet) else() target_sources(server PRIVATE dummywallet.cpp) endif() # ZeroMQ if(BUILD_BITCOIN_ZMQ) add_subdirectory(zmq) target_link_libraries(server zmq) endif() # RPC client support add_library(rpcclient compat/stdin.cpp rpc/client.cpp ) target_link_libraries(rpcclient univalue util) # bitcoin-seeder if(BUILD_BITCOIN_SEEDER) add_subdirectory(seeder) endif() # bitcoin-cli if(BUILD_BITCOIN_CLI) add_executable(bitcoin-cli bitcoin-cli.cpp) generate_windows_version_info(bitcoin-cli DESCRIPTION "JSON-RPC client for ${PACKAGE_NAME}" ) target_link_libraries(bitcoin-cli common rpcclient) link_event(bitcoin-cli event) add_to_symbols_check(bitcoin-cli) add_to_security_check(bitcoin-cli) install_target(bitcoin-cli) install_manpages(bitcoin-cli) endif() # bitcoin-tx if(BUILD_BITCOIN_TX) add_executable(bitcoin-tx bitcoin-tx.cpp) generate_windows_version_info(bitcoin-tx DESCRIPTION "CLI Bitcoin transaction editor utility" ) target_link_libraries(bitcoin-tx bitcoinconsensus) add_to_symbols_check(bitcoin-tx) add_to_security_check(bitcoin-tx) install_target(bitcoin-tx) install_manpages(bitcoin-tx) endif() # bitcoind add_executable(bitcoind bitcoind.cpp) target_link_libraries(bitcoind server) generate_windows_version_info(bitcoind DESCRIPTION "Bitcoin node with a JSON-RPC server" ) add_to_symbols_check(bitcoind) add_to_security_check(bitcoind) install_target(bitcoind) install_manpages(bitcoind) # Bitcoin-qt if(BUILD_BITCOIN_QT) add_subdirectory(qt) endif() diff --git a/src/banman.cpp b/src/banman.cpp index feb63945a..f5246d110 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -1,230 +1,230 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include -#include +#include #include #include #include BanMan::BanMan(fs::path ban_file, const CChainParams &chainparams, CClientUIInterface *client_interface, int64_t default_ban_time) : m_client_interface(client_interface), m_ban_db(std::move(ban_file), chainparams), m_default_ban_time(default_ban_time) { if (m_client_interface) { m_client_interface->InitMessage(_("Loading banlist...").translated); } int64_t n_start = GetTimeMillis(); m_is_dirty = false; banmap_t banmap; if (m_ban_db.Read(banmap)) { // thread save setter SetBanned(banmap); // no need to write down, just read data SetBannedSetDirty(false); // sweep out unused entries SweepBanned(); LogPrint(BCLog::NET, "Loaded %d banned node ips/subnets from banlist.dat %dms\n", banmap.size(), GetTimeMillis() - n_start); } else { LogPrintf("Invalid or missing banlist.dat; recreating\n"); // force write SetBannedSetDirty(true); DumpBanlist(); } } BanMan::~BanMan() { DumpBanlist(); } void BanMan::DumpBanlist() { // clean unused entries (if bantime has expired) SweepBanned(); if (!BannedSetIsDirty()) { return; } int64_t n_start = GetTimeMillis(); banmap_t banmap; GetBanned(banmap); if (m_ban_db.Write(banmap)) { SetBannedSetDirty(false); } LogPrint(BCLog::NET, "Flushed %d banned node ips/subnets to banlist.dat %dms\n", banmap.size(), GetTimeMillis() - n_start); } void BanMan::ClearBanned() { { LOCK(m_cs_banned); m_discouraged.reset(); m_banned.clear(); m_is_dirty = true; } // store banlist to disk DumpBanlist(); if (m_client_interface) { m_client_interface->BannedListChanged(); } } bool BanMan::IsDiscouraged(const CNetAddr &net_addr) { LOCK(m_cs_banned); return m_discouraged.contains(net_addr.GetAddrBytes()); } bool BanMan::IsBanned(const CNetAddr &net_addr) { auto current_time = GetTime(); LOCK(m_cs_banned); for (const auto &it : m_banned) { CSubNet sub_net = it.first; CBanEntry ban_entry = it.second; if (current_time < ban_entry.nBanUntil && sub_net.Match(net_addr)) { return true; } } return false; } bool BanMan::IsBanned(const CSubNet &sub_net) { auto current_time = GetTime(); LOCK(m_cs_banned); banmap_t::iterator i = m_banned.find(sub_net); if (i != m_banned.end()) { CBanEntry ban_entry = (*i).second; if (current_time < ban_entry.nBanUntil) { return true; } } return false; } void BanMan::Ban(const CNetAddr &net_addr, int64_t ban_time_offset, bool since_unix_epoch) { CSubNet sub_net(net_addr); Ban(sub_net, ban_time_offset, since_unix_epoch); } void BanMan::Discourage(const CNetAddr &net_addr) { LOCK(m_cs_banned); m_discouraged.insert(net_addr.GetAddrBytes()); } void BanMan::Ban(const CSubNet &sub_net, int64_t ban_time_offset, bool since_unix_epoch) { CBanEntry ban_entry(GetTime()); int64_t normalized_ban_time_offset = ban_time_offset; bool normalized_since_unix_epoch = since_unix_epoch; if (ban_time_offset <= 0) { normalized_ban_time_offset = m_default_ban_time; normalized_since_unix_epoch = false; } ban_entry.nBanUntil = (normalized_since_unix_epoch ? 0 : GetTime()) + normalized_ban_time_offset; { LOCK(m_cs_banned); if (m_banned[sub_net].nBanUntil < ban_entry.nBanUntil) { m_banned[sub_net] = ban_entry; m_is_dirty = true; } else { return; } } if (m_client_interface) { m_client_interface->BannedListChanged(); } // store banlist to disk immediately DumpBanlist(); } bool BanMan::Unban(const CNetAddr &net_addr) { CSubNet sub_net(net_addr); return Unban(sub_net); } bool BanMan::Unban(const CSubNet &sub_net) { { LOCK(m_cs_banned); if (m_banned.erase(sub_net) == 0) { return false; } m_is_dirty = true; } if (m_client_interface) { m_client_interface->BannedListChanged(); } // store banlist to disk immediately DumpBanlist(); return true; } void BanMan::GetBanned(banmap_t &banmap) { LOCK(m_cs_banned); // Sweep the banlist so expired bans are not returned SweepBanned(); // create a thread safe copy banmap = m_banned; } void BanMan::SetBanned(const banmap_t &banmap) { LOCK(m_cs_banned); m_banned = banmap; m_is_dirty = true; } void BanMan::SweepBanned() { int64_t now = GetTime(); bool notify_ui = false; { LOCK(m_cs_banned); banmap_t::iterator it = m_banned.begin(); while (it != m_banned.end()) { CSubNet sub_net = (*it).first; CBanEntry ban_entry = (*it).second; if (now > ban_entry.nBanUntil) { m_banned.erase(it++); m_is_dirty = true; notify_ui = true; LogPrint( BCLog::NET, "%s: Removed banned node ip/subnet from banlist.dat: %s\n", __func__, sub_net.ToString()); } else { ++it; } } } // update UI if (notify_ui && m_client_interface) { m_client_interface->BannedListChanged(); } } bool BanMan::BannedSetIsDirty() { LOCK(m_cs_banned); return m_is_dirty; } void BanMan::SetBannedSetDirty(bool dirty) { // reuse m_banned lock for the m_is_dirty flag LOCK(m_cs_banned); m_is_dirty = dirty; } diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 4dbd72a46..896a91459 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -1,211 +1,211 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) #include #endif #include #include #include #include #include #include #include #include #include +#include #include #include -#include #include #include #include #include #include #include const std::function G_TRANSLATION_FUN = nullptr; static void WaitForShutdown(NodeContext &node) { while (!ShutdownRequested()) { UninterruptibleSleep(std::chrono::milliseconds{200}); } Interrupt(node); } ////////////////////////////////////////////////////////////////////////////// // // Start // static bool AppInit(int argc, char *argv[]) { // FIXME: Ideally, we'd like to build the config here, but that's currently // not possible as the whole application has too many global state. However, // this is a first step. auto &config = const_cast(GetConfig()); RPCServer rpcServer; NodeContext node; util::Ref context{node}; HTTPRPCRequestProcessor httpRPCRequestProcessor(config, rpcServer, context); bool fRet = false; util::ThreadSetInternalName("init"); // // Parameters // // If Qt is used, parameters/bitcoin.conf are parsed in qt/bitcoin.cpp's // main() SetupServerArgs(node); std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { return InitError(Untranslated( strprintf("Error parsing command line arguments: %s\n", error))); } // Process help and version before taking care about datadir if (HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { std::string strUsage = PACKAGE_NAME " version " + FormatFullVersion() + " (" + NETWORK_NAME + " network)\n"; if (gArgs.IsArgSet("-version")) { strUsage += FormatParagraph(LicenseInfo()) + "\n"; } else { strUsage += "\nUsage: bitcoind [options] " "Start " PACKAGE_NAME "\n"; strUsage += "\n" + gArgs.GetHelpMessage(); } tfm::format(std::cout, "%s", strUsage); return true; } try { if (!CheckDataDirOption()) { return InitError(Untranslated( strprintf("Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "")))); } if (!gArgs.ReadConfigFiles(error, true)) { return InitError(Untranslated( strprintf("Error reading configuration file: %s\n", error))); } // Check for -chain, -testnet or -regtest parameter (Params() calls are // only valid after this clause) try { SelectParams(gArgs.GetChainName()); node.chain = interfaces::MakeChain(node, config.GetChainParams()); } catch (const std::exception &e) { return InitError(Untranslated(strprintf("%s\n", e.what()))); } // Make sure we create the net-specific data directory early on: if it // is new, this has a side effect of also creating // //wallets/. // // TODO: this should be removed once GetDataDir() no longer creates the // wallets/ subdirectory. // See more info at: // https://reviews.bitcoinabc.org/D3312 GetDataDir(true); // Error out when loose non-argument tokens are encountered on command // line for (int i = 1; i < argc; i++) { if (!IsSwitchChar(argv[i][0])) { return InitError(Untranslated( strprintf("Command line contains unexpected token '%s', " "see bitcoind -h for a list of options.\n", argv[i]))); } } if (!gArgs.InitSettings(error)) { InitError(Untranslated(error)); return false; } // -server defaults to true for bitcoind but not for the GUI so do this // here gArgs.SoftSetBoolArg("-server", true); // Set this early so that parameter interactions go to console InitLogging(); InitParameterInteraction(); if (!AppInitBasicSetup()) { // InitError will have been called with detailed error, which ends // up on console return false; } if (!AppInitParameterInteraction(config)) { // InitError will have been called with detailed error, which ends // up on console return false; } if (!AppInitSanityChecks()) { // InitError will have been called with detailed error, which ends // up on console return false; } if (gArgs.GetBoolArg("-daemon", false)) { #if HAVE_DECL_DAEMON #if defined(MAC_OSX) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif tfm::format(std::cout, PACKAGE_NAME " starting\n"); // Daemonize if (daemon(1, 0)) { // don't chdir (1), do close FDs (0) return InitError(Untranslated( strprintf("daemon() failed: %s\n", strerror(errno)))); } #if defined(MAC_OSX) #pragma GCC diagnostic pop #endif #else return InitError(Untranslated( "-daemon is not supported on this operating system\n")); #endif // HAVE_DECL_DAEMON } // Lock data directory after daemonization if (!AppInitLockDataDirectory()) { // If locking the data directory failed, exit immediately return false; } fRet = AppInitMain(config, rpcServer, httpRPCRequestProcessor, node); } catch (const std::exception &e) { PrintExceptionContinue(&e, "AppInit()"); } catch (...) { PrintExceptionContinue(nullptr, "AppInit()"); } if (!fRet) { Interrupt(node); } else { WaitForShutdown(node); } Shutdown(node); return fRet; } int main(int argc, char *argv[]) { #ifdef WIN32 util::WinCmdLineArgs winArgs; std::tie(argc, argv) = winArgs.get(); #endif SetupEnvironment(); // Connect bitcoind signal handlers noui_connect(); return (AppInit(argc, argv) ? EXIT_SUCCESS : EXIT_FAILURE); } diff --git a/src/httprpc.cpp b/src/httprpc.cpp index e979cc785..9e09af5a2 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -1,489 +1,489 @@ // Copyright (c) 2015-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include +#include #include -#include #include #include #include #include #include #include // boost::trim #include #include #include #include #include #include #include /** WWW-Authenticate to present with 401 Unauthorized response */ static const char *WWW_AUTH_HEADER_DATA = "Basic realm=\"jsonrpc\""; /** RPC auth failure delay to make brute-forcing expensive */ static const int64_t RPC_AUTH_BRUTE_FORCE_DELAY = 250; /** * Simple one-shot callback timer to be used by the RPC mechanism to e.g. * re-lock the wallet. */ class HTTPRPCTimer : public RPCTimerBase { public: HTTPRPCTimer(struct event_base *eventBase, std::function &func, int64_t millis) : ev(eventBase, false, func) { struct timeval tv; tv.tv_sec = millis / 1000; tv.tv_usec = (millis % 1000) * 1000; ev.trigger(&tv); } private: HTTPEvent ev; }; class HTTPRPCTimerInterface : public RPCTimerInterface { public: explicit HTTPRPCTimerInterface(struct event_base *_base) : base(_base) {} const char *Name() override { return "HTTP"; } RPCTimerBase *NewTimer(std::function &func, int64_t millis) override { return new HTTPRPCTimer(base, func, millis); } private: struct event_base *base; }; /* Pre-base64-encoded authentication token */ static std::string strRPCUserColonPass; /* Pre-base64-encoded authentication token */ static std::string strRPCCORSDomain; /* Stored RPC timer interface (for unregistration) */ static std::unique_ptr httpRPCTimerInterface; /* RPC Auth Whitelist */ static std::map> g_rpc_whitelist; static bool g_rpc_whitelist_default = false; static void JSONErrorReply(HTTPRequest *req, const UniValue &objError, const UniValue &id) { // Send error reply from json-rpc error object. int nStatus = HTTP_INTERNAL_SERVER_ERROR; int code = find_value(objError, "code").get_int(); if (code == RPC_INVALID_REQUEST) { nStatus = HTTP_BAD_REQUEST; } else if (code == RPC_METHOD_NOT_FOUND) { nStatus = HTTP_NOT_FOUND; } std::string strReply = JSONRPCReply(NullUniValue, objError, id); req->WriteHeader("Content-Type", "application/json"); req->WriteReply(nStatus, strReply); } /* * This function checks username and password against -rpcauth entries from * config file. */ static bool multiUserAuthorized(std::string strUserPass) { if (strUserPass.find(':') == std::string::npos) { return false; } std::string strUser = strUserPass.substr(0, strUserPass.find(':')); std::string strPass = strUserPass.substr(strUserPass.find(':') + 1); for (const std::string &strRPCAuth : gArgs.GetArgs("-rpcauth")) { // Search for multi-user login/pass "rpcauth" from config std::vector vFields; boost::split(vFields, strRPCAuth, boost::is_any_of(":$")); if (vFields.size() != 3) { // Incorrect formatting in config file continue; } std::string strName = vFields[0]; if (!TimingResistantEqual(strName, strUser)) { continue; } std::string strSalt = vFields[1]; std::string strHash = vFields[2]; static const unsigned int KEY_SIZE = 32; uint8_t out[KEY_SIZE]; CHMAC_SHA256(reinterpret_cast(strSalt.data()), strSalt.size()) .Write(reinterpret_cast(strPass.data()), strPass.size()) .Finalize(out); std::vector hexvec(out, out + KEY_SIZE); std::string strHashFromPass = HexStr(hexvec); if (TimingResistantEqual(strHashFromPass, strHash)) { return true; } } return false; } static bool RPCAuthorized(const std::string &strAuth, std::string &strAuthUsernameOut) { // Belt-and-suspenders measure if InitRPCAuthentication was not called. if (strRPCUserColonPass.empty()) { return false; } if (strAuth.substr(0, 6) != "Basic ") { return false; } std::string strUserPass64 = strAuth.substr(6); boost::trim(strUserPass64); std::string strUserPass = DecodeBase64(strUserPass64); if (strUserPass.find(':') != std::string::npos) { strAuthUsernameOut = strUserPass.substr(0, strUserPass.find(':')); } // Check if authorized under single-user field if (TimingResistantEqual(strUserPass, strRPCUserColonPass)) { return true; } return multiUserAuthorized(strUserPass); } static bool checkCORS(HTTPRequest *req) { // https://www.w3.org/TR/cors/#resource-requests // 1. If the Origin header is not present terminate this set of steps. // The request is outside the scope of this specification. std::pair origin = req->GetHeader("origin"); if (!origin.first) { return false; } // 2. If the value of the Origin header is not a case-sensitive match for // any of the values in list of origins do not set any additional headers // and terminate this set of steps. // Note: Always matching is acceptable since the list of origins can be // unbounded. if (origin.second != strRPCCORSDomain) { return false; } if (req->GetRequestMethod() == HTTPRequest::OPTIONS) { // 6.2 Preflight Request // In response to a preflight request the resource indicates which // methods and headers (other than simple methods and simple // headers) it is willing to handle and whether it supports // credentials. // Resources must use the following set of steps to determine which // additional headers to use in the response: // 3. Let method be the value as result of parsing the // Access-Control-Request-Method header. // If there is no Access-Control-Request-Method header or if parsing // failed, do not set any additional headers and terminate this set // of steps. The request is outside the scope of this specification. std::pair method = req->GetHeader("access-control-request-method"); if (!method.first) { return false; } // 4. Let header field-names be the values as result of parsing // the Access-Control-Request-Headers headers. // If there are no Access-Control-Request-Headers headers let header // field-names be the empty list. // If parsing failed do not set any additional headers and terminate // this set of steps. The request is outside the scope of this // specification. std::pair header_field_names = req->GetHeader("access-control-request-headers"); // 5. If method is not a case-sensitive match for any of the // values in list of methods do not set any additional headers // and terminate this set of steps. // Note: Always matching is acceptable since the list of methods // can be unbounded. if (method.second != "POST") { return false; } // 6. If any of the header field-names is not a ASCII case- // insensitive match for any of the values in list of headers do not // set any additional headers and terminate this set of steps. // Note: Always matching is acceptable since the list of headers can // be unbounded. const std::string &list_of_headers = "authorization,content-type"; // 7. If the resource supports credentials add a single // Access-Control-Allow-Origin header, with the value of the Origin // header as value, and add a single // Access-Control-Allow-Credentials header with the case-sensitive // string "true" as value. req->WriteHeader("Access-Control-Allow-Origin", origin.second); req->WriteHeader("Access-Control-Allow-Credentials", "true"); // 8. Optionally add a single Access-Control-Max-Age header with as // value the amount of seconds the user agent is allowed to cache // the result of the request. // 9. If method is a simple method this step may be skipped. // Add one or more Access-Control-Allow-Methods headers consisting // of (a subset of) the list of methods. // If a method is a simple method it does not need to be listed, but // this is not prohibited. // Note: Since the list of methods can be unbounded, simply // returning the method indicated by // Access-Control-Request-Method (if supported) can be enough. req->WriteHeader("Access-Control-Allow-Methods", method.second); // 10. If each of the header field-names is a simple header and none // is Content-Type, this step may be skipped. // Add one or more Access-Control-Allow-Headers headers consisting // of (a subset of) the list of headers. req->WriteHeader("Access-Control-Allow-Headers", header_field_names.first ? header_field_names.second : list_of_headers); req->WriteReply(HTTP_OK); return true; } // 6.1 Simple Cross-Origin Request, Actual Request, and Redirects // In response to a simple cross-origin request or actual request the // resource indicates whether or not to share the response. // If the resource has been relocated, it indicates whether to share its // new URL. // Resources must use the following set of steps to determine which // additional headers to use in the response: // 3. If the resource supports credentials add a single // Access-Control-Allow-Origin header, with the value of the Origin // header as value, and add a single Access-Control-Allow-Credentials // header with the case-sensitive string "true" as value. req->WriteHeader("Access-Control-Allow-Origin", origin.second); req->WriteHeader("Access-Control-Allow-Credentials", "true"); // 4. If the list of exposed headers is not empty add one or more // Access-Control-Expose-Headers headers, with as values the header // field names given in the list of exposed headers. req->WriteHeader("Access-Control-Expose-Headers", "WWW-Authenticate"); return false; } bool HTTPRPCRequestProcessor::ProcessHTTPRequest(HTTPRequest *req) { // First, check and/or set CORS headers if (checkCORS(req)) { return true; } // JSONRPC handles only POST if (req->GetRequestMethod() != HTTPRequest::POST) { req->WriteReply(HTTP_BAD_METHOD, "JSONRPC server handles only POST requests"); return false; } // Check authorization std::pair authHeader = req->GetHeader("authorization"); if (!authHeader.first) { req->WriteHeader("WWW-Authenticate", WWW_AUTH_HEADER_DATA); req->WriteReply(HTTP_UNAUTHORIZED); return false; } JSONRPCRequest jreq(context); jreq.peerAddr = req->GetPeer().ToString(); if (!RPCAuthorized(authHeader.second, jreq.authUser)) { LogPrintf("ThreadRPCServer incorrect password attempt from %s\n", jreq.peerAddr); /** * Deter brute-forcing. * If this results in a DoS the user really shouldn't have their RPC * port exposed. */ UninterruptibleSleep( std::chrono::milliseconds{RPC_AUTH_BRUTE_FORCE_DELAY}); req->WriteHeader("WWW-Authenticate", WWW_AUTH_HEADER_DATA); req->WriteReply(HTTP_UNAUTHORIZED); return false; } try { // Parse request UniValue valRequest; if (!valRequest.read(req->ReadBody())) { throw JSONRPCError(RPC_PARSE_ERROR, "Parse error"); } // Set the URI jreq.URI = req->GetURI(); std::string strReply; bool user_has_whitelist = g_rpc_whitelist.count(jreq.authUser); if (!user_has_whitelist && g_rpc_whitelist_default) { LogPrintf("RPC User %s not allowed to call any methods\n", jreq.authUser); req->WriteReply(HTTP_FORBIDDEN); return false; // singleton request } else if (valRequest.isObject()) { jreq.parse(valRequest); if (user_has_whitelist && !g_rpc_whitelist[jreq.authUser].count(jreq.strMethod)) { LogPrintf("RPC User %s not allowed to call method %s\n", jreq.authUser, jreq.strMethod); req->WriteReply(HTTP_FORBIDDEN); return false; } UniValue result = rpcServer.ExecuteCommand(config, jreq); // Send reply strReply = JSONRPCReply(result, NullUniValue, jreq.id); // array of requests } else if (valRequest.isArray()) { if (user_has_whitelist) { for (unsigned int reqIdx = 0; reqIdx < valRequest.size(); reqIdx++) { if (!valRequest[reqIdx].isObject()) { throw JSONRPCError(RPC_INVALID_REQUEST, "Invalid Request object"); } else { const UniValue &request = valRequest[reqIdx].get_obj(); // Parse method std::string strMethod = find_value(request, "method").get_str(); if (!g_rpc_whitelist[jreq.authUser].count(strMethod)) { LogPrintf( "RPC User %s not allowed to call method %s\n", jreq.authUser, strMethod); req->WriteReply(HTTP_FORBIDDEN); return false; } } } } strReply = JSONRPCExecBatch(config, rpcServer, jreq, valRequest.get_array()); } else { throw JSONRPCError(RPC_PARSE_ERROR, "Top-level object parse error"); } req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strReply); } catch (const UniValue &objError) { JSONErrorReply(req, objError, jreq.id); return false; } catch (const std::exception &e) { JSONErrorReply(req, JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq.id); return false; } return true; } static bool InitRPCAuthentication() { if (gArgs.GetArg("-rpcpassword", "") == "") { LogPrintf("No rpcpassword set - using random cookie authentication.\n"); if (!GenerateAuthCookie(&strRPCUserColonPass)) { // Same message as AbortNode. uiInterface.ThreadSafeMessageBox( _("Error: A fatal internal error occurred, see debug.log for " "details"), "", CClientUIInterface::MSG_ERROR); return false; } } else { LogPrintf("Config options rpcuser and rpcpassword will soon be " "deprecated. Locally-run instances may remove rpcuser to use " "cookie-based auth, or may be replaced with rpcauth. Please " "see share/rpcauth for rpcauth auth generation.\n"); strRPCUserColonPass = gArgs.GetArg("-rpcuser", "") + ":" + gArgs.GetArg("-rpcpassword", ""); } strRPCCORSDomain = gArgs.GetArg("-rpccorsdomain", ""); if (gArgs.GetArg("-rpcauth", "") != "") { LogPrintf("Using rpcauth authentication.\n"); } g_rpc_whitelist_default = gArgs.GetBoolArg("-rpcwhitelistdefault", gArgs.IsArgSet("-rpcwhitelist")); for (const std::string &strRPCWhitelist : gArgs.GetArgs("-rpcwhitelist")) { auto pos = strRPCWhitelist.find(':'); std::string strUser = strRPCWhitelist.substr(0, pos); bool intersect = g_rpc_whitelist.count(strUser); std::set &whitelist = g_rpc_whitelist[strUser]; if (pos != std::string::npos) { std::string strWhitelist = strRPCWhitelist.substr(pos + 1); std::set new_whitelist; boost::split(new_whitelist, strWhitelist, boost::is_any_of(", ")); if (intersect) { std::set tmp_whitelist; std::set_intersection( new_whitelist.begin(), new_whitelist.end(), whitelist.begin(), whitelist.end(), std::inserter(tmp_whitelist, tmp_whitelist.end())); new_whitelist = std::move(tmp_whitelist); } whitelist = std::move(new_whitelist); } } return true; } bool StartHTTPRPC(HTTPRPCRequestProcessor &httpRPCRequestProcessor) { LogPrint(BCLog::RPC, "Starting HTTP RPC server\n"); if (!InitRPCAuthentication()) { return false; } const std::function &rpcFunction = std::bind(&HTTPRPCRequestProcessor::DelegateHTTPRequest, &httpRPCRequestProcessor, std::placeholders::_2); RegisterHTTPHandler("/", true, rpcFunction); if (g_wallet_init_interface.HasWalletSupport()) { RegisterHTTPHandler("/wallet/", false, rpcFunction); } struct event_base *eventBase = EventBase(); assert(eventBase); httpRPCTimerInterface = std::make_unique(eventBase); RPCSetTimerInterface(httpRPCTimerInterface.get()); return true; } void InterruptHTTPRPC() { LogPrint(BCLog::RPC, "Interrupting HTTP RPC server\n"); } void StopHTTPRPC() { LogPrint(BCLog::RPC, "Stopping HTTP RPC server\n"); UnregisterHTTPHandler("/", true); if (g_wallet_init_interface.HasWalletSupport()) { UnregisterHTTPHandler("/wallet/", false); } if (httpRPCTimerInterface) { RPCUnsetTimerInterface(httpRPCTimerInterface.get()); httpRPCTimerInterface.reset(); } } diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 41817352a..5faa1e86b 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -1,697 +1,697 @@ // Copyright (c) 2015-2016 The Bitcoin Core developers // Copyright (c) 2018-2019 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include +#include #include // For HTTP status codes #include #include -#include #include #include #include #include #include #include #include #include #include #include #ifdef EVENT__HAVE_NETINET_IN_H #include #ifdef _XOPEN_SOURCE_EXTENDED #include #endif #endif #include #include #include #include #include #include #include /** Maximum size of http request (request line + headers) */ static const size_t MAX_HEADERS_SIZE = 8192; /** * Maximum HTTP post body size. Twice the maximum block size is added to this * value in practice. */ static const size_t MIN_SUPPORTED_BODY_SIZE = 0x02000000; /** HTTP request work item */ class HTTPWorkItem final : public HTTPClosure { public: HTTPWorkItem(Config &_config, std::unique_ptr _req, const std::string &_path, const HTTPRequestHandler &_func) : req(std::move(_req)), path(_path), func(_func), config(&_config) {} void operator()() override { func(*config, req.get(), path); } std::unique_ptr req; private: std::string path; HTTPRequestHandler func; Config *config; }; /** * Simple work queue for distributing work over multiple threads. * Work items are simply callable objects. */ template class WorkQueue { private: /** Mutex protects entire object */ Mutex cs; std::condition_variable cond; std::deque> queue; bool running; size_t maxDepth; public: explicit WorkQueue(size_t _maxDepth) : running(true), maxDepth(_maxDepth) {} /** * Precondition: worker threads have all stopped (they have all been joined) */ ~WorkQueue() {} /** Enqueue a work item */ bool Enqueue(WorkItem *item) { LOCK(cs); if (queue.size() >= maxDepth) { return false; } queue.emplace_back(std::unique_ptr(item)); cond.notify_one(); return true; } /** Thread function */ void Run() { while (true) { std::unique_ptr i; { WAIT_LOCK(cs, lock); while (running && queue.empty()) { cond.wait(lock); } if (!running) { break; } i = std::move(queue.front()); queue.pop_front(); } (*i)(); } } /** Interrupt and exit loops */ void Interrupt() { LOCK(cs); running = false; cond.notify_all(); } }; struct HTTPPathHandler { HTTPPathHandler(std::string _prefix, bool _exactMatch, HTTPRequestHandler _handler) : prefix(_prefix), exactMatch(_exactMatch), handler(_handler) {} std::string prefix; bool exactMatch; HTTPRequestHandler handler; }; /** HTTP module state */ //! libevent event loop static struct event_base *eventBase = nullptr; //! HTTP server static struct evhttp *eventHTTP = nullptr; //! List of subnets to allow RPC connections from static std::vector rpc_allow_subnets; //! Work queue for handling longer requests off the event loop thread static WorkQueue *workQueue = nullptr; //! Handlers for (sub)paths static std::vector pathHandlers; //! Bound listening sockets static std::vector boundSockets; /** Check if a network address is allowed to access the HTTP server */ static bool ClientAllowed(const CNetAddr &netaddr) { if (!netaddr.IsValid()) { return false; } for (const CSubNet &subnet : rpc_allow_subnets) { if (subnet.Match(netaddr)) { return true; } } return false; } /** Initialize ACL list for HTTP server */ static bool InitHTTPAllowList() { rpc_allow_subnets.clear(); CNetAddr localv4; CNetAddr localv6; LookupHost("127.0.0.1", localv4, false); LookupHost("::1", localv6, false); // always allow IPv4 local subnet. rpc_allow_subnets.push_back(CSubNet(localv4, 8)); // always allow IPv6 localhost. rpc_allow_subnets.push_back(CSubNet(localv6)); for (const std::string &strAllow : gArgs.GetArgs("-rpcallowip")) { CSubNet subnet; LookupSubNet(strAllow.c_str(), subnet); if (!subnet.IsValid()) { uiInterface.ThreadSafeMessageBox( strprintf( Untranslated("Invalid -rpcallowip subnet specification: " "%s. Valid are a single IP (e.g. 1.2.3.4), a " "network/netmask (e.g. 1.2.3.4/255.255.255.0) " "or a network/CIDR (e.g. 1.2.3.4/24)."), strAllow), "", CClientUIInterface::MSG_ERROR); return false; } rpc_allow_subnets.push_back(subnet); } std::string strAllowed; for (const CSubNet &subnet : rpc_allow_subnets) { strAllowed += subnet.ToString() + " "; } LogPrint(BCLog::HTTP, "Allowing HTTP connections from: %s\n", strAllowed); return true; } /** HTTP request method as string - use for logging only */ static std::string RequestMethodString(HTTPRequest::RequestMethod m) { switch (m) { case HTTPRequest::GET: return "GET"; case HTTPRequest::POST: return "POST"; case HTTPRequest::HEAD: return "HEAD"; case HTTPRequest::PUT: return "PUT"; case HTTPRequest::OPTIONS: return "OPTIONS"; default: return "unknown"; } } /** HTTP request callback */ static void http_request_cb(struct evhttp_request *req, void *arg) { Config &config = *reinterpret_cast(arg); // Disable reading to work around a libevent bug, fixed in 2.2.0. if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02020001) { evhttp_connection *conn = evhttp_request_get_connection(req); if (conn) { bufferevent *bev = evhttp_connection_get_bufferevent(conn); if (bev) { bufferevent_disable(bev, EV_READ); } } } auto hreq = std::make_unique(req); // Early address-based allow check if (!ClientAllowed(hreq->GetPeer())) { LogPrint(BCLog::HTTP, "HTTP request from %s rejected: Client network is not allowed " "RPC access\n", hreq->GetPeer().ToString()); hreq->WriteReply(HTTP_FORBIDDEN); return; } // Early reject unknown HTTP methods if (hreq->GetRequestMethod() == HTTPRequest::UNKNOWN) { LogPrint(BCLog::HTTP, "HTTP request from %s rejected: Unknown HTTP request method\n", hreq->GetPeer().ToString()); hreq->WriteReply(HTTP_BADMETHOD); return; } LogPrint(BCLog::HTTP, "Received a %s request for %s from %s\n", RequestMethodString(hreq->GetRequestMethod()), SanitizeString(hreq->GetURI(), SAFE_CHARS_URI).substr(0, 100), hreq->GetPeer().ToString()); // Find registered handler for prefix std::string strURI = hreq->GetURI(); std::string path; std::vector::const_iterator i = pathHandlers.begin(); std::vector::const_iterator iend = pathHandlers.end(); for (; i != iend; ++i) { bool match = false; if (i->exactMatch) { match = (strURI == i->prefix); } else { match = (strURI.substr(0, i->prefix.size()) == i->prefix); } if (match) { path = strURI.substr(i->prefix.size()); break; } } // Dispatch to worker thread. if (i != iend) { std::unique_ptr item( new HTTPWorkItem(config, std::move(hreq), path, i->handler)); assert(workQueue); if (workQueue->Enqueue(item.get())) { /* if true, queue took ownership */ item.release(); } else { LogPrintf("WARNING: request rejected because http work queue depth " "exceeded, it can be increased with the -rpcworkqueue= " "setting\n"); item->req->WriteReply(HTTP_INTERNAL, "Work queue depth exceeded"); } } else { hreq->WriteReply(HTTP_NOTFOUND); } } /** Callback to reject HTTP requests after shutdown. */ static void http_reject_request_cb(struct evhttp_request *req, void *) { LogPrint(BCLog::HTTP, "Rejecting request while shutting down\n"); evhttp_send_error(req, HTTP_SERVUNAVAIL, nullptr); } /** Event dispatcher thread */ static bool ThreadHTTP(struct event_base *base) { util::ThreadRename("http"); LogPrint(BCLog::HTTP, "Entering http event loop\n"); event_base_dispatch(base); // Event loop will be interrupted by InterruptHTTPServer() LogPrint(BCLog::HTTP, "Exited http event loop\n"); return event_base_got_break(base) == 0; } /** Bind HTTP server to specified addresses */ static bool HTTPBindAddresses(struct evhttp *http) { int http_port = gArgs.GetArg("-rpcport", BaseParams().RPCPort()); std::vector> endpoints; // Determine what addresses to bind to if (!(gArgs.IsArgSet("-rpcallowip") && gArgs.IsArgSet("-rpcbind"))) { // Default to loopback if not allowing external IPs. endpoints.push_back(std::make_pair("::1", http_port)); endpoints.push_back(std::make_pair("127.0.0.1", http_port)); if (gArgs.IsArgSet("-rpcallowip")) { LogPrintf("WARNING: option -rpcallowip was specified without " "-rpcbind; this doesn't usually make sense\n"); } if (gArgs.IsArgSet("-rpcbind")) { LogPrintf("WARNING: option -rpcbind was ignored because " "-rpcallowip was not specified, refusing to allow " "everyone to connect\n"); } } else if (gArgs.IsArgSet("-rpcbind")) { // Specific bind address. for (const std::string &strRPCBind : gArgs.GetArgs("-rpcbind")) { int port = http_port; std::string host; SplitHostPort(strRPCBind, port, host); endpoints.push_back(std::make_pair(host, port)); } } // Bind addresses for (std::vector>::iterator i = endpoints.begin(); i != endpoints.end(); ++i) { LogPrint(BCLog::HTTP, "Binding RPC on address %s port %i\n", i->first, i->second); evhttp_bound_socket *bind_handle = evhttp_bind_socket_with_handle( http, i->first.empty() ? nullptr : i->first.c_str(), i->second); if (bind_handle) { CNetAddr addr; if (i->first.empty() || (LookupHost(i->first.c_str(), addr, false) && addr.IsBindAny())) { LogPrintf("WARNING: the RPC server is not safe to expose to " "untrusted networks such as the public internet\n"); } boundSockets.push_back(bind_handle); } else { LogPrintf("Binding RPC on address %s port %i failed.\n", i->first, i->second); } } return !boundSockets.empty(); } /** Simple wrapper to set thread name and run work queue */ static void HTTPWorkQueueRun(WorkQueue *queue, int worker_num) { util::ThreadRename(strprintf("httpworker.%i", worker_num)); queue->Run(); } /** libevent event log callback */ static void libevent_log_cb(int severity, const char *msg) { #ifndef EVENT_LOG_WARN // EVENT_LOG_WARN was added in 2.0.19; but before then _EVENT_LOG_WARN existed. #define EVENT_LOG_WARN _EVENT_LOG_WARN #endif // Log warn messages and higher without debug category. if (severity >= EVENT_LOG_WARN) { LogPrintf("libevent: %s\n", msg); } else { LogPrint(BCLog::LIBEVENT, "libevent: %s\n", msg); } } bool InitHTTPServer(Config &config) { if (!InitHTTPAllowList()) { return false; } // Redirect libevent's logging to our own log event_set_log_callback(&libevent_log_cb); // Update libevent's log handling. Returns false if our version of // libevent doesn't support debug logging, in which case we should // clear the BCLog::LIBEVENT flag. if (!UpdateHTTPServerLogging( LogInstance().WillLogCategory(BCLog::LIBEVENT))) { LogInstance().DisableCategory(BCLog::LIBEVENT); } #ifdef WIN32 evthread_use_windows_threads(); #else evthread_use_pthreads(); #endif raii_event_base base_ctr = obtain_event_base(); /* Create a new evhttp object to handle requests. */ raii_evhttp http_ctr = obtain_evhttp(base_ctr.get()); struct evhttp *http = http_ctr.get(); if (!http) { LogPrintf("couldn't create evhttp. Exiting.\n"); return false; } evhttp_set_timeout( http, gArgs.GetArg("-rpcservertimeout", DEFAULT_HTTP_SERVER_TIMEOUT)); evhttp_set_max_headers_size(http, MAX_HEADERS_SIZE); evhttp_set_max_body_size(http, MIN_SUPPORTED_BODY_SIZE + 2 * config.GetMaxBlockSize()); evhttp_set_gencb(http, http_request_cb, &config); // Only POST and OPTIONS are supported, but we return HTTP 405 for the // others evhttp_set_allowed_methods( http, EVHTTP_REQ_GET | EVHTTP_REQ_POST | EVHTTP_REQ_HEAD | EVHTTP_REQ_PUT | EVHTTP_REQ_DELETE | EVHTTP_REQ_OPTIONS); if (!HTTPBindAddresses(http)) { LogPrintf("Unable to bind any endpoint for RPC server\n"); return false; } LogPrint(BCLog::HTTP, "Initialized HTTP server\n"); int workQueueDepth = std::max( (long)gArgs.GetArg("-rpcworkqueue", DEFAULT_HTTP_WORKQUEUE), 1L); LogPrintf("HTTP: creating work queue of depth %d\n", workQueueDepth); workQueue = new WorkQueue(workQueueDepth); // transfer ownership to eventBase/HTTP via .release() eventBase = base_ctr.release(); eventHTTP = http_ctr.release(); return true; } bool UpdateHTTPServerLogging(bool enable) { #if LIBEVENT_VERSION_NUMBER >= 0x02010100 if (enable) { event_enable_debug_logging(EVENT_DBG_ALL); } else { event_enable_debug_logging(EVENT_DBG_NONE); } return true; #else // Can't update libevent logging if version < 02010100 return false; #endif } static std::thread threadHTTP; static std::vector g_thread_http_workers; void StartHTTPServer() { LogPrint(BCLog::HTTP, "Starting HTTP server\n"); int rpcThreads = std::max((long)gArgs.GetArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); LogPrintf("HTTP: starting %d worker threads\n", rpcThreads); threadHTTP = std::thread(ThreadHTTP, eventBase); for (int i = 0; i < rpcThreads; i++) { g_thread_http_workers.emplace_back(HTTPWorkQueueRun, workQueue, i); } } void InterruptHTTPServer() { LogPrint(BCLog::HTTP, "Interrupting HTTP server\n"); if (eventHTTP) { // Reject requests on current connections evhttp_set_gencb(eventHTTP, http_reject_request_cb, nullptr); } if (workQueue) { workQueue->Interrupt(); } } void StopHTTPServer() { LogPrint(BCLog::HTTP, "Stopping HTTP server\n"); if (workQueue) { LogPrint(BCLog::HTTP, "Waiting for HTTP worker threads to exit\n"); for (auto &thread : g_thread_http_workers) { thread.join(); } g_thread_http_workers.clear(); delete workQueue; workQueue = nullptr; } // Unlisten sockets, these are what make the event loop running, which means // that after this and all connections are closed the event loop will quit. for (evhttp_bound_socket *socket : boundSockets) { evhttp_del_accept_socket(eventHTTP, socket); } boundSockets.clear(); if (eventBase) { LogPrint(BCLog::HTTP, "Waiting for HTTP event thread to exit\n"); threadHTTP.join(); } if (eventHTTP) { evhttp_free(eventHTTP); eventHTTP = nullptr; } if (eventBase) { event_base_free(eventBase); eventBase = nullptr; } LogPrint(BCLog::HTTP, "Stopped HTTP server\n"); } struct event_base *EventBase() { return eventBase; } static void httpevent_callback_fn(evutil_socket_t, short, void *data) { // Static handler: simply call inner handler HTTPEvent *self = static_cast(data); self->handler(); if (self->deleteWhenTriggered) { delete self; } } HTTPEvent::HTTPEvent(struct event_base *base, bool _deleteWhenTriggered, const std::function &_handler) : deleteWhenTriggered(_deleteWhenTriggered), handler(_handler) { ev = event_new(base, -1, 0, httpevent_callback_fn, this); assert(ev); } HTTPEvent::~HTTPEvent() { event_free(ev); } void HTTPEvent::trigger(struct timeval *tv) { if (tv == nullptr) { // Immediately trigger event in main thread. event_active(ev, 0, 0); } else { // Trigger after timeval passed. evtimer_add(ev, tv); } } HTTPRequest::HTTPRequest(struct evhttp_request *_req) : req(_req), replySent(false) {} HTTPRequest::~HTTPRequest() { if (!replySent) { // Keep track of whether reply was sent to avoid request leaks LogPrintf("%s: Unhandled request\n", __func__); WriteReply(HTTP_INTERNAL, "Unhandled request"); } // evhttpd cleans up the request, as long as a reply was sent. } std::pair HTTPRequest::GetHeader(const std::string &hdr) const { const struct evkeyvalq *headers = evhttp_request_get_input_headers(req); assert(headers); const char *val = evhttp_find_header(headers, hdr.c_str()); if (val) { return std::make_pair(true, val); } else { return std::make_pair(false, ""); } } std::string HTTPRequest::ReadBody() { struct evbuffer *buf = evhttp_request_get_input_buffer(req); if (!buf) { return ""; } size_t size = evbuffer_get_length(buf); /** * Trivial implementation: if this is ever a performance bottleneck, * internal copying can be avoided in multi-segment buffers by using * evbuffer_peek and an awkward loop. Though in that case, it'd be even * better to not copy into an intermediate string but use a stream * abstraction to consume the evbuffer on the fly in the parsing algorithm. */ const char *data = (const char *)evbuffer_pullup(buf, size); // returns nullptr in case of empty buffer. if (!data) { return ""; } std::string rv(data, size); evbuffer_drain(buf, size); return rv; } void HTTPRequest::WriteHeader(const std::string &hdr, const std::string &value) { struct evkeyvalq *headers = evhttp_request_get_output_headers(req); assert(headers); evhttp_add_header(headers, hdr.c_str(), value.c_str()); } /** * Closure sent to main thread to request a reply to be sent to a HTTP request. * Replies must be sent in the main loop in the main http thread, this cannot be * done from worker threads. */ void HTTPRequest::WriteReply(int nStatus, const std::string &strReply) { assert(!replySent && req); if (ShutdownRequested()) { WriteHeader("Connection", "close"); } // Send event to main http thread to send reply message struct evbuffer *evb = evhttp_request_get_output_buffer(req); assert(evb); evbuffer_add(evb, strReply.data(), strReply.size()); auto req_copy = req; HTTPEvent *ev = new HTTPEvent(eventBase, true, [req_copy, nStatus] { evhttp_send_reply(req_copy, nStatus, nullptr, nullptr); // Re-enable reading from the socket. This is the second part of the // libevent workaround above. if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02020001) { evhttp_connection *conn = evhttp_request_get_connection(req_copy); if (conn) { bufferevent *bev = evhttp_connection_get_bufferevent(conn); if (bev) { bufferevent_enable(bev, EV_READ | EV_WRITE); } } } }); ev->trigger(nullptr); replySent = true; // transferred back to main thread. req = nullptr; } CService HTTPRequest::GetPeer() const { evhttp_connection *con = evhttp_request_get_connection(req); CService peer; if (con) { // evhttp retains ownership over returned address string const char *address = ""; uint16_t port = 0; evhttp_connection_get_peer(con, (char **)&address, &port); peer = LookupNumeric(address, port); } return peer; } std::string HTTPRequest::GetURI() const { return evhttp_request_get_uri(req); } HTTPRequest::RequestMethod HTTPRequest::GetRequestMethod() const { switch (evhttp_request_get_command(req)) { case EVHTTP_REQ_GET: return GET; case EVHTTP_REQ_POST: return POST; case EVHTTP_REQ_HEAD: return HEAD; case EVHTTP_REQ_PUT: return PUT; case EVHTTP_REQ_OPTIONS: return OPTIONS; default: return UNKNOWN; } } void RegisterHTTPHandler(const std::string &prefix, bool exactMatch, const HTTPRequestHandler &handler) { LogPrint(BCLog::HTTP, "Registering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); pathHandlers.push_back(HTTPPathHandler(prefix, exactMatch, handler)); } void UnregisterHTTPHandler(const std::string &prefix, bool exactMatch) { std::vector::iterator i = pathHandlers.begin(); std::vector::iterator iend = pathHandlers.end(); for (; i != iend; ++i) { if (i->prefix == prefix && i->exactMatch == exactMatch) { break; } } if (i != iend) { LogPrint(BCLog::HTTP, "Unregistering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); pathHandlers.erase(i); } } diff --git a/src/index/base.cpp b/src/index/base.cpp index 450f0855b..e1f5158d3 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -1,331 +1,328 @@ // Copyright (c) 2017-2018 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include +#include #include #include -#include #include #include #include #include constexpr char DB_BEST_BLOCK = 'B'; constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds template static void FatalError(const char *fmt, const Args &... args) { std::string strMessage = tfm::format(fmt, args...); SetMiscWarning(strMessage); LogPrintf("*** %s\n", strMessage); - uiInterface.ThreadSafeMessageBox( - Untranslated("Error: A fatal internal error occurred, see debug.log " - "for details"), - "", CClientUIInterface::MSG_ERROR); + AbortError(_("A fatal internal error occurred, see debug.log for details")); StartShutdown(); } BaseIndex::DB::DB(const fs::path &path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) : CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate) {} bool BaseIndex::DB::ReadBestBlock(CBlockLocator &locator) const { bool success = Read(DB_BEST_BLOCK, locator); if (!success) { locator.SetNull(); } return success; } void BaseIndex::DB::WriteBestBlock(CDBBatch &batch, const CBlockLocator &locator) { batch.Write(DB_BEST_BLOCK, locator); } BaseIndex::~BaseIndex() { Interrupt(); Stop(); } bool BaseIndex::Init() { CBlockLocator locator; if (!GetDB().ReadBestBlock(locator)) { locator.SetNull(); } LOCK(cs_main); if (locator.IsNull()) { m_best_block_index = nullptr; } else { m_best_block_index = FindForkInGlobalIndex(::ChainActive(), locator); } m_synced = m_best_block_index.load() == ::ChainActive().Tip(); return true; } static const CBlockIndex *NextSyncBlock(const CBlockIndex *pindex_prev) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { AssertLockHeld(cs_main); if (!pindex_prev) { return ::ChainActive().Genesis(); } const CBlockIndex *pindex = ::ChainActive().Next(pindex_prev); if (pindex) { return pindex; } return ::ChainActive().Next(::ChainActive().FindFork(pindex_prev)); } void BaseIndex::ThreadSync() { const CBlockIndex *pindex = m_best_block_index.load(); if (!m_synced) { auto &consensus_params = GetConfig().GetChainParams().GetConsensus(); int64_t last_log_time = 0; int64_t last_locator_write_time = 0; while (true) { if (m_interrupt) { m_best_block_index = pindex; // No need to handle errors in Commit. If it fails, the error // will be already be logged. The best way to recover is to // continue, as index cannot be corrupted by a missed commit to // disk for an advanced index state. Commit(); return; } { LOCK(cs_main); const CBlockIndex *pindex_next = NextSyncBlock(pindex); if (!pindex_next) { m_best_block_index = pindex; m_synced = true; // No need to handle errors in Commit. See rationale above. Commit(); break; } if (pindex_next->pprev != pindex && !Rewind(pindex, pindex_next->pprev)) { FatalError( "%s: Failed to rewind index %s to a previous chain tip", __func__, GetName()); return; } pindex = pindex_next; } int64_t current_time = GetTime(); if (last_log_time + SYNC_LOG_INTERVAL < current_time) { LogPrintf("Syncing %s with block chain from height %d\n", GetName(), pindex->nHeight); last_log_time = current_time; } if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { m_best_block_index = pindex; last_locator_write_time = current_time; // No need to handle errors in Commit. See rationale above. Commit(); } CBlock block; if (!ReadBlockFromDisk(block, pindex, consensus_params)) { FatalError("%s: Failed to read block %s from disk", __func__, pindex->GetBlockHash().ToString()); return; } if (!WriteBlock(block, pindex)) { FatalError("%s: Failed to write block %s to index database", __func__, pindex->GetBlockHash().ToString()); return; } } } if (pindex) { LogPrintf("%s is enabled at height %d\n", GetName(), pindex->nHeight); } else { LogPrintf("%s is enabled\n", GetName()); } } bool BaseIndex::Commit() { CDBBatch batch(GetDB()); if (!CommitInternal(batch) || !GetDB().WriteBatch(batch)) { return error("%s: Failed to commit latest %s state", __func__, GetName()); } return true; } bool BaseIndex::CommitInternal(CDBBatch &batch) { LOCK(cs_main); GetDB().WriteBestBlock(batch, ::ChainActive().GetLocator(m_best_block_index)); return true; } bool BaseIndex::Rewind(const CBlockIndex *current_tip, const CBlockIndex *new_tip) { assert(current_tip == m_best_block_index); assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); // In the case of a reorg, ensure persisted block locator is not stale. m_best_block_index = new_tip; if (!Commit()) { // If commit fails, revert the best block index to avoid corruption. m_best_block_index = current_tip; return false; } return true; } void BaseIndex::BlockConnected( const std::shared_ptr &block, const CBlockIndex *pindex, const std::vector &txn_conflicted) { if (!m_synced) { return; } const CBlockIndex *best_block_index = m_best_block_index.load(); if (!best_block_index) { if (pindex->nHeight != 0) { FatalError("%s: First block connected is not the genesis block " "(height=%d)", __func__, pindex->nHeight); return; } } else { // Ensure block connects to an ancestor of the current best block. This // should be the case most of the time, but may not be immediately after // the the sync thread catches up and sets m_synced. Consider the case // where there is a reorg and the blocks on the stale branch are in the // ValidationInterface queue backlog even after the sync thread has // caught up to the new chain tip. In this unlikely event, log a warning // and let the queue clear. if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) { LogPrintf("%s: WARNING: Block %s does not connect to an ancestor " "of known best chain (tip=%s); not updating index\n", __func__, pindex->GetBlockHash().ToString(), best_block_index->GetBlockHash().ToString()); return; } if (best_block_index != pindex->pprev && !Rewind(best_block_index, pindex->pprev)) { FatalError("%s: Failed to rewind index %s to a previous chain tip", __func__, GetName()); return; } } if (WriteBlock(*block, pindex)) { m_best_block_index = pindex; } else { FatalError("%s: Failed to write block %s to index", __func__, pindex->GetBlockHash().ToString()); return; } } void BaseIndex::ChainStateFlushed(const CBlockLocator &locator) { if (!m_synced) { return; } const BlockHash &locator_tip_hash = locator.vHave.front(); const CBlockIndex *locator_tip_index; { LOCK(cs_main); locator_tip_index = LookupBlockIndex(locator_tip_hash); } if (!locator_tip_index) { FatalError("%s: First block (hash=%s) in locator was not found", __func__, locator_tip_hash.ToString()); return; } // This checks that ChainStateFlushed callbacks are received after // BlockConnected. The check may fail immediately after the the sync thread // catches up and sets m_synced. Consider the case where there is a reorg // and the blocks on the stale branch are in the ValidationInterface queue // backlog even after the sync thread has caught up to the new chain tip. In // this unlikely event, log a warning and let the queue clear. const CBlockIndex *best_block_index = m_best_block_index.load(); if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) { LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known " "best chain (tip=%s); not writing index locator\n", __func__, locator_tip_hash.ToString(), best_block_index->GetBlockHash().ToString()); return; } // No need to handle errors in Commit. If it fails, the error will be // already be logged. The best way to recover is to continue, as index // cannot be corrupted by a missed commit to disk for an advanced index // state. Commit(); } bool BaseIndex::BlockUntilSyncedToCurrentChain() const { AssertLockNotHeld(cs_main); if (!m_synced) { return false; } { // Skip the queue-draining stuff if we know we're caught up with // ::ChainActive().Tip(). LOCK(cs_main); const CBlockIndex *chain_tip = ::ChainActive().Tip(); const CBlockIndex *best_block_index = m_best_block_index.load(); if (best_block_index->GetAncestor(chain_tip->nHeight) == chain_tip) { return true; } } LogPrintf("%s: %s is catching up on block notifications\n", __func__, GetName()); SyncWithValidationInterfaceQueue(); return true; } void BaseIndex::Interrupt() { m_interrupt(); } void BaseIndex::Start() { // Need to register this ValidationInterface before running Init(), so that // callbacks are not missed if Init sets m_synced to true. RegisterValidationInterface(this); if (!Init()) { FatalError("%s: %s failed to initialize", __func__, GetName()); return; } m_thread_sync = std::thread(&TraceThread>, GetName(), std::bind(&BaseIndex::ThreadSync, this)); } void BaseIndex::Stop() { UnregisterValidationInterface(this); if (m_thread_sync.joinable()) { m_thread_sync.join(); } } diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 0af436f1e..334912b24 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -1,298 +1,298 @@ // Copyright (c) 2017-2018 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include +#include #include -#include #include #include #include #include constexpr char DB_BEST_BLOCK = 'B'; constexpr char DB_TXINDEX = 't'; constexpr char DB_TXINDEX_BLOCK = 'T'; std::unique_ptr g_txindex; struct CDiskTxPos : public FlatFilePos { unsigned int nTxOffset; // after header ADD_SERIALIZE_METHODS; template inline void SerializationOp(Stream &s, Operation ser_action) { READWRITEAS(FlatFilePos, *this); READWRITE(VARINT(nTxOffset)); } CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {} CDiskTxPos() { SetNull(); } void SetNull() { FlatFilePos::SetNull(); nTxOffset = 0; } }; /** * Access to the txindex database (indexes/txindex/) * * The database stores a block locator of the chain the database is synced to * so that the TxIndex can efficiently determine the point it last stopped at. * A locator is used instead of a simple hash of the chain tip because blocks * and block index entries may not be flushed to disk until after this database * is updated. */ class TxIndex::DB : public BaseIndex::DB { public: explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); /// Read the disk location of the transaction data with the given ID. /// Returns false if the transaction ID is not indexed. bool ReadTxPos(const TxId &txid, CDiskTxPos &pos) const; /// Write a batch of transaction positions to the DB. bool WriteTxs(const std::vector> &v_pos); /// Migrate txindex data from the block tree DB, where it may be for older /// nodes that have not been upgraded yet to the new database. bool MigrateData(CBlockTreeDB &block_tree_db, const CBlockLocator &best_locator); }; TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) : BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe) {} bool TxIndex::DB::ReadTxPos(const TxId &txid, CDiskTxPos &pos) const { return Read(std::make_pair(DB_TXINDEX, txid), pos); } bool TxIndex::DB::WriteTxs( const std::vector> &v_pos) { CDBBatch batch(*this); for (const auto &tuple : v_pos) { batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second); } return WriteBatch(batch); } /* * Safely persist a transfer of data from the old txindex database to the new * one, and compact the range of keys updated. This is used internally by * MigrateData. */ static void WriteTxIndexMigrationBatches(CDBWrapper &newdb, CDBWrapper &olddb, CDBBatch &batch_newdb, CDBBatch &batch_olddb, const std::pair &begin_key, const std::pair &end_key) { // Sync new DB changes to disk before deleting from old DB. newdb.WriteBatch(batch_newdb, /*fSync=*/true); olddb.WriteBatch(batch_olddb); olddb.CompactRange(begin_key, end_key); batch_newdb.Clear(); batch_olddb.Clear(); } bool TxIndex::DB::MigrateData(CBlockTreeDB &block_tree_db, const CBlockLocator &best_locator) { // The prior implementation of txindex was always in sync with block index // and presence was indicated with a boolean DB flag. If the flag is set, // this means the txindex from a previous version is valid and in sync with // the chain tip. The first step of the migration is to unset the flag and // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the // index entries are copied over in batches to the new database. Finally, // DB_TXINDEX_BLOCK is erased from the old database and the block hash is // written to the new database. // // Unsetting the boolean flag ensures that if the node is downgraded to a // previous version, it will not see a corrupted, partially migrated index // -- it will see that the txindex is disabled. When the node is upgraded // again, the migration will pick up where it left off and sync to the block // with hash DB_TXINDEX_BLOCK. bool f_legacy_flag = false; block_tree_db.ReadFlag("txindex", f_legacy_flag); if (f_legacy_flag) { if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) { return error("%s: cannot write block indicator", __func__); } if (!block_tree_db.WriteFlag("txindex", false)) { return error("%s: cannot write block index db flag", __func__); } } CBlockLocator locator; if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) { return true; } int64_t count = 0; uiInterface.InitMessage(_("Upgrading txindex database").translated); LogPrintf("Upgrading txindex database... [0%%]\n"); uiInterface.ShowProgress(_("Upgrading txindex database").translated, 0, true); int report_done = 0; const size_t batch_size = 1 << 24; // 16 MiB CDBBatch batch_newdb(*this); CDBBatch batch_olddb(block_tree_db); std::pair key; std::pair begin_key{DB_TXINDEX, uint256()}; std::pair prev_key = begin_key; bool interrupted = false; std::unique_ptr cursor(block_tree_db.NewIterator()); for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) { boost::this_thread::interruption_point(); if (ShutdownRequested()) { interrupted = true; break; } if (!cursor->GetKey(key)) { return error("%s: cannot get key from valid cursor", __func__); } if (key.first != DB_TXINDEX) { break; } // Log progress every 10%. if (++count % 256 == 0) { // Since txids are uniformly random and traversed in increasing // order, the high 16 bits of the ID can be used to estimate the // current progress. const uint256 &txid = key.second; uint32_t high_nibble = (static_cast(*(txid.begin() + 0)) << 8) + (static_cast(*(txid.begin() + 1)) << 0); int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5); uiInterface.ShowProgress(_("Upgrading txindex database").translated, percentage_done, true); if (report_done < percentage_done / 10) { LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done); report_done = percentage_done / 10; } } CDiskTxPos value; if (!cursor->GetValue(value)) { return error("%s: cannot parse txindex record", __func__); } batch_newdb.Write(key, value); batch_olddb.Erase(key); if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) { // NOTE: it's OK to delete the key pointed at by the current DB // cursor while iterating because LevelDB iterators are guaranteed // to provide a consistent view of the underlying data, like a // lightweight snapshot. WriteTxIndexMigrationBatches(*this, block_tree_db, batch_newdb, batch_olddb, prev_key, key); prev_key = key; } } // If these final DB batches complete the migration, write the best block // hash marker to the new database and delete from the old one. This signals // that the former is fully caught up to that point in the blockchain and // that all txindex entries have been removed from the latter. if (!interrupted) { batch_olddb.Erase(DB_TXINDEX_BLOCK); batch_newdb.Write(DB_BEST_BLOCK, locator); } WriteTxIndexMigrationBatches(*this, block_tree_db, batch_newdb, batch_olddb, begin_key, key); if (interrupted) { LogPrintf("[CANCELLED].\n"); return false; } uiInterface.ShowProgress("", 100, false); LogPrintf("[DONE].\n"); return true; } TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe) : m_db(std::make_unique(n_cache_size, f_memory, f_wipe)) {} TxIndex::~TxIndex() {} bool TxIndex::Init() { LOCK(cs_main); // Attempt to migrate txindex from the old database to the new one. Even if // chain_tip is null, the node could be reindexing and we still want to // delete txindex records in the old database. if (!m_db->MigrateData(*pblocktree, ::ChainActive().GetLocator())) { return false; } return BaseIndex::Init(); } bool TxIndex::WriteBlock(const CBlock &block, const CBlockIndex *pindex) { // Exclude genesis block transaction because outputs are not spendable. if (pindex->nHeight == 0) { return true; } CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); std::vector> vPos; vPos.reserve(block.vtx.size()); for (const auto &tx : block.vtx) { vPos.emplace_back(tx->GetId(), pos); pos.nTxOffset += ::GetSerializeSize(*tx, CLIENT_VERSION); } return m_db->WriteTxs(vPos); } BaseIndex::DB &TxIndex::GetDB() const { return *m_db; } bool TxIndex::FindTx(const TxId &txid, BlockHash &block_hash, CTransactionRef &tx) const { CDiskTxPos postx; if (!m_db->ReadTxPos(txid, postx)) { return false; } CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); if (file.IsNull()) { return error("%s: OpenBlockFile failed", __func__); } CBlockHeader header; try { file >> header; if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) { return error("%s: fseek(...) failed", __func__); } file >> tx; } catch (const std::exception &e) { return error("%s: Deserialize or I/O error - %s", __func__, e.what()); } if (tx->GetId() != txid) { return error("%s: txid mismatch", __func__); } block_hash = header.GetHash(); return true; } diff --git a/src/init.cpp b/src/init.cpp index db78e1406..8ead12346 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1,2924 +1,2924 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2018 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include #include #include #include #include #include #include #include #include