diff --git a/src/Makefile.am b/src/Makefile.am index 2dc0bfd59f..ce8c873b07 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,652 +1,654 @@ # Copyright (c) 2013-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. DIST_SUBDIRS = secp256k1 univalue AM_LDFLAGS = $(PTHREAD_CFLAGS) $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(SANITIZER_LDFLAGS) AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(ERROR_CXXFLAGS) $(SANITIZER_CXXFLAGS) AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) EXTRA_LIBRARIES = if EMBEDDED_UNIVALUE LIBUNIVALUE = univalue/libunivalue.la $(LIBUNIVALUE): $(wildcard univalue/lib/*) $(wildcard univalue/include/*) $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) else LIBUNIVALUE = $(UNIVALUE_LIBS) endif BITCOIN_INCLUDES=-I$(builddir) -I$(builddir)/obj $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) $(SSL_CFLAGS) BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) BITCOIN_SEEDER_INCLUDES = -I$(srcdir)/seeder BITCOIN_SEEDER_INCLUDES += $(BITCOIN_INCLUDES) LIBBITCOIN_SERVER=libbitcoin_server.a LIBBITCOIN_COMMON=libbitcoin_common.a LIBBITCOIN_CONSENSUS=libbitcoin_consensus.a LIBBITCOIN_CLI=libbitcoin_cli.a LIBBITCOIN_UTIL=libbitcoin_util.a LIBBITCOIN_CRYPTO_BASE=crypto/libbitcoin_crypto_base.a LIBBITCOINQT=qt/libbitcoinqt.a LIBSECP256K1=secp256k1/libsecp256k1.la if ENABLE_ZMQ LIBBITCOIN_ZMQ=libbitcoin_zmq.a endif if BUILD_BITCOIN_LIBS LIBBITCOINCONSENSUS=libbitcoinconsensus.la endif if BUILD_BITCOIN_SEEDER LIBBITCOIN_SEEDER=libbitcoin_seeder.a endif if ENABLE_WALLET LIBBITCOIN_WALLET=libbitcoin_wallet.a endif LIBBITCOIN_CRYPTO= $(LIBBITCOIN_CRYPTO_BASE) if ENABLE_SSE41 LIBBITCOIN_CRYPTO_SSE41 = crypto/libbitcoin_crypto_sse41.a LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SSE41) endif if ENABLE_AVX2 LIBBITCOIN_CRYPTO_AVX2 = crypto/libbitcoin_crypto_avx2.a LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_AVX2) endif if ENABLE_SHANI LIBBITCOIN_CRYPTO_SHANI = crypto/libbitcoin_crypto_shani.a LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SHANI) endif $(LIBSECP256K1): $(wildcard secp256k1/src/*) $(wildcard secp256k1/include/*) $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) # Make is not made aware of per-object dependencies to avoid limiting building parallelization # But to build the less dependent modules first, we manually select their order here: EXTRA_LIBRARIES += \ $(LIBBITCOIN_CRYPTO) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_CLI) \ $(LIBBITCOIN_SEEDER) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_ZMQ) lib_LTLIBRARIES = $(LIBBITCOINCONSENSUS) bin_PROGRAMS = noinst_PROGRAMS = TESTS = BENCHMARKS = if BUILD_BITCOIND bin_PROGRAMS += bitcoind endif if BUILD_BITCOIN_SEEDER bin_PROGRAMS += bitcoin-seeder endif if BUILD_BITCOIN_UTILS bin_PROGRAMS += bitcoin-cli bitcoin-tx endif .PHONY: FORCE check-symbols check-security # bitcoin core # BITCOIN_CORE_H = \ addrdb.h \ addrman.h \ avalanche.h \ base58.h \ bloom.h \ blockencodings.h \ blockfileinfo.h \ blockfilter.h \ blockindexworkcomparator.h \ blockstatus.h \ blockvalidity.h \ cashaddr.h \ cashaddrenc.h \ chain.h \ chainparams.h \ chainparamsbase.h \ chainparamsseeds.h \ checkpoints.h \ checkqueue.h \ clientversion.h \ coins.h \ compat.h \ compat/byteswap.h \ compat/endian.h \ compat/sanity.h \ compressor.h \ config.h \ consensus/activation.h \ consensus/consensus.h \ consensus/tx_verify.h \ core_io.h \ core_memusage.h \ cuckoocache.h \ diskblockpos.h \ dstencode.h \ fs.h \ globals.h \ httprpc.h \ httpserver.h \ indirectmap.h \ init.h \ interfaces/handler.h \ interfaces/node.h \ key.h \ keystore.h \ dbwrapper.h \ limitedmap.h \ logging.h \ memusage.h \ merkleblock.h \ miner.h \ net.h \ net_processing.h \ netaddress.h \ netbase.h \ netmessagemaker.h \ noui.h \ policy/fees.h \ policy/policy.h \ pow.h \ protocol.h \ radix.h \ random.h \ rcu.h \ reverse_iterator.h \ reverselock.h \ rpc/blockchain.h \ rpc/client.h \ rpc/command.h \ rpc/jsonrpcrequest.h \ rpc/mining.h \ rpc/misc.h \ rpc/protocol.h \ rpc/rawtransaction.h \ rpc/server.h \ rpc/tojson.h \ rpc/register.h \ rwcollection.h \ scheduler.h \ script/scriptcache.h \ script/sigcache.h \ script/sign.h \ script/standard.h \ script/ismine.h \ streams.h \ support/allocators/secure.h \ support/allocators/zeroafterfree.h \ support/cleanse.h \ support/events.h \ support/lockedpool.h \ sync.h \ threadsafety.h \ threadinterrupt.h \ timedata.h \ torcontrol.h \ txdb.h \ txmempool.h \ ui_interface.h \ undo.h \ util.h \ utilmoneystr.h \ utiltime.h \ validation.h \ validationinterface.h \ versionbits.h \ walletinitinterface.h \ wallet/coincontrol.h \ wallet/crypter.h \ wallet/db.h \ wallet/finaltx.h \ wallet/rpcdump.h \ wallet/fees.h \ wallet/rpcwallet.h \ wallet/wallet.h \ wallet/walletdb.h \ wallet/walletutil.h \ warnings.h \ zmq/zmqabstractnotifier.h \ zmq/zmqconfig.h\ zmq/zmqnotificationinterface.h \ zmq/zmqpublishnotifier.h obj/build.h: FORCE @$(MKDIR_P) "$(builddir)/obj" @$(top_srcdir)/share/genbuild.sh "$(abs_top_builddir)/src/obj/build.h" \ "$(abs_top_srcdir)" libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h # server: shared between bitcoind and bitcoin-qt libbitcoin_server_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_server_a_SOURCES = \ addrman.cpp \ addrdb.cpp \ avalanche.cpp \ bloom.cpp \ blockencodings.cpp \ blockfilter.cpp \ chain.cpp \ checkpoints.cpp \ config.cpp \ consensus/activation.cpp \ consensus/tx_verify.cpp \ globals.cpp \ httprpc.cpp \ httpserver.cpp \ init.cpp \ interfaces/handler.cpp \ interfaces/node.cpp \ dbwrapper.cpp \ merkleblock.cpp \ miner.cpp \ net.cpp \ net_processing.cpp \ noui.cpp \ policy/fees.cpp \ policy/policy.cpp \ pow.cpp \ rest.cpp \ rpc/abc.cpp \ rpc/blockchain.cpp \ rpc/command.cpp \ rpc/jsonrpcrequest.cpp \ rpc/mining.cpp \ rpc/misc.cpp \ rpc/net.cpp \ rpc/rawtransaction.cpp \ rpc/server.cpp \ script/scriptcache.cpp \ script/sigcache.cpp \ script/ismine.cpp \ timedata.cpp \ torcontrol.cpp \ txdb.cpp \ txmempool.cpp \ ui_interface.cpp \ validation.cpp \ validationinterface.cpp \ $(BITCOIN_CORE_H) if ENABLE_ZMQ libbitcoin_zmq_a_CPPFLAGS = $(BITCOIN_INCLUDES) $(ZMQ_CFLAGS) libbitcoin_zmq_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_zmq_a_SOURCES = \ zmq/zmqabstractnotifier.cpp \ zmq/zmqnotificationinterface.cpp \ zmq/zmqpublishnotifier.cpp endif # wallet: shared between bitcoind and bitcoin-qt, but only linked # when wallet enabled libbitcoin_wallet_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_a_SOURCES = \ wallet/crypter.cpp \ wallet/db.cpp \ wallet/finaltx.cpp \ wallet/fees.cpp \ wallet/init.cpp \ wallet/rpcdump.cpp \ wallet/rpcwallet.cpp \ wallet/wallet.cpp \ wallet/walletdb.cpp \ wallet/walletutil.cpp \ $(BITCOIN_CORE_H) # crypto primitives library crypto_libbitcoin_crypto_base_a_CPPFLAGS = $(AM_CPPFLAGS) crypto_libbitcoin_crypto_base_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crypto_libbitcoin_crypto_base_a_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ crypto/chacha20.h \ crypto/chacha20.cpp \ crypto/common.h \ crypto/hmac_sha256.cpp \ crypto/hmac_sha256.h \ crypto/hmac_sha512.cpp \ crypto/hmac_sha512.h \ crypto/ripemd160.cpp \ crypto/ripemd160.h \ crypto/sha1.cpp \ crypto/sha1.h \ crypto/sha256.cpp \ crypto/sha256.h \ crypto/sha512.cpp \ - crypto/sha512.h + crypto/sha512.h \ + crypto/siphash.cpp \ + crypto/siphash.h if USE_ASM crypto_libbitcoin_crypto_base_a_SOURCES += crypto/sha256_sse4.cpp endif crypto_libbitcoin_crypto_sse41_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crypto_libbitcoin_crypto_sse41_a_CPPFLAGS = $(AM_CPPFLAGS) crypto_libbitcoin_crypto_sse41_a_CXXFLAGS += $(SSE41_CXXFLAGS) crypto_libbitcoin_crypto_sse41_a_CPPFLAGS += -DENABLE_SSE41 crypto_libbitcoin_crypto_sse41_a_SOURCES = crypto/sha256_sse41.cpp crypto_libbitcoin_crypto_avx2_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crypto_libbitcoin_crypto_avx2_a_CPPFLAGS = $(AM_CPPFLAGS) crypto_libbitcoin_crypto_avx2_a_CXXFLAGS += $(AVX2_CXXFLAGS) crypto_libbitcoin_crypto_avx2_a_CPPFLAGS += -DENABLE_AVX2 crypto_libbitcoin_crypto_avx2_a_SOURCES = crypto/sha256_avx2.cpp crypto_libbitcoin_crypto_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crypto_libbitcoin_crypto_shani_a_CPPFLAGS = $(AM_CPPFLAGS) crypto_libbitcoin_crypto_shani_a_CXXFLAGS += $(SHANI_CXXFLAGS) crypto_libbitcoin_crypto_shani_a_CPPFLAGS += -DENABLE_SHANI crypto_libbitcoin_crypto_shani_a_SOURCES = crypto/sha256_shani.cpp # consensus: shared between all executables that validate any consensus rules. libbitcoin_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_consensus_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_consensus_a_SOURCES = \ amount.h \ arith_uint256.cpp \ arith_uint256.h \ consensus/merkle.cpp \ consensus/merkle.h \ consensus/params.h \ consensus/validation.h \ feerate.h \ hash.cpp \ hash.h \ prevector.h \ primitives/block.cpp \ primitives/block.h \ primitives/transaction.cpp \ primitives/transaction.h \ primitives/txid.h \ pubkey.cpp \ pubkey.h \ script/bitcoinconsensus.cpp \ script/sighashtype.h \ script/interpreter.cpp \ script/interpreter.h \ script/script.cpp \ script/script.h \ script/script_error.cpp \ script/script_error.h \ script/script_flags.h \ script/sigencoding.cpp \ script/sigencoding.h \ serialize.h \ tinyformat.h \ uint256.cpp \ uint256.h \ utilstrencodings.cpp \ utilstrencodings.h \ version.h # common: shared between bitcoind, and bitcoin-qt and non-server tools libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ amount.cpp \ base58.cpp \ cashaddr.cpp \ cashaddrenc.cpp \ chainparams.cpp \ config.cpp \ coins.cpp \ compressor.cpp \ dstencode.cpp \ feerate.cpp \ globals.cpp \ core_read.cpp \ core_write.cpp \ key.cpp \ keystore.cpp \ netaddress.cpp \ netbase.cpp \ protocol.cpp \ scheduler.cpp \ script/sign.cpp \ script/standard.cpp \ warnings.cpp \ $(BITCOIN_CORE_H) # util: shared between all executables. # This library *must* be included to make sure that the glibc # backward-compatibility objects and their sanity checks are linked. libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ support/lockedpool.cpp \ chainparamsbase.cpp \ clientversion.cpp \ compat/glibc_sanity.cpp \ compat/glibcxx_sanity.cpp \ compat/strnlen.cpp \ fs.cpp \ logging.cpp \ random.cpp \ rcu.cpp \ rpc/protocol.cpp \ support/cleanse.cpp \ sync.cpp \ threadinterrupt.cpp \ uint256.cpp \ uint256.h \ util.cpp \ utilmoneystr.cpp \ utilstrencodings.cpp \ utiltime.cpp \ $(BITCOIN_CORE_H) if GLIBC_BACK_COMPAT libbitcoin_util_a_SOURCES += compat/glibc_compat.cpp AM_LDFLAGS += $(COMPAT_LDFLAGS) endif # cli: shared between bitcoin-cli and bitcoin-qt libbitcoin_cli_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_cli_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_cli_a_SOURCES = \ rpc/client.cpp \ $(BITCOIN_CORE_H) # seeder library libbitcoin_seeder_a_CPPFLAGS = $(AM_CPPFLAGS) $(PIE_FLAGS) $(BITCOIN_SEEDER_INCLUDES) libbitcoin_seeder_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_seeder_a_SOURCES = \ seeder/bitcoin.cpp \ seeder/bitcoin.h \ seeder/db.cpp \ seeder/db.h \ seeder/dns.cpp \ seeder/dns.h \ seeder/strlcpy.h \ seeder/util.h nodist_libbitcoin_util_a_SOURCES = $(srcdir)/obj/build.h # # bitcoind binary # bitcoind_SOURCES = bitcoind.cpp bitcoind_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) bitcoind_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bitcoind_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) if TARGET_WINDOWS bitcoind_SOURCES += bitcoind-res.rc endif bitcoind_LDADD = \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_COMMON) \ $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_ZMQ) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) # bitcoin-cli binary # bitcoin_cli_SOURCES = bitcoin-cli.cpp bitcoin_cli_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CFLAGS) bitcoin_cli_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bitcoin_cli_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) if TARGET_WINDOWS bitcoin_cli_SOURCES += bitcoin-cli-res.rc endif bitcoin_cli_LDADD = \ $(LIBBITCOIN_CLI) \ $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) bitcoin_cli_LDADD += $(BOOST_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(EVENT_LIBS) # # bitcoin-seeder binary # bitcoin_seeder_SOURCES = seeder/main.cpp bitcoin_seeder_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_SEEDER_INCLUDES) bitcoin_seeder_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bitcoin_seeder_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) bitcoin_seeder_LDADD = \ $(LIBBITCOIN_SEEDER) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) bitcoin_seeder_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) # # bitcoin-tx binary # bitcoin_tx_SOURCES = bitcoin-tx.cpp bitcoin_tx_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) bitcoin_tx_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bitcoin_tx_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) if TARGET_WINDOWS bitcoin_tx_SOURCES += bitcoin-tx-res.rc endif bitcoin_tx_LDADD = \ $(LIBUNIVALUE) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBSECP256K1) bitcoin_tx_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) # # bitcoinconsensus library # if BUILD_BITCOIN_LIBS include_HEADERS = script/bitcoinconsensus.h libbitcoinconsensus_la_SOURCES = $(crypto_libbitcoin_crypto_base_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) if GLIBC_BACK_COMPAT libbitcoinconsensus_la_SOURCES += compat/glibc_compat.cpp endif libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1) libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL libbitcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) endif # CTAES_DIST = crypto/ctaes/bench.c CTAES_DIST += crypto/ctaes/ctaes.c CTAES_DIST += crypto/ctaes/ctaes.h CTAES_DIST += crypto/ctaes/README.md CTAES_DIST += crypto/ctaes/test.c CLEANFILES = $(EXTRA_LIBRARIES) CLEANFILES += *.gcda *.gcno CLEANFILES += compat/*.gcda compat/*.gcno CLEANFILES += consensus/*.gcda consensus/*.gcno CLEANFILES += crypto/*.gcda crypto/*.gcno CLEANFILES += policy/*.gcda policy/*.gcno CLEANFILES += primitives/*.gcda primitives/*.gcno CLEANFILES += script/*.gcda script/*.gcno CLEANFILES += support/*.gcda support/*.gcno CLEANFILES += univalue/*.gcda univalue/*.gcno CLEANFILES += wallet/*.gcda wallet/*.gcno CLEANFILES += wallet/test/*.gcda wallet/test/*.gcno CLEANFILES += zmq/*.gcda zmq/*.gcno DISTCLEANFILES = obj/build.h EXTRA_DIST = $(CTAES_DIST) clean-local: -$(MAKE) -C secp256k1 clean -$(MAKE) -C univalue clean -rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno -rm -rf test/__pycache__ .rc.o: @test -f $(WINDRES) ## FIXME: How to get the appropriate modulename_CPPFLAGS in here? $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ .mm.o: $(AM_V_CXX) $(OBJCXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ $(CPPFLAGS) $(AM_CXXFLAGS) $(QT_INCLUDES) $(AM_CXXFLAGS) $(PIE_FLAGS) $(CXXFLAGS) -c -o $@ $< check-symbols: $(bin_PROGRAMS) if GLIBC_BACK_COMPAT @echo "Checking glibc back compat..." $(AM_V_at) READELF=$(READELF) CPPFILT=$(CPPFILT) $(top_srcdir)/contrib/devtools/symbol-check.py < $(bin_PROGRAMS) endif check-security: $(bin_PROGRAMS) if HARDEN @echo "Checking binary security..." $(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) $(top_srcdir)/contrib/devtools/security-check.py < $(bin_PROGRAMS) endif %.pb.cc %.pb.h: %.proto @test -f $(PROTOC) $(AM_V_GEN) $(PROTOC) --cpp_out=$(@D) --proto_path=$( #include #include #include #include #include #include +#include #include #include #include #include /* Number of bytes to hash per iteration */ static const uint64_t BUFFER_SIZE = 1000 * 1000; static void RIPEMD160(benchmark::State &state) { uint8_t hash[CRIPEMD160::OUTPUT_SIZE]; std::vector in(BUFFER_SIZE, 0); while (state.KeepRunning()) CRIPEMD160().Write(in.data(), in.size()).Finalize(hash); } static void SHA1(benchmark::State &state) { uint8_t hash[CSHA1::OUTPUT_SIZE]; std::vector in(BUFFER_SIZE, 0); while (state.KeepRunning()) CSHA1().Write(in.data(), in.size()).Finalize(hash); } static void SHA256(benchmark::State &state) { uint8_t hash[CSHA256::OUTPUT_SIZE]; std::vector in(BUFFER_SIZE, 0); while (state.KeepRunning()) CSHA256().Write(in.data(), in.size()).Finalize(hash); } static void SHA256_32b(benchmark::State &state) { std::vector in(32, 0); while (state.KeepRunning()) { CSHA256().Write(in.data(), in.size()).Finalize(in.data()); } } static void SHA256D64_1024(benchmark::State &state) { std::vector in(64 * 1024, 0); while (state.KeepRunning()) { SHA256D64(in.data(), in.data(), 1024); } } static void SHA512(benchmark::State &state) { uint8_t hash[CSHA512::OUTPUT_SIZE]; std::vector in(BUFFER_SIZE, 0); while (state.KeepRunning()) CSHA512().Write(in.data(), in.size()).Finalize(hash); } static void SipHash_32b(benchmark::State &state) { uint256 x; uint64_t k1 = 0; while (state.KeepRunning()) { *((uint64_t *)x.begin()) = SipHashUint256(0, ++k1, x); } } static void FastRandom_32bit(benchmark::State &state) { FastRandomContext rng(true); uint32_t x = 0; while (state.KeepRunning()) { x += rng.rand32(); } } static void FastRandom_1bit(benchmark::State &state) { FastRandomContext rng(true); uint32_t x = 0; while (state.KeepRunning()) { x += rng.randbool(); } } BENCHMARK(RIPEMD160, 440); BENCHMARK(SHA1, 570); BENCHMARK(SHA256, 340); BENCHMARK(SHA512, 330); BENCHMARK(SHA256_32b, 4700 * 1000); BENCHMARK(SipHash_32b, 40 * 1000 * 1000); BENCHMARK(SHA256D64_1024, 7400); BENCHMARK(FastRandom_32bit, 110 * 1000 * 1000); BENCHMARK(FastRandom_1bit, 440 * 1000 * 1000); diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index da1a4dd0ee..23b1e1da90 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -1,270 +1,271 @@ // Copyright (c) 2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include -#include +#include +#include #include #include #include #include #include #include CBlockHeaderAndShortTxIDs::CBlockHeaderAndShortTxIDs(const CBlock &block) : nonce(GetRand(std::numeric_limits::max())), shorttxids(block.vtx.size() - 1), prefilledtxn(1), header(block) { FillShortTxIDSelector(); // TODO: Use our mempool prior to block acceptance to predictively fill more // than just the coinbase. prefilledtxn[0] = {0, block.vtx[0]}; for (size_t i = 1; i < block.vtx.size(); i++) { const CTransaction &tx = *block.vtx[i]; shorttxids[i - 1] = GetShortID(tx.GetHash()); } } void CBlockHeaderAndShortTxIDs::FillShortTxIDSelector() const { CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << header << nonce; CSHA256 hasher; hasher.Write((uint8_t *)&(*stream.begin()), stream.end() - stream.begin()); uint256 shorttxidhash; hasher.Finalize(shorttxidhash.begin()); shorttxidk0 = shorttxidhash.GetUint64(0); shorttxidk1 = shorttxidhash.GetUint64(1); } uint64_t CBlockHeaderAndShortTxIDs::GetShortID(const uint256 &txhash) const { static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids calculation assumes 6-byte shorttxids"); return SipHashUint256(shorttxidk0, shorttxidk1, txhash) & 0xffffffffffffL; } ReadStatus PartiallyDownloadedBlock::InitData( const CBlockHeaderAndShortTxIDs &cmpctblock, const std::vector> &extra_txns) { if (cmpctblock.header.IsNull() || (cmpctblock.shorttxids.empty() && cmpctblock.prefilledtxn.empty())) { return READ_STATUS_INVALID; } if (cmpctblock.shorttxids.size() + cmpctblock.prefilledtxn.size() > config->GetMaxBlockSize() / MIN_TRANSACTION_SIZE) { return READ_STATUS_INVALID; } assert(header.IsNull() && txns_available.empty()); header = cmpctblock.header; txns_available.resize(cmpctblock.BlockTxCount()); int64_t lastprefilledindex = -1; for (size_t i = 0; i < cmpctblock.prefilledtxn.size(); i++) { auto &prefilledtxn = cmpctblock.prefilledtxn[i]; if (prefilledtxn.tx->IsNull()) { return READ_STATUS_INVALID; } // index is a uint32_t, so can't overflow here. lastprefilledindex += prefilledtxn.index + 1; if (lastprefilledindex > std::numeric_limits::max()) { return READ_STATUS_INVALID; } if (uint32_t(lastprefilledindex) > cmpctblock.shorttxids.size() + i) { // If we are inserting a tx at an index greater than our full list // of shorttxids plus the number of prefilled txn we've inserted, // then we have txn for which we have neither a prefilled txn or a // shorttxid! return READ_STATUS_INVALID; } txns_available[lastprefilledindex] = prefilledtxn.tx; } prefilled_count = cmpctblock.prefilledtxn.size(); // Calculate map of txids -> positions and check mempool to see what we have // (or don't). Because well-formed cmpctblock messages will have a // (relatively) uniform distribution of short IDs, any highly-uneven // distribution of elements can be safely treated as a READ_STATUS_FAILED. std::unordered_map shorttxids( cmpctblock.shorttxids.size()); uint32_t index_offset = 0; for (size_t i = 0; i < cmpctblock.shorttxids.size(); i++) { while (txns_available[i + index_offset]) { index_offset++; } shorttxids[cmpctblock.shorttxids[i]] = i + index_offset; // To determine the chance that the number of entries in a bucket // exceeds N, we use the fact that the number of elements in a single // bucket is binomially distributed (with n = the number of shorttxids // S, and p = 1 / the number of buckets), that in the worst case the // number of buckets is equal to S (due to std::unordered_map having a // default load factor of 1.0), and that the chance for any bucket to // exceed N elements is at most buckets * (the chance that any given // bucket is above N elements). Thus: P(max_elements_per_bucket > N) <= // S * (1 - cdf(binomial(n=S,p=1/S), N)). If we assume blocks of up to // 16000, allowing 12 elements per bucket should only fail once per ~1 // million block transfers (per peer and connection). if (shorttxids.bucket_size( shorttxids.bucket(cmpctblock.shorttxids[i])) > 12) { return READ_STATUS_FAILED; } } // TODO: in the shortid-collision case, we should instead request both // transactions which collided. Falling back to full-block-request here is // overkill. if (shorttxids.size() != cmpctblock.shorttxids.size()) { // Short ID collision return READ_STATUS_FAILED; } std::vector have_txn(txns_available.size()); { LOCK(pool->cs); const std::vector> &vTxHashes = pool->vTxHashes; for (auto txHash : vTxHashes) { uint64_t shortid = cmpctblock.GetShortID(txHash.first); std::unordered_map::iterator idit = shorttxids.find(shortid); if (idit != shorttxids.end()) { if (!have_txn[idit->second]) { txns_available[idit->second] = txHash.second->GetSharedTx(); have_txn[idit->second] = true; mempool_count++; } else { // If we find two mempool txn that match the short id, just // request it. This should be rare enough that the extra // bandwidth doesn't matter, but eating a round-trip due to // FillBlock failure would be annoying. if (txns_available[idit->second]) { txns_available[idit->second].reset(); mempool_count--; } } } // Though ideally we'd continue scanning for the // two-txn-match-shortid case, the performance win of an early exit // here is too good to pass up and worth the extra risk. if (mempool_count == shorttxids.size()) { break; } } } for (auto &extra_txn : extra_txns) { uint64_t shortid = cmpctblock.GetShortID(extra_txn.first); std::unordered_map::iterator idit = shorttxids.find(shortid); if (idit != shorttxids.end()) { if (!have_txn[idit->second]) { txns_available[idit->second] = extra_txn.second; have_txn[idit->second] = true; mempool_count++; extra_count++; } else { // If we find two mempool/extra txn that match the short id, // just request it. This should be rare enough that the extra // bandwidth doesn't matter, but eating a round-trip due to // FillBlock failure would be annoying. Note that we dont want // duplication between extra_txns and mempool to trigger this // case, so we compare hashes first. if (txns_available[idit->second] && txns_available[idit->second]->GetHash() != extra_txn.second->GetHash()) { txns_available[idit->second].reset(); mempool_count--; extra_count--; } } } // Though ideally we'd continue scanning for the two-txn-match-shortid // case, the performance win of an early exit here is too good to pass // up and worth the extra risk. if (mempool_count == shorttxids.size()) { break; } } LogPrint(BCLog::CMPCTBLOCK, "Initialized PartiallyDownloadedBlock for block %s using a " "cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), GetSerializeSize(cmpctblock, SER_NETWORK, PROTOCOL_VERSION)); return READ_STATUS_OK; } bool PartiallyDownloadedBlock::IsTxAvailable(size_t index) const { assert(!header.IsNull()); assert(index < txns_available.size()); return txns_available[index] ? true : false; } ReadStatus PartiallyDownloadedBlock::FillBlock( CBlock &block, const std::vector &vtx_missing) { assert(!header.IsNull()); uint256 hash = header.GetHash(); block = header; block.vtx.resize(txns_available.size()); size_t tx_missing_offset = 0; for (size_t i = 0; i < txns_available.size(); i++) { auto &txn_available = txns_available[i]; if (!txn_available) { if (vtx_missing.size() <= tx_missing_offset) { return READ_STATUS_INVALID; } block.vtx[i] = vtx_missing[tx_missing_offset++]; } else { block.vtx[i] = std::move(txn_available); } } // Make sure we can't call FillBlock again. header.SetNull(); txns_available.clear(); if (vtx_missing.size() != tx_missing_offset) { return READ_STATUS_INVALID; } CValidationState state; if (!CheckBlock(*config, block, state)) { // TODO: We really want to just check merkle tree manually here, but // that is expensive, and CheckBlock caches a block's "checked-status" // (in the CBlock?). CBlock should be able to check its own merkle root // and cache that check. if (state.CorruptionPossible()) { // Possible Short ID collision. return READ_STATUS_FAILED; } return READ_STATUS_CHECKBLOCK_FAILED; } LogPrint(BCLog::CMPCTBLOCK, "Successfully reconstructed block %s with %lu txn prefilled, %lu " "txn from mempool (incl at least %lu from extra pool) and %lu txn " "requested\n", hash.ToString(), prefilled_count, mempool_count, extra_count, vtx_missing.size()); if (vtx_missing.size() < 5) { for (const auto &tx : vtx_missing) { LogPrint(BCLog::CMPCTBLOCK, "Reconstructed block %s required tx %s\n", hash.ToString(), tx->GetId().ToString()); } } return READ_STATUS_OK; } diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 29d735bd34..98a16fcbfa 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -1,198 +1,198 @@ // Copyright (c) 2018 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include -#include +#include #include /// SerType used to serialize parameters in GCS filter encoding. static constexpr int GCS_SER_TYPE = SER_NETWORK; /// Protocol version used to serialize parameters in GCS filter encoding. static constexpr int GCS_SER_VERSION = 0; template static void GolombRiceEncode(BitStreamWriter &bitwriter, uint8_t P, uint64_t x) { // Write quotient as unary-encoded: q 1's followed by one 0. uint64_t q = x >> P; while (q > 0) { int nbits = q <= 64 ? static_cast(q) : 64; bitwriter.Write(~0ULL, nbits); q -= nbits; } bitwriter.Write(0, 1); // Write the remainder in P bits. Since the remainder is just the bottom // P bits of x, there is no need to mask first. bitwriter.Write(x, P); } template static uint64_t GolombRiceDecode(BitStreamReader &bitreader, uint8_t P) { // Read unary-encoded quotient: q 1's followed by one 0. uint64_t q = 0; while (bitreader.Read(1) == 1) { ++q; } uint64_t r = bitreader.Read(P); return (q << P) + r; } // Map a value x that is uniformly distributed in the range [0, 2^64) to a // value uniformly distributed in [0, n) by returning the upper 64 bits of // x * n. // // See: // https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ static uint64_t MapIntoRange(uint64_t x, uint64_t n) { #ifdef __SIZEOF_INT128__ return (static_cast(x) * static_cast(n)) >> 64; #else // To perform the calculation on 64-bit numbers without losing the // result to overflow, split the numbers into the most significant and // least significant 32 bits and perform multiplication piece-wise. // // See: https://stackoverflow.com/a/26855440 uint64_t x_hi = x >> 32; uint64_t x_lo = x & 0xFFFFFFFF; uint64_t n_hi = n >> 32; uint64_t n_lo = n & 0xFFFFFFFF; uint64_t ac = x_hi * n_hi; uint64_t ad = x_hi * n_lo; uint64_t bc = x_lo * n_hi; uint64_t bd = x_lo * n_lo; uint64_t mid34 = (bd >> 32) + (bc & 0xFFFFFFFF) + (ad & 0xFFFFFFFF); uint64_t upper64 = ac + (bc >> 32) + (ad >> 32) + (mid34 >> 32); return upper64; #endif } uint64_t GCSFilter::HashToRange(const Element &element) const { uint64_t hash = CSipHasher(m_siphash_k0, m_siphash_k1) .Write(element.data(), element.size()) .Finalize(); return MapIntoRange(hash, m_F); } std::vector GCSFilter::BuildHashedSet(const ElementSet &elements) const { std::vector hashed_elements; hashed_elements.reserve(elements.size()); for (const Element &element : elements) { hashed_elements.push_back(HashToRange(element)); } std::sort(hashed_elements.begin(), hashed_elements.end()); return hashed_elements; } GCSFilter::GCSFilter(uint64_t siphash_k0, uint64_t siphash_k1, uint8_t P, uint32_t M) : m_siphash_k0(siphash_k0), m_siphash_k1(siphash_k1), m_P(P), m_M(M), m_N(0), m_F(0) {} GCSFilter::GCSFilter(uint64_t siphash_k0, uint64_t siphash_k1, uint8_t P, uint32_t M, std::vector encoded_filter) : GCSFilter(siphash_k0, siphash_k1, P, M) { m_encoded = std::move(encoded_filter); VectorReader stream(GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0); uint64_t N = ReadCompactSize(stream); m_N = static_cast(N); if (m_N != N) { throw std::ios_base::failure("N must be <2^32"); } m_F = static_cast(m_N) * static_cast(m_M); // Verify that the encoded filter contains exactly N elements. If it has too // much or too little data, a std::ios_base::failure exception will be // raised. BitStreamReader bitreader(stream); for (uint64_t i = 0; i < m_N; ++i) { GolombRiceDecode(bitreader, m_P); } if (!stream.empty()) { throw std::ios_base::failure("encoded_filter contains excess data"); } } GCSFilter::GCSFilter(uint64_t siphash_k0, uint64_t siphash_k1, uint8_t P, uint32_t M, const ElementSet &elements) : GCSFilter(siphash_k0, siphash_k1, P, M) { size_t N = elements.size(); m_N = static_cast(N); if (m_N != N) { throw std::invalid_argument("N must be <2^32"); } m_F = static_cast(m_N) * static_cast(m_M); CVectorWriter stream(GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0); WriteCompactSize(stream, m_N); if (elements.empty()) { return; } BitStreamWriter bitwriter(stream); uint64_t last_value = 0; for (uint64_t value : BuildHashedSet(elements)) { uint64_t delta = value - last_value; GolombRiceEncode(bitwriter, m_P, delta); last_value = value; } bitwriter.Flush(); } bool GCSFilter::MatchInternal(const uint64_t *element_hashes, size_t size) const { VectorReader stream(GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0); // Seek forward by size of N uint64_t N = ReadCompactSize(stream); assert(N == m_N); BitStreamReader bitreader(stream); uint64_t value = 0; size_t hashes_index = 0; for (uint32_t i = 0; i < m_N; ++i) { uint64_t delta = GolombRiceDecode(bitreader, m_P); value += delta; while (true) { if (hashes_index == size) { return false; } else if (element_hashes[hashes_index] == value) { return true; } else if (element_hashes[hashes_index] > value) { break; } hashes_index++; } } return false; } bool GCSFilter::Match(const Element &element) const { uint64_t query = HashToRange(element); return MatchInternal(&query, 1); } bool GCSFilter::MatchAny(const ElementSet &elements) const { const std::vector queries = BuildHashedSet(elements); return MatchInternal(queries.data(), queries.size()); } diff --git a/src/coins.cpp b/src/coins.cpp index 85d94b805b..3483affa36 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -1,344 +1,345 @@ // Copyright (c) 2012-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include +#include #include bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } bool CCoinsView::HaveCoin(const COutPoint &outpoint) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } std::vector CCoinsView::GetHeadBlocks() const { return std::vector(); } bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return false; } CCoinsViewCursor *CCoinsView::Cursor() const { return nullptr; } CCoinsViewBacked::CCoinsViewBacked(CCoinsView *viewIn) : base(viewIn) {} bool CCoinsViewBacked::GetCoin(const COutPoint &outpoint, Coin &coin) const { return base->GetCoin(outpoint, coin); } bool CCoinsViewBacked::HaveCoin(const COutPoint &outpoint) const { return base->HaveCoin(outpoint); } uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); } std::vector CCoinsViewBacked::GetHeadBlocks() const { return base->GetHeadBlocks(); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); } size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); } SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), cachedCoinsUsage(0) {} size_t CCoinsViewCache::DynamicMemoryUsage() const { return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; } CCoinsMap::iterator CCoinsViewCache::FetchCoin(const COutPoint &outpoint) const { CCoinsMap::iterator it = cacheCoins.find(outpoint); if (it != cacheCoins.end()) { return it; } Coin tmp; if (!base->GetCoin(outpoint, tmp)) { return cacheCoins.end(); } CCoinsMap::iterator ret = cacheCoins .emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::forward_as_tuple(std::move(tmp))) .first; if (ret->second.coin.IsSpent()) { // The parent only has an empty entry for this outpoint; we can consider // our version as fresh. ret->second.flags = CCoinsCacheEntry::FRESH; } cachedCoinsUsage += ret->second.coin.DynamicMemoryUsage(); return ret; } bool CCoinsViewCache::GetCoin(const COutPoint &outpoint, Coin &coin) const { CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) { return false; } coin = it->second.coin; return true; } void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin coin, bool possible_overwrite) { assert(!coin.IsSpent()); if (coin.GetTxOut().scriptPubKey.IsUnspendable()) { return; } CCoinsMap::iterator it; bool inserted; std::tie(it, inserted) = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::tuple<>()); bool fresh = false; if (!inserted) { cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); } if (!possible_overwrite) { if (!it->second.coin.IsSpent()) { throw std::logic_error( "Adding new coin that replaces non-pruned entry"); } fresh = !(it->second.flags & CCoinsCacheEntry::DIRTY); } it->second.coin = std::move(coin); it->second.flags |= CCoinsCacheEntry::DIRTY | (fresh ? CCoinsCacheEntry::FRESH : 0); cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); } void AddCoins(CCoinsViewCache &cache, const CTransaction &tx, int nHeight, bool check) { bool fCoinbase = tx.IsCoinBase(); const TxId txid = tx.GetId(); for (size_t i = 0; i < tx.vout.size(); ++i) { const COutPoint outpoint(txid, i); bool overwrite = check ? cache.HaveCoin(outpoint) : fCoinbase; // Always set the possible_overwrite flag to AddCoin for coinbase txn, // in order to correctly deal with the pre-BIP30 occurrences of // duplicate coinbase transactions. cache.AddCoin(outpoint, Coin(tx.vout[i], nHeight, fCoinbase), overwrite); } } bool CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin *moveout) { CCoinsMap::iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) { return false; } cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); if (moveout) { *moveout = std::move(it->second.coin); } if (it->second.flags & CCoinsCacheEntry::FRESH) { cacheCoins.erase(it); } else { it->second.flags |= CCoinsCacheEntry::DIRTY; it->second.coin.Clear(); } return true; } static const Coin coinEmpty; const Coin &CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const { CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) { return coinEmpty; } return it->second.coin; } bool CCoinsViewCache::HaveCoin(const COutPoint &outpoint) const { CCoinsMap::const_iterator it = FetchCoin(outpoint); return it != cacheCoins.end() && !it->second.coin.IsSpent(); } bool CCoinsViewCache::HaveCoinInCache(const COutPoint &outpoint) const { CCoinsMap::const_iterator it = cacheCoins.find(outpoint); return it != cacheCoins.end(); } uint256 CCoinsViewCache::GetBestBlock() const { if (hashBlock.IsNull()) { hashBlock = base->GetBestBlock(); } return hashBlock; } void CCoinsViewCache::SetBestBlock(const uint256 &hashBlockIn) { hashBlock = hashBlockIn; } bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn) { for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { // Ignore non-dirty entries (optimization). if (it->second.flags & CCoinsCacheEntry::DIRTY) { CCoinsMap::iterator itUs = cacheCoins.find(it->first); if (itUs == cacheCoins.end()) { // The parent cache does not have an entry, while the child does // We can ignore it if it's both FRESH and pruned in the child if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coin.IsSpent())) { // Otherwise we will need to create it in the parent and // move the data up and mark it as dirty CCoinsCacheEntry &entry = cacheCoins[it->first]; entry.coin = std::move(it->second.coin); cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); entry.flags = CCoinsCacheEntry::DIRTY; // We can mark it FRESH in the parent if it was FRESH in the // child. Otherwise it might have just been flushed from the // parent's cache and already exist in the grandparent if (it->second.flags & CCoinsCacheEntry::FRESH) entry.flags |= CCoinsCacheEntry::FRESH; } } else { // Assert that the child cache entry was not marked FRESH if the // parent cache entry has unspent outputs. If this ever happens, // it means the FRESH flag was misapplied and there is a logic // error in the calling code. if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coin.IsSpent()) throw std::logic_error("FRESH flag misapplied to cache " "entry for base transaction with " "spendable outputs"); // Found the entry in the parent cache if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coin.IsSpent()) { // The grandparent does not have an entry, and the child is // modified and being pruned. This means we can just delete // it from the parent. cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); cacheCoins.erase(itUs); } else { // A normal modification. cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); itUs->second.coin = std::move(it->second.coin); cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); itUs->second.flags |= CCoinsCacheEntry::DIRTY; // NOTE: It is possible the child has a FRESH flag here in // the event the entry we found in the parent is pruned. But // we must not copy that FRESH flag to the parent as that // pruned state likely still needs to be communicated to the // grandparent. } } } CCoinsMap::iterator itOld = it++; mapCoins.erase(itOld); } hashBlock = hashBlockIn; return true; } bool CCoinsViewCache::Flush() { bool fOk = base->BatchWrite(cacheCoins, hashBlock); cacheCoins.clear(); cachedCoinsUsage = 0; return fOk; } void CCoinsViewCache::Uncache(const COutPoint &outpoint) { CCoinsMap::iterator it = cacheCoins.find(outpoint); if (it != cacheCoins.end() && it->second.flags == 0) { cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); cacheCoins.erase(it); } } unsigned int CCoinsViewCache::GetCacheSize() const { return cacheCoins.size(); } const CTxOut &CCoinsViewCache::GetOutputFor(const CTxIn &input) const { const Coin &coin = AccessCoin(input.prevout); assert(!coin.IsSpent()); return coin.GetTxOut(); } Amount CCoinsViewCache::GetValueIn(const CTransaction &tx) const { if (tx.IsCoinBase()) { return Amount::zero(); } Amount nResult = Amount::zero(); for (size_t i = 0; i < tx.vin.size(); i++) { nResult += GetOutputFor(tx.vin[i]).nValue; } return nResult; } bool CCoinsViewCache::HaveInputs(const CTransaction &tx) const { if (tx.IsCoinBase()) { return true; } for (size_t i = 0; i < tx.vin.size(); i++) { if (!HaveCoin(tx.vin[i].prevout)) { return false; } } return true; } double CCoinsViewCache::GetPriority(const CTransaction &tx, int nHeight, Amount &inChainInputValue) const { inChainInputValue = Amount::zero(); if (tx.IsCoinBase()) { return 0.0; } double dResult = 0.0; for (const CTxIn &txin : tx.vin) { const Coin &coin = AccessCoin(txin.prevout); if (coin.IsSpent()) { continue; } if (int64_t(coin.GetHeight()) <= nHeight) { dResult += double(coin.GetTxOut().nValue / SATOSHI) * (nHeight - coin.GetHeight()); inChainInputValue += coin.GetTxOut().nValue; } } return tx.ComputePriority(dResult); } // TODO: merge with similar definition in undo.h. static const size_t MAX_OUTPUTS_PER_TX = MAX_TX_SIZE / ::GetSerializeSize(CTxOut(), SER_NETWORK, PROTOCOL_VERSION); const Coin &AccessByTxid(const CCoinsViewCache &view, const TxId &txid) { for (uint32_t n = 0; n < MAX_OUTPUTS_PER_TX; n++) { const Coin &alternate = view.AccessCoin(COutPoint(txid, n)); if (!alternate.IsSpent()) { return alternate; } } return coinEmpty; } diff --git a/src/coins.h b/src/coins.h index f5959f1493..7c0c477334 100644 --- a/src/coins.h +++ b/src/coins.h @@ -1,316 +1,316 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_COINS_H #define BITCOIN_COINS_H #include #include -#include +#include #include #include #include #include #include #include /** * A UTXO entry. * * Serialized format: * - VARINT((coinbase ? 1 : 0) | (height << 1)) * - the non-spent CTxOut (via CTxOutCompressor) */ class Coin { //! Unspent transaction output. CTxOut out; //! Whether containing transaction was a coinbase and height at which the //! transaction was included into a block. uint32_t nHeightAndIsCoinBase; public: //! Empty constructor Coin() : nHeightAndIsCoinBase(0) {} //! Constructor from a CTxOut and height/coinbase information. Coin(CTxOut outIn, uint32_t nHeightIn, bool IsCoinbase) : out(std::move(outIn)), nHeightAndIsCoinBase((nHeightIn << 1) | IsCoinbase) {} uint32_t GetHeight() const { return nHeightAndIsCoinBase >> 1; } bool IsCoinBase() const { return nHeightAndIsCoinBase & 0x01; } bool IsSpent() const { return out.IsNull(); } CTxOut &GetTxOut() { return out; } const CTxOut &GetTxOut() const { return out; } void Clear() { out.SetNull(); nHeightAndIsCoinBase = 0; } template void Serialize(Stream &s) const { assert(!IsSpent()); ::Serialize(s, VARINT(nHeightAndIsCoinBase)); ::Serialize(s, CTxOutCompressor(REF(out))); } template void Unserialize(Stream &s) { ::Unserialize(s, VARINT(nHeightAndIsCoinBase)); ::Unserialize(s, REF(CTxOutCompressor(out))); } size_t DynamicMemoryUsage() const { return memusage::DynamicUsage(out.scriptPubKey); } }; class SaltedOutpointHasher { private: /** Salt */ const uint64_t k0, k1; public: SaltedOutpointHasher(); /** * This *must* return size_t. With Boost 1.46 on 32-bit systems the * unordered_map will behave unpredictably if the custom hasher returns a * uint64_t, resulting in failures when syncing the chain (#4634). * Note: This information above might be outdated as the unordered map * container type has meanwhile been switched to the C++ standard library * implementation. */ size_t operator()(const COutPoint &outpoint) const { return SipHashUint256Extra(k0, k1, outpoint.GetTxId(), outpoint.GetN()); } }; struct CCoinsCacheEntry { // The actual cached data. Coin coin; uint8_t flags; enum Flags { // This cache entry is potentially different from the version in the // parent view. DIRTY = (1 << 0), // The parent view does not have this entry (or it is pruned). FRESH = (1 << 1), /* Note that FRESH is a performance optimization with which we can erase coins that are fully spent if we know we do not need to flush the changes to the parent cache. It is always safe to not mark FRESH if that condition is not guaranteed. */ }; CCoinsCacheEntry() : flags(0) {} explicit CCoinsCacheEntry(Coin coinIn) : coin(std::move(coinIn)), flags(0) {} }; typedef std::unordered_map CCoinsMap; /** Cursor for iterating over CoinsView state */ class CCoinsViewCursor { public: CCoinsViewCursor(const uint256 &hashBlockIn) : hashBlock(hashBlockIn) {} virtual ~CCoinsViewCursor() {} virtual bool GetKey(COutPoint &key) const = 0; virtual bool GetValue(Coin &coin) const = 0; virtual unsigned int GetValueSize() const = 0; virtual bool Valid() const = 0; virtual void Next() = 0; //! Get best block at the time this cursor was created const uint256 &GetBestBlock() const { return hashBlock; } private: uint256 hashBlock; }; /** Abstract view on the open txout dataset. */ class CCoinsView { public: //! Retrieve the Coin (unspent transaction output) for a given outpoint. virtual bool GetCoin(const COutPoint &outpoint, Coin &coin) const; //! Just check whether we have data for a given outpoint. //! This may (but cannot always) return true for spent outputs. virtual bool HaveCoin(const COutPoint &outpoint) const; //! Retrieve the block hash whose state this CCoinsView currently represents virtual uint256 GetBestBlock() const; //! Retrieve the range of blocks that may have been only partially written. //! If the database is in a consistent state, the result is the empty //! vector. //! Otherwise, a two-element vector is returned consisting of the new and //! the old block hash, in that order. virtual std::vector GetHeadBlocks() const; //! Do a bulk modification (multiple Coin changes + BestBlock change). //! The passed mapCoins can be modified. virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); //! Get a cursor to iterate over the whole state virtual CCoinsViewCursor *Cursor() const; //! As we use CCoinsViews polymorphically, have a virtual destructor virtual ~CCoinsView() {} //! Estimate database size (0 if not implemented) virtual size_t EstimateSize() const { return 0; } }; /** CCoinsView backed by another CCoinsView */ class CCoinsViewBacked : public CCoinsView { protected: CCoinsView *base; public: CCoinsViewBacked(CCoinsView *viewIn); bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; bool HaveCoin(const COutPoint &outpoint) const override; uint256 GetBestBlock() const override; std::vector GetHeadBlocks() const override; void SetBackend(CCoinsView &viewIn); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; CCoinsViewCursor *Cursor() const override; size_t EstimateSize() const override; }; /** * CCoinsView that adds a memory cache for transactions to another CCoinsView */ class CCoinsViewCache : public CCoinsViewBacked { protected: /** * Make mutable so that we can "fill the cache" even from Get-methods * declared as "const". */ mutable uint256 hashBlock; mutable CCoinsMap cacheCoins; /* Cached dynamic memory usage for the inner Coin objects. */ mutable size_t cachedCoinsUsage; public: CCoinsViewCache(CCoinsView *baseIn); // Standard CCoinsView methods bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; bool HaveCoin(const COutPoint &outpoint) const override; uint256 GetBestBlock() const override; void SetBestBlock(const uint256 &hashBlock); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; CCoinsViewCursor *Cursor() const override { throw std::logic_error( "CCoinsViewCache cursor iteration not supported."); } /** * Check if we have the given utxo already loaded in this cache. * The semantics are the same as HaveCoin(), but no calls to the backing * CCoinsView are made. */ bool HaveCoinInCache(const COutPoint &outpoint) const; /** * Return a reference to a Coin in the cache, or a pruned one if not found. * This is more efficient than GetCoin. Modifications to other cache entries * are allowed while accessing the returned pointer. */ const Coin &AccessCoin(const COutPoint &output) const; /** * Add a coin. Set potential_overwrite to true if a non-pruned version may * already exist. */ void AddCoin(const COutPoint &outpoint, Coin coin, bool potential_overwrite); /** * Spend a coin. Pass moveto in order to get the deleted data. * If no unspent output exists for the passed outpoint, this call has no * effect. */ bool SpendCoin(const COutPoint &outpoint, Coin *moveto = nullptr); /** * Push the modifications applied to this cache to its base. * Failure to call this method before destruction will cause the changes to * be forgotten. If false is returned, the state of this cache (and its * backing view) will be undefined. */ bool Flush(); /** * Removes the UTXO with the given outpoint from the cache, if it is not * modified. */ void Uncache(const COutPoint &outpoint); //! Calculate the size of the cache (in number of transaction outputs) unsigned int GetCacheSize() const; //! Calculate the size of the cache (in bytes) size_t DynamicMemoryUsage() const; /** * Amount of bitcoins coming in to a transaction * Note that lightweight clients may not know anything besides the hash of * previous transactions, so may not be able to calculate this. * * @param[in] tx transaction for which we are checking input total * @return Sum of value of all inputs (scriptSigs) */ Amount GetValueIn(const CTransaction &tx) const; //! Check whether all prevouts of the transaction are present in the UTXO //! set represented by this view bool HaveInputs(const CTransaction &tx) const; /** * Return priority of tx at height nHeight. Also calculate the sum of the * values of the inputs that are already in the chain. These are the inputs * that will age and increase priority as new blocks are added to the chain. */ double GetPriority(const CTransaction &tx, int nHeight, Amount &inChainInputValue) const; const CTxOut &GetOutputFor(const CTxIn &input) const; private: CCoinsMap::iterator FetchCoin(const COutPoint &outpoint) const; /** * By making the copy constructor private, we prevent accidentally using it * when one intends to create a cache on top of a base cache. */ CCoinsViewCache(const CCoinsViewCache &); }; //! Utility function to add all of a transaction's outputs to a cache. // When check is false, this assumes that overwrites are only possible for // coinbase transactions. // When check is true, the underlying view may be queried to determine whether // an addition is an overwrite. // TODO: pass in a boolean to limit these possible overwrites to known // (pre-BIP34) cases. void AddCoins(CCoinsViewCache &cache, const CTransaction &tx, int nHeight, bool check = false); //! Utility function to find any unspent output with a given txid. // This function can be quite expensive because in the event of a transaction // which is not found in the cache, it can cause up to MAX_OUTPUTS_PER_BLOCK // lookups to database, so it should be used with care. const Coin &AccessByTxid(const CCoinsViewCache &cache, const TxId &txid); #endif // BITCOIN_COINS_H diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 703573dbb7..dc99b8c8b9 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -1,35 +1,36 @@ -# Copyright (c) 2017 The Bitcoin developers +# Copyright (c) 2017-2019 The Bitcoin developers project(crypto) # The library add_library(crypto aes.cpp chacha20.cpp hmac_sha256.cpp hmac_sha512.cpp ripemd160.cpp sha1.cpp sha256.cpp sha256_sse4.cpp sha512.cpp + siphash.cpp ) target_include_directories(crypto PRIVATE .. PUBLIC # To access the config. ${CMAKE_CURRENT_BINARY_DIR}/.. ) target_compile_definitions(crypto PUBLIC HAVE_CONFIG_H) # Use assembly is specified option(CRYPTO_USE_ASM "Use assembly version of crypto primitives" ON) if(CRYPTO_USE_ASM) target_compile_definitions(crypto PRIVATE USE_ASM) endif() # Dependencies target_link_libraries(crypto OpenSSL::Crypto) diff --git a/src/hash.cpp b/src/crypto/siphash.cpp similarity index 67% copy from src/hash.cpp copy to src/crypto/siphash.cpp index 10b75ad835..27b82093cf 100644 --- a/src/hash.cpp +++ b/src/crypto/siphash.cpp @@ -1,260 +1,177 @@ -// Copyright (c) 2013-2016 The Bitcoin Core developers +// Copyright (c) 2016-2018 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include - -#include -#include -#include - -inline uint32_t ROTL32(uint32_t x, int8_t r) { - return (x << r) | (x >> (32 - r)); -} - -uint32_t MurmurHash3(uint32_t nHashSeed, - const std::vector &vDataToHash) { - // The following is MurmurHash3 (x86_32), see - // http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp - uint32_t h1 = nHashSeed; - if (vDataToHash.size() > 0) { - const uint32_t c1 = 0xcc9e2d51; - const uint32_t c2 = 0x1b873593; - - const int nblocks = vDataToHash.size() / 4; - - //---------- - // body - const uint8_t *blocks = &vDataToHash[0] + nblocks * 4; - - for (int i = -nblocks; i; i++) { - uint32_t k1 = ReadLE32(blocks + i * 4); - - k1 *= c1; - k1 = ROTL32(k1, 15); - k1 *= c2; - - h1 ^= k1; - h1 = ROTL32(h1, 13); - h1 = h1 * 5 + 0xe6546b64; - } - - //---------- - // tail - const uint8_t *tail = (const uint8_t *)(&vDataToHash[0] + nblocks * 4); - - uint32_t k1 = 0; - - switch (vDataToHash.size() & 3) { - case 3: - k1 ^= tail[2] << 16; - // FALLTHROUGH - case 2: - k1 ^= tail[1] << 8; - // FALLTHROUGH - case 1: - k1 ^= tail[0]; - k1 *= c1; - k1 = ROTL32(k1, 15); - k1 *= c2; - h1 ^= k1; - } - } - - //---------- - // finalization - h1 ^= vDataToHash.size(); - h1 ^= h1 >> 16; - h1 *= 0x85ebca6b; - h1 ^= h1 >> 13; - h1 *= 0xc2b2ae35; - h1 ^= h1 >> 16; - - return h1; -} - -void BIP32Hash(const ChainCode &chainCode, uint32_t nChild, uint8_t header, - const uint8_t data[32], uint8_t output[64]) { - uint8_t num[4]; - num[0] = (nChild >> 24) & 0xFF; - num[1] = (nChild >> 16) & 0xFF; - num[2] = (nChild >> 8) & 0xFF; - num[3] = (nChild >> 0) & 0xFF; - CHMAC_SHA512(chainCode.begin(), chainCode.size()) - .Write(&header, 1) - .Write(data, 32) - .Write(num, 4) - .Finalize(output); -} +#include #define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) #define SIPROUND \ do { \ v0 += v1; \ v1 = ROTL(v1, 13); \ v1 ^= v0; \ v0 = ROTL(v0, 32); \ v2 += v3; \ v3 = ROTL(v3, 16); \ v3 ^= v2; \ v0 += v3; \ v3 = ROTL(v3, 21); \ v3 ^= v0; \ v2 += v1; \ v1 = ROTL(v1, 17); \ v1 ^= v2; \ v2 = ROTL(v2, 32); \ } while (0) CSipHasher::CSipHasher(uint64_t k0, uint64_t k1) { v[0] = 0x736f6d6570736575ULL ^ k0; v[1] = 0x646f72616e646f6dULL ^ k1; v[2] = 0x6c7967656e657261ULL ^ k0; v[3] = 0x7465646279746573ULL ^ k1; count = 0; tmp = 0; } CSipHasher &CSipHasher::Write(uint64_t data) { uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; assert(count % 8 == 0); v3 ^= data; SIPROUND; SIPROUND; v0 ^= data; v[0] = v0; v[1] = v1; v[2] = v2; v[3] = v3; count += 8; return *this; } CSipHasher &CSipHasher::Write(const uint8_t *data, size_t size) { uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; uint64_t t = tmp; int c = count; while (size--) { t |= uint64_t(*(data++)) << (8 * (c % 8)); c++; if ((c & 7) == 0) { v3 ^= t; SIPROUND; SIPROUND; v0 ^= t; t = 0; } } v[0] = v0; v[1] = v1; v[2] = v2; v[3] = v3; count = c; tmp = t; return *this; } uint64_t CSipHasher::Finalize() const { uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; uint64_t t = tmp | (uint64_t(count) << 56); v3 ^= t; SIPROUND; SIPROUND; v0 ^= t; v2 ^= 0xFF; SIPROUND; SIPROUND; SIPROUND; SIPROUND; return v0 ^ v1 ^ v2 ^ v3; } uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256 &val) { /* Specialized implementation for efficiency */ uint64_t d = val.GetUint64(0); uint64_t v0 = 0x736f6d6570736575ULL ^ k0; uint64_t v1 = 0x646f72616e646f6dULL ^ k1; uint64_t v2 = 0x6c7967656e657261ULL ^ k0; uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d; SIPROUND; SIPROUND; v0 ^= d; d = val.GetUint64(1); v3 ^= d; SIPROUND; SIPROUND; v0 ^= d; d = val.GetUint64(2); v3 ^= d; SIPROUND; SIPROUND; v0 ^= d; d = val.GetUint64(3); v3 ^= d; SIPROUND; SIPROUND; v0 ^= d; v3 ^= uint64_t(4) << 59; SIPROUND; SIPROUND; v0 ^= uint64_t(4) << 59; v2 ^= 0xFF; SIPROUND; SIPROUND; SIPROUND; SIPROUND; return v0 ^ v1 ^ v2 ^ v3; } uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256 &val, uint32_t extra) { /* Specialized implementation for efficiency */ uint64_t d = val.GetUint64(0); uint64_t v0 = 0x736f6d6570736575ULL ^ k0; uint64_t v1 = 0x646f72616e646f6dULL ^ k1; uint64_t v2 = 0x6c7967656e657261ULL ^ k0; uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d; SIPROUND; SIPROUND; v0 ^= d; d = val.GetUint64(1); v3 ^= d; SIPROUND; SIPROUND; v0 ^= d; d = val.GetUint64(2); v3 ^= d; SIPROUND; SIPROUND; v0 ^= d; d = val.GetUint64(3); v3 ^= d; SIPROUND; SIPROUND; v0 ^= d; d = (uint64_t(36) << 56) | extra; v3 ^= d; SIPROUND; SIPROUND; v0 ^= d; v2 ^= 0xFF; SIPROUND; SIPROUND; SIPROUND; SIPROUND; return v0 ^ v1 ^ v2 ^ v3; } diff --git a/src/crypto/siphash.h b/src/crypto/siphash.h new file mode 100644 index 0000000000..597ddc88fc --- /dev/null +++ b/src/crypto/siphash.h @@ -0,0 +1,50 @@ +// Copyright (c) 2016-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_SIPHASH_H +#define BITCOIN_CRYPTO_SIPHASH_H + +#include + +#include + +/** SipHash-2-4 */ +class CSipHasher { +private: + uint64_t v[4]; + uint64_t tmp; + int count; + +public: + /** Construct a SipHash calculator initialized with 128-bit key (k0, k1) */ + CSipHasher(uint64_t k0, uint64_t k1); + /** + * Hash a 64-bit integer worth of data. + * It is treated as if this was the little-endian interpretation of 8 bytes. + * This function can only be used when a multiple of 8 bytes have been + * written so far. + */ + CSipHasher &Write(uint64_t data); + /** Hash arbitrary bytes. */ + CSipHasher &Write(const uint8_t *data, size_t size); + /** Compute the 64-bit SipHash-2-4 of the data written so far. The object + * remains untouched. */ + uint64_t Finalize() const; +}; + +/** Optimized SipHash-2-4 implementation for uint256. + * + * It is identical to: + * SipHasher(k0, k1) + * .Write(val.GetUint64(0)) + * .Write(val.GetUint64(1)) + * .Write(val.GetUint64(2)) + * .Write(val.GetUint64(3)) + * .Finalize() + */ +uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256 &val); +uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256 &val, + uint32_t extra); + +#endif // BITCOIN_CRYPTO_SIPHASH_H diff --git a/src/hash.cpp b/src/hash.cpp index 10b75ad835..b91281fe0b 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -1,260 +1,88 @@ // Copyright (c) 2013-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include inline uint32_t ROTL32(uint32_t x, int8_t r) { return (x << r) | (x >> (32 - r)); } uint32_t MurmurHash3(uint32_t nHashSeed, const std::vector &vDataToHash) { // The following is MurmurHash3 (x86_32), see // http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp uint32_t h1 = nHashSeed; if (vDataToHash.size() > 0) { const uint32_t c1 = 0xcc9e2d51; const uint32_t c2 = 0x1b873593; const int nblocks = vDataToHash.size() / 4; //---------- // body const uint8_t *blocks = &vDataToHash[0] + nblocks * 4; for (int i = -nblocks; i; i++) { uint32_t k1 = ReadLE32(blocks + i * 4); k1 *= c1; k1 = ROTL32(k1, 15); k1 *= c2; h1 ^= k1; h1 = ROTL32(h1, 13); h1 = h1 * 5 + 0xe6546b64; } //---------- // tail const uint8_t *tail = (const uint8_t *)(&vDataToHash[0] + nblocks * 4); uint32_t k1 = 0; switch (vDataToHash.size() & 3) { case 3: k1 ^= tail[2] << 16; // FALLTHROUGH case 2: k1 ^= tail[1] << 8; // FALLTHROUGH case 1: k1 ^= tail[0]; k1 *= c1; k1 = ROTL32(k1, 15); k1 *= c2; h1 ^= k1; } } //---------- // finalization h1 ^= vDataToHash.size(); h1 ^= h1 >> 16; h1 *= 0x85ebca6b; h1 ^= h1 >> 13; h1 *= 0xc2b2ae35; h1 ^= h1 >> 16; return h1; } void BIP32Hash(const ChainCode &chainCode, uint32_t nChild, uint8_t header, const uint8_t data[32], uint8_t output[64]) { uint8_t num[4]; num[0] = (nChild >> 24) & 0xFF; num[1] = (nChild >> 16) & 0xFF; num[2] = (nChild >> 8) & 0xFF; num[3] = (nChild >> 0) & 0xFF; CHMAC_SHA512(chainCode.begin(), chainCode.size()) .Write(&header, 1) .Write(data, 32) .Write(num, 4) .Finalize(output); } - -#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) - -#define SIPROUND \ - do { \ - v0 += v1; \ - v1 = ROTL(v1, 13); \ - v1 ^= v0; \ - v0 = ROTL(v0, 32); \ - v2 += v3; \ - v3 = ROTL(v3, 16); \ - v3 ^= v2; \ - v0 += v3; \ - v3 = ROTL(v3, 21); \ - v3 ^= v0; \ - v2 += v1; \ - v1 = ROTL(v1, 17); \ - v1 ^= v2; \ - v2 = ROTL(v2, 32); \ - } while (0) - -CSipHasher::CSipHasher(uint64_t k0, uint64_t k1) { - v[0] = 0x736f6d6570736575ULL ^ k0; - v[1] = 0x646f72616e646f6dULL ^ k1; - v[2] = 0x6c7967656e657261ULL ^ k0; - v[3] = 0x7465646279746573ULL ^ k1; - count = 0; - tmp = 0; -} - -CSipHasher &CSipHasher::Write(uint64_t data) { - uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; - - assert(count % 8 == 0); - - v3 ^= data; - SIPROUND; - SIPROUND; - v0 ^= data; - - v[0] = v0; - v[1] = v1; - v[2] = v2; - v[3] = v3; - - count += 8; - return *this; -} - -CSipHasher &CSipHasher::Write(const uint8_t *data, size_t size) { - uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; - uint64_t t = tmp; - int c = count; - - while (size--) { - t |= uint64_t(*(data++)) << (8 * (c % 8)); - c++; - if ((c & 7) == 0) { - v3 ^= t; - SIPROUND; - SIPROUND; - v0 ^= t; - t = 0; - } - } - - v[0] = v0; - v[1] = v1; - v[2] = v2; - v[3] = v3; - count = c; - tmp = t; - - return *this; -} - -uint64_t CSipHasher::Finalize() const { - uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; - - uint64_t t = tmp | (uint64_t(count) << 56); - - v3 ^= t; - SIPROUND; - SIPROUND; - v0 ^= t; - v2 ^= 0xFF; - SIPROUND; - SIPROUND; - SIPROUND; - SIPROUND; - return v0 ^ v1 ^ v2 ^ v3; -} - -uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256 &val) { - /* Specialized implementation for efficiency */ - uint64_t d = val.GetUint64(0); - - uint64_t v0 = 0x736f6d6570736575ULL ^ k0; - uint64_t v1 = 0x646f72616e646f6dULL ^ k1; - uint64_t v2 = 0x6c7967656e657261ULL ^ k0; - uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d; - - SIPROUND; - SIPROUND; - v0 ^= d; - d = val.GetUint64(1); - v3 ^= d; - SIPROUND; - SIPROUND; - v0 ^= d; - d = val.GetUint64(2); - v3 ^= d; - SIPROUND; - SIPROUND; - v0 ^= d; - d = val.GetUint64(3); - v3 ^= d; - SIPROUND; - SIPROUND; - v0 ^= d; - v3 ^= uint64_t(4) << 59; - SIPROUND; - SIPROUND; - v0 ^= uint64_t(4) << 59; - v2 ^= 0xFF; - SIPROUND; - SIPROUND; - SIPROUND; - SIPROUND; - return v0 ^ v1 ^ v2 ^ v3; -} - -uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256 &val, - uint32_t extra) { - /* Specialized implementation for efficiency */ - uint64_t d = val.GetUint64(0); - - uint64_t v0 = 0x736f6d6570736575ULL ^ k0; - uint64_t v1 = 0x646f72616e646f6dULL ^ k1; - uint64_t v2 = 0x6c7967656e657261ULL ^ k0; - uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d; - - SIPROUND; - SIPROUND; - v0 ^= d; - d = val.GetUint64(1); - v3 ^= d; - SIPROUND; - SIPROUND; - v0 ^= d; - d = val.GetUint64(2); - v3 ^= d; - SIPROUND; - SIPROUND; - v0 ^= d; - d = val.GetUint64(3); - v3 ^= d; - SIPROUND; - SIPROUND; - v0 ^= d; - d = (uint64_t(36) << 56) | extra; - v3 ^= d; - SIPROUND; - SIPROUND; - v0 ^= d; - v2 ^= 0xFF; - SIPROUND; - SIPROUND; - SIPROUND; - SIPROUND; - return v0 ^ v1 ^ v2 ^ v3; -} diff --git a/src/hash.h b/src/hash.h index 8a0cc5deee..93234aa3bf 100644 --- a/src/hash.h +++ b/src/hash.h @@ -1,254 +1,216 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_HASH_H #define BITCOIN_HASH_H #include #include #include #include #include #include #include typedef uint256 ChainCode; /** A hasher class for Bitcoin's 256-bit hash (double SHA-256). */ class CHash256 { private: CSHA256 sha; public: static const size_t OUTPUT_SIZE = CSHA256::OUTPUT_SIZE; void Finalize(uint8_t hash[OUTPUT_SIZE]) { uint8_t buf[CSHA256::OUTPUT_SIZE]; sha.Finalize(buf); sha.Reset().Write(buf, CSHA256::OUTPUT_SIZE).Finalize(hash); } CHash256 &Write(const uint8_t *data, size_t len) { sha.Write(data, len); return *this; } CHash256 &Reset() { sha.Reset(); return *this; } }; /** A hasher class for Bitcoin's 160-bit hash (SHA-256 + RIPEMD-160). */ class CHash160 { private: CSHA256 sha; public: static const size_t OUTPUT_SIZE = CRIPEMD160::OUTPUT_SIZE; void Finalize(uint8_t hash[OUTPUT_SIZE]) { uint8_t buf[CSHA256::OUTPUT_SIZE]; sha.Finalize(buf); CRIPEMD160().Write(buf, CSHA256::OUTPUT_SIZE).Finalize(hash); } CHash160 &Write(const uint8_t *data, size_t len) { sha.Write(data, len); return *this; } CHash160 &Reset() { sha.Reset(); return *this; } }; /** Compute the 256-bit hash of an object. */ template inline uint256 Hash(const T1 pbegin, const T1 pend) { static const uint8_t pblank[1] = {}; uint256 result; CHash256() .Write(pbegin == pend ? pblank : (const uint8_t *)&pbegin[0], (pend - pbegin) * sizeof(pbegin[0])) .Finalize((uint8_t *)&result); return result; } /** Compute the 256-bit hash of the concatenation of two objects. */ template inline uint256 Hash(const T1 p1begin, const T1 p1end, const T2 p2begin, const T2 p2end) { static const uint8_t pblank[1] = {}; uint256 result; CHash256() .Write(p1begin == p1end ? pblank : (const uint8_t *)&p1begin[0], (p1end - p1begin) * sizeof(p1begin[0])) .Write(p2begin == p2end ? pblank : (const uint8_t *)&p2begin[0], (p2end - p2begin) * sizeof(p2begin[0])) .Finalize((uint8_t *)&result); return result; } /** Compute the 256-bit hash of the concatenation of three objects. */ template inline uint256 Hash(const T1 p1begin, const T1 p1end, const T2 p2begin, const T2 p2end, const T3 p3begin, const T3 p3end) { static const uint8_t pblank[1] = {}; uint256 result; CHash256() .Write(p1begin == p1end ? pblank : (const uint8_t *)&p1begin[0], (p1end - p1begin) * sizeof(p1begin[0])) .Write(p2begin == p2end ? pblank : (const uint8_t *)&p2begin[0], (p2end - p2begin) * sizeof(p2begin[0])) .Write(p3begin == p3end ? pblank : (const uint8_t *)&p3begin[0], (p3end - p3begin) * sizeof(p3begin[0])) .Finalize((uint8_t *)&result); return result; } /** Compute the 160-bit hash an object. */ template inline uint160 Hash160(const T1 pbegin, const T1 pend) { static uint8_t pblank[1] = {}; uint160 result; CHash160() .Write(pbegin == pend ? pblank : (const uint8_t *)&pbegin[0], (pend - pbegin) * sizeof(pbegin[0])) .Finalize((uint8_t *)&result); return result; } /** Compute the 160-bit hash of a vector. */ inline uint160 Hash160(const std::vector &vch) { return Hash160(vch.begin(), vch.end()); } /** Compute the 160-bit hash of a vector. */ template inline uint160 Hash160(const prevector &vch) { return Hash160(vch.begin(), vch.end()); } /** A writer stream (for serialization) that computes a 256-bit hash. */ class CHashWriter { private: CHash256 ctx; const int nType; const int nVersion; public: CHashWriter(int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn) {} int GetType() const { return nType; } int GetVersion() const { return nVersion; } void write(const char *pch, size_t size) { ctx.Write((const uint8_t *)pch, size); } // invalidates the object uint256 GetHash() { uint256 result; ctx.Finalize((uint8_t *)&result); return result; } template CHashWriter &operator<<(const T &obj) { // Serialize to this stream ::Serialize(*this, obj); return (*this); } }; /** * Reads data from an underlying stream, while hashing the read data. */ template class CHashVerifier : public CHashWriter { private: Source *source; public: explicit CHashVerifier(Source *source_) : CHashWriter(source_->GetType(), source_->GetVersion()), source(source_) {} void read(char *pch, size_t nSize) { source->read(pch, nSize); this->write(pch, nSize); } void ignore(size_t nSize) { char data[1024]; while (nSize > 0) { size_t now = std::min(nSize, 1024); read(data, now); nSize -= now; } } template CHashVerifier &operator>>(T &obj) { // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } }; /** Compute the 256-bit hash of an object's serialization. */ template uint256 SerializeHash(const T &obj, int nType = SER_GETHASH, int nVersion = PROTOCOL_VERSION) { CHashWriter ss(nType, nVersion); ss << obj; return ss.GetHash(); } uint32_t MurmurHash3(uint32_t nHashSeed, const std::vector &vDataToHash); void BIP32Hash(const ChainCode &chainCode, uint32_t nChild, uint8_t header, const uint8_t data[32], uint8_t output[64]); -/** SipHash-2-4 */ -class CSipHasher { -private: - uint64_t v[4]; - uint64_t tmp; - int count; - -public: - /** Construct a SipHash calculator initialized with 128-bit key (k0, k1) */ - CSipHasher(uint64_t k0, uint64_t k1); - /** - * Hash a 64-bit integer worth of data. - * It is treated as if this was the little-endian interpretation of 8 bytes. - * This function can only be used when a multiple of 8 bytes have been - * written so far. - */ - CSipHasher &Write(uint64_t data); - /** Hash arbitrary bytes. */ - CSipHasher &Write(const uint8_t *data, size_t size); - /** Compute the 64-bit SipHash-2-4 of the data written so far. The object - * remains untouched. */ - uint64_t Finalize() const; -}; - -/** Optimized SipHash-2-4 implementation for uint256. - * - * It is identical to: - * SipHasher(k0, k1) - * .Write(val.GetUint64(0)) - * .Write(val.GetUint64(1)) - * .Write(val.GetUint64(2)) - * .Write(val.GetUint64(3)) - * .Finalize() - */ -uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256 &val); -uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256 &val, - uint32_t extra); - #endif // BITCOIN_HASH_H diff --git a/src/net.h b/src/net.h index 57b8d181cf..4111c0c393 100644 --- a/src/net.h +++ b/src/net.h @@ -1,869 +1,870 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Copyright (c) 2017 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_NET_H #define BITCOIN_NET_H #include #include #include #include #include #include +#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef WIN32 #include #endif class Config; class CNode; class CScheduler; /** * Time between pings automatically sent out for latency probing and keepalive * (in seconds). */ static const int PING_INTERVAL = 2 * 60; /** * Time after which to disconnect, after waiting for a ping response (or * inactivity). */ static const int TIMEOUT_INTERVAL = 20 * 60; /** Run the feeler connection loop once every 2 minutes or 120 seconds. **/ static const int FEELER_INTERVAL = 120; /** The maximum number of entries in an 'inv' protocol message */ static const unsigned int MAX_INV_SZ = 50000; static_assert(MAX_PROTOCOL_MESSAGE_LENGTH > MAX_INV_SZ * sizeof(CInv), "Max protocol message length must be greater than largest " "possible INV message"); /** The maximum number of new addresses to accumulate before announcing. */ static const unsigned int MAX_ADDR_TO_SEND = 1000; /** Maximum length of strSubVer in `version` message */ static const unsigned int MAX_SUBVERSION_LENGTH = 256; /** Maximum number of automatic outgoing nodes */ static const int MAX_OUTBOUND_CONNECTIONS = 8; /** Maximum number of addnode outgoing nodes */ static const int MAX_ADDNODE_CONNECTIONS = 8; /** -listen default */ static const bool DEFAULT_LISTEN = true; /** -upnp default */ #ifdef USE_UPNP static const bool DEFAULT_UPNP = USE_UPNP; #else static const bool DEFAULT_UPNP = false; #endif /** The maximum number of entries in mapAskFor */ static const size_t MAPASKFOR_MAX_SZ = MAX_INV_SZ; /** The maximum number of entries in setAskFor (larger due to getdata latency)*/ static const size_t SETASKFOR_MAX_SZ = 2 * MAX_INV_SZ; /** The maximum number of peer connections to maintain. */ static const unsigned int DEFAULT_MAX_PEER_CONNECTIONS = 125; /** The default for -maxuploadtarget. 0 = Unlimited */ static const uint64_t DEFAULT_MAX_UPLOAD_TARGET = 0; /** The default timeframe for -maxuploadtarget. 1 day. */ static const uint64_t MAX_UPLOAD_TIMEFRAME = 60 * 60 * 24; /** Default for blocks only*/ static const bool DEFAULT_BLOCKSONLY = false; static const bool DEFAULT_FORCEDNSSEED = false; static const size_t DEFAULT_MAXRECEIVEBUFFER = 5 * 1000; static const size_t DEFAULT_MAXSENDBUFFER = 1 * 1000; // Default 24-hour ban. // NOTE: When adjusting this, update rpcnet:setban's help ("24h") static const unsigned int DEFAULT_MISBEHAVING_BANTIME = 60 * 60 * 24; typedef int64_t NodeId; struct AddedNodeInfo { std::string strAddedNode; CService resolvedAddress; bool fConnected; bool fInbound; }; struct CNodeStats; class CClientUIInterface; struct CSerializedNetMsg { CSerializedNetMsg() = default; CSerializedNetMsg(CSerializedNetMsg &&) = default; CSerializedNetMsg &operator=(CSerializedNetMsg &&) = default; // No copying, only moves. CSerializedNetMsg(const CSerializedNetMsg &msg) = delete; CSerializedNetMsg &operator=(const CSerializedNetMsg &) = delete; std::vector data; std::string command; }; class NetEventsInterface; class CConnman { public: enum NumConnections { CONNECTIONS_NONE = 0, CONNECTIONS_IN = (1U << 0), CONNECTIONS_OUT = (1U << 1), CONNECTIONS_ALL = (CONNECTIONS_IN | CONNECTIONS_OUT), }; struct Options { ServiceFlags nLocalServices = NODE_NONE; int nMaxConnections = 0; int nMaxOutbound = 0; int nMaxAddnode = 0; int nMaxFeeler = 0; int nBestHeight = 0; CClientUIInterface *uiInterface = nullptr; NetEventsInterface *m_msgproc = nullptr; unsigned int nSendBufferMaxSize = 0; unsigned int nReceiveFloodSize = 0; uint64_t nMaxOutboundTimeframe = 0; uint64_t nMaxOutboundLimit = 0; std::vector vSeedNodes; std::vector vWhitelistedRange; std::vector vBinds, vWhiteBinds; bool m_use_addrman_outgoing = true; std::vector m_specified_outgoing; std::vector m_added_nodes; }; void Init(const Options &connOptions) { nLocalServices = connOptions.nLocalServices; nMaxConnections = connOptions.nMaxConnections; nMaxOutbound = std::min(connOptions.nMaxOutbound, connOptions.nMaxConnections); nMaxAddnode = connOptions.nMaxAddnode; nMaxFeeler = connOptions.nMaxFeeler; nBestHeight = connOptions.nBestHeight; clientInterface = connOptions.uiInterface; m_msgproc = connOptions.m_msgproc; nSendBufferMaxSize = connOptions.nSendBufferMaxSize; nReceiveFloodSize = connOptions.nReceiveFloodSize; nMaxOutboundTimeframe = connOptions.nMaxOutboundTimeframe; nMaxOutboundLimit = connOptions.nMaxOutboundLimit; vWhitelistedRange = connOptions.vWhitelistedRange; vAddedNodes = connOptions.m_added_nodes; } CConnman(const Config &configIn, uint64_t seed0, uint64_t seed1); ~CConnman(); bool Start(CScheduler &scheduler, const Options &options); void Stop(); void Interrupt(); bool GetNetworkActive() const { return fNetworkActive; }; void SetNetworkActive(bool active); void OpenNetworkConnection(const CAddress &addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound = nullptr, const char *strDest = nullptr, bool fOneShot = false, bool fFeeler = false, bool manual_connection = false); bool CheckIncomingNonce(uint64_t nonce); bool ForNode(NodeId id, std::function func); void PushMessage(CNode *pnode, CSerializedNetMsg &&msg); template void ForEachNode(Callable &&func) { LOCK(cs_vNodes); for (auto &&node : vNodes) { if (NodeFullyConnected(node)) { func(node); } } }; template void ForEachNode(Callable &&func) const { LOCK(cs_vNodes); for (auto &&node : vNodes) { if (NodeFullyConnected(node)) { func(node); } } }; template void ForEachNodeThen(Callable &&pre, CallableAfter &&post) { LOCK(cs_vNodes); for (auto &&node : vNodes) { if (NodeFullyConnected(node)) { pre(node); } } post(); }; template void ForEachNodeThen(Callable &&pre, CallableAfter &&post) const { LOCK(cs_vNodes); for (auto &&node : vNodes) { if (NodeFullyConnected(node)) { pre(node); } } post(); }; // Addrman functions size_t GetAddressCount() const; void SetServices(const CService &addr, ServiceFlags nServices); void MarkAddressGood(const CAddress &addr); void AddNewAddresses(const std::vector &vAddr, const CAddress &addrFrom, int64_t nTimePenalty = 0); std::vector GetAddresses(); // Denial-of-service detection/prevention. The idea is to detect peers that // are behaving badly and disconnect/ban them, but do it in a // one-coding-mistake-won't-shatter-the-entire-network way. // IMPORTANT: There should be nothing I can give a node that it will forward // on that will make that node's peers drop it. If there is, an attacker can // isolate a node and/or try to split the network. Dropping a node for // sending stuff that is invalid now but might be valid in a later version // is also dangerous, because it can cause a network split between nodes // running old code and nodes running new code. void Ban(const CNetAddr &netAddr, const BanReason &reason, int64_t bantimeoffset = 0, bool sinceUnixEpoch = false); void Ban(const CSubNet &subNet, const BanReason &reason, int64_t bantimeoffset = 0, bool sinceUnixEpoch = false); // Needed for unit testing. void ClearBanned(); bool IsBanned(CNetAddr ip); bool IsBanned(CSubNet subnet); bool Unban(const CNetAddr &ip); bool Unban(const CSubNet &ip); void GetBanned(banmap_t &banmap); void SetBanned(const banmap_t &banmap); // This allows temporarily exceeding nMaxOutbound, with the goal of finding // a peer that is better than all our current peers. void SetTryNewOutboundPeer(bool flag); bool GetTryNewOutboundPeer(); // Return the number of outbound peers we have in excess of our target (eg, // if we previously called SetTryNewOutboundPeer(true), and have since set // to false, we may have extra peers that we wish to disconnect). This may // return a value less than (num_outbound_connections - num_outbound_slots) // in cases where some outbound connections are not yet fully connected, or // not yet fully disconnected. int GetExtraOutboundCount(); bool AddNode(const std::string &node); bool RemoveAddedNode(const std::string &node); std::vector GetAddedNodeInfo(); size_t GetNodeCount(NumConnections num); void GetNodeStats(std::vector &vstats); bool DisconnectNode(const std::string &node); bool DisconnectNode(NodeId id); ServiceFlags GetLocalServices() const; //! set the max outbound target in bytes. void SetMaxOutboundTarget(uint64_t limit); uint64_t GetMaxOutboundTarget(); //! set the timeframe for the max outbound target. void SetMaxOutboundTimeframe(uint64_t timeframe); uint64_t GetMaxOutboundTimeframe(); //! check if the outbound target is reached. // If param historicalBlockServingLimit is set true, the function will // response true if the limit for serving historical blocks has been // reached. bool OutboundTargetReached(bool historicalBlockServingLimit); //! response the bytes left in the current max outbound cycle // in case of no limit, it will always response 0 uint64_t GetOutboundTargetBytesLeft(); //! response the time in second left in the current max outbound cycle // in case of no limit, it will always response 0 uint64_t GetMaxOutboundTimeLeftInCycle(); uint64_t GetTotalBytesRecv(); uint64_t GetTotalBytesSent(); void SetBestHeight(int height); int GetBestHeight() const; /** Get a unique deterministic randomizer. */ CSipHasher GetDeterministicRandomizer(uint64_t id) const; unsigned int GetReceiveFloodSize() const; void WakeMessageHandler(); private: struct ListenSocket { SOCKET socket; bool whitelisted; ListenSocket(SOCKET socket_, bool whitelisted_) : socket(socket_), whitelisted(whitelisted_) {} }; bool BindListenPort(const CService &bindAddr, std::string &strError, bool fWhitelisted = false); bool Bind(const CService &addr, unsigned int flags); bool InitBinds(const std::vector &binds, const std::vector &whiteBinds); void ThreadOpenAddedConnections(); void AddOneShot(const std::string &strDest); void ProcessOneShot(); void ThreadOpenConnections(std::vector connect); void ThreadMessageHandler(); void AcceptConnection(const ListenSocket &hListenSocket); void ThreadSocketHandler(); void ThreadDNSAddressSeed(); uint64_t CalculateKeyedNetGroup(const CAddress &ad) const; CNode *FindNode(const CNetAddr &ip); CNode *FindNode(const CSubNet &subNet); CNode *FindNode(const std::string &addrName); CNode *FindNode(const CService &addr); bool AttemptToEvictConnection(); CNode *ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure); bool IsWhitelistedRange(const CNetAddr &addr); void DeleteNode(CNode *pnode); NodeId GetNewNodeId(); size_t SocketSendData(CNode *pnode) const; //! check is the banlist has unwritten changes bool BannedSetIsDirty(); //! set the "dirty" flag for the banlist void SetBannedSetDirty(bool dirty = true); //! clean unused entries (if bantime has expired) void SweepBanned(); void DumpAddresses(); void DumpData(); void DumpBanlist(); // Network stats void RecordBytesRecv(uint64_t bytes); void RecordBytesSent(uint64_t bytes); // Whether the node should be passed out in ForEach* callbacks static bool NodeFullyConnected(const CNode *pnode); const Config *config; // Network usage totals CCriticalSection cs_totalBytesRecv; CCriticalSection cs_totalBytesSent; uint64_t nTotalBytesRecv; uint64_t nTotalBytesSent; // outbound limit & stats uint64_t nMaxOutboundTotalBytesSentInCycle; uint64_t nMaxOutboundCycleStartTime; uint64_t nMaxOutboundLimit; uint64_t nMaxOutboundTimeframe; // Whitelisted ranges. Any node connecting from these is automatically // whitelisted (as well as those connecting to whitelisted binds). std::vector vWhitelistedRange; unsigned int nSendBufferMaxSize; unsigned int nReceiveFloodSize; std::vector vhListenSocket; std::atomic fNetworkActive; banmap_t setBanned; CCriticalSection cs_setBanned; bool setBannedIsDirty; bool fAddressesInitialized; CAddrMan addrman; std::deque vOneShots; CCriticalSection cs_vOneShots; std::vector vAddedNodes; CCriticalSection cs_vAddedNodes; std::vector vNodes; std::list vNodesDisconnected; mutable CCriticalSection cs_vNodes; std::atomic nLastNodeId; /** Services this instance offers */ ServiceFlags nLocalServices; std::unique_ptr semOutbound; std::unique_ptr semAddnode; int nMaxConnections; int nMaxOutbound; int nMaxAddnode; int nMaxFeeler; std::atomic nBestHeight; CClientUIInterface *clientInterface; NetEventsInterface *m_msgproc; /** SipHasher seeds for deterministic randomness */ const uint64_t nSeed0, nSeed1; /** flag for waking the message processor. */ bool fMsgProcWake; std::condition_variable condMsgProc; CWaitableCriticalSection mutexMsgProc; std::atomic flagInterruptMsgProc; CThreadInterrupt interruptNet; std::thread threadDNSAddressSeed; std::thread threadSocketHandler; std::thread threadOpenAddedConnections; std::thread threadOpenConnections; std::thread threadMessageHandler; /** * Flag for deciding to connect to an extra outbound peer, in excess of * nMaxOutbound. * This takes the place of a feeler connection. */ std::atomic_bool m_try_another_outbound_peer; friend struct CConnmanTest; }; extern std::unique_ptr g_connman; void Discover(); void StartMapPort(); void InterruptMapPort(); void StopMapPort(); unsigned short GetListenPort(); bool BindListenPort(const CService &bindAddr, std::string &strError, bool fWhitelisted = false); /** * Interface for message handling */ class NetEventsInterface { public: virtual bool ProcessMessages(const Config &config, CNode *pnode, std::atomic &interrupt) = 0; virtual bool SendMessages(const Config &config, CNode *pnode, std::atomic &interrupt) = 0; virtual void InitializeNode(const Config &config, CNode *pnode) = 0; virtual void FinalizeNode(const Config &config, NodeId id, bool &update_connection_time) = 0; protected: /** * Protected destructor so that instances can only be deleted by derived * classes. If that restriction is no longer desired, this should be made * public and virtual. */ ~NetEventsInterface() = default; }; enum { // unknown LOCAL_NONE, // address a local interface listens on LOCAL_IF, // address explicit bound to LOCAL_BIND, // address reported by UPnP LOCAL_UPNP, // address explicitly specified (-externalip=) LOCAL_MANUAL, LOCAL_MAX }; bool IsPeerAddrLocalGood(CNode *pnode); void AdvertiseLocal(CNode *pnode); void SetLimited(enum Network net, bool fLimited = true); bool IsLimited(enum Network net); bool IsLimited(const CNetAddr &addr); bool AddLocal(const CService &addr, int nScore = LOCAL_NONE); bool AddLocal(const CNetAddr &addr, int nScore = LOCAL_NONE); void RemoveLocal(const CService &addr); bool SeenLocal(const CService &addr); bool IsLocal(const CService &addr); bool GetLocal(CService &addr, const CNetAddr *paddrPeer = nullptr); bool IsReachable(enum Network net); bool IsReachable(const CNetAddr &addr); CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices); extern bool fDiscover; extern bool fListen; extern bool fRelayTxes; extern limitedmap mapAlreadyAskedFor; struct LocalServiceInfo { int nScore; int nPort; }; extern CCriticalSection cs_mapLocalHost; extern std::map mapLocalHost; // Command, total bytes typedef std::map mapMsgCmdSize; /** * POD that contains various stats about a node. * Usually constructed from CConman::GetNodeStats. Stats are filled from the * node using CNode::copyStats. */ struct CNodeStats { NodeId nodeid; ServiceFlags nServices; bool fRelayTxes; int64_t nLastSend; int64_t nLastRecv; int64_t nTimeConnected; int64_t nTimeOffset; std::string addrName; int nVersion; std::string cleanSubVer; bool fInbound; bool m_manual_connection; int nStartingHeight; uint64_t nSendBytes; mapMsgCmdSize mapSendBytesPerMsgCmd; uint64_t nRecvBytes; mapMsgCmdSize mapRecvBytesPerMsgCmd; bool fWhitelisted; double dPingTime; double dPingWait; double dMinPing; // Our address, as reported by the peer std::string addrLocal; // Address of this peer CAddress addr; // Bind address of our side of the connection CAddress addrBind; }; class CNetMessage { private: mutable CHash256 hasher; mutable uint256 data_hash; public: // Parsing header (false) or data (true) bool in_data; // Partially received header. CDataStream hdrbuf; // Complete header. CMessageHeader hdr; uint32_t nHdrPos; // Received message data. CDataStream vRecv; uint32_t nDataPos; // Time (in microseconds) of message receipt. int64_t nTime; CNetMessage(const CMessageHeader::MessageMagic &pchMessageStartIn, int nTypeIn, int nVersionIn) : hdrbuf(nTypeIn, nVersionIn), hdr(pchMessageStartIn), vRecv(nTypeIn, nVersionIn) { hdrbuf.resize(24); in_data = false; nHdrPos = 0; nDataPos = 0; nTime = 0; } bool complete() const { if (!in_data) { return false; } return (hdr.nMessageSize == nDataPos); } const uint256 &GetMessageHash() const; void SetVersion(int nVersionIn) { hdrbuf.SetVersion(nVersionIn); vRecv.SetVersion(nVersionIn); } int readHeader(const Config &config, const char *pch, uint32_t nBytes); int readData(const char *pch, uint32_t nBytes); }; /** Information about a peer */ class CNode { friend class CConnman; public: // socket std::atomic nServices; SOCKET hSocket; // Total size of all vSendMsg entries. size_t nSendSize; // Offset inside the first vSendMsg already sent. size_t nSendOffset; uint64_t nSendBytes; std::deque> vSendMsg; CCriticalSection cs_vSend; CCriticalSection cs_hSocket; CCriticalSection cs_vRecv; CCriticalSection cs_vProcessMsg; std::list vProcessMsg; size_t nProcessQueueSize; CCriticalSection cs_sendProcessing; std::deque vRecvGetData; uint64_t nRecvBytes; std::atomic nRecvVersion; std::atomic nLastSend; std::atomic nLastRecv; const int64_t nTimeConnected; std::atomic nTimeOffset; // Address of this peer const CAddress addr; // Bind address of our side of the connection const CAddress addrBind; std::atomic nVersion; // strSubVer is whatever byte array we read from the wire. However, this // field is intended to be printed out, displayed to humans in various forms // and so on. So we sanitize it and store the sanitized version in // cleanSubVer. The original should be used when dealing with the network or // wire types and the cleaned string used when displayed or logged. std::string strSubVer, cleanSubVer; // Used for both cleanSubVer and strSubVer. CCriticalSection cs_SubVer; // This peer can bypass DoS banning. bool fWhitelisted; // If true this node is being used as a short lived feeler. bool fFeeler; bool fOneShot; bool m_manual_connection; bool fClient; // after BIP159 bool m_limited_node; const bool fInbound; std::atomic_bool fSuccessfullyConnected; std::atomic_bool fDisconnect; // We use fRelayTxes for two purposes - // a) it allows us to not relay tx invs before receiving the peer's version // message. // b) the peer may tell us in its version message that we should not relay // tx invs unless it loads a bloom filter. // protected by cs_filter bool fRelayTxes; bool fSentAddr; CSemaphoreGrant grantOutbound; CCriticalSection cs_filter; std::unique_ptr pfilter; std::atomic nRefCount; const uint64_t nKeyedNetGroup; std::atomic_bool fPauseRecv; std::atomic_bool fPauseSend; protected: mapMsgCmdSize mapSendBytesPerMsgCmd; mapMsgCmdSize mapRecvBytesPerMsgCmd; public: uint256 hashContinue; std::atomic nStartingHeight; // flood relay std::vector vAddrToSend; CRollingBloomFilter addrKnown; bool fGetAddr; std::set setKnown; int64_t nNextAddrSend; int64_t nNextLocalAddrSend; // Inventory based relay. CRollingBloomFilter filterInventoryKnown; // Set of transaction ids we still have to announce. They are sorted by the // mempool before relay, so the order is not important. std::set setInventoryTxToSend; // List of block ids we still have announce. There is no final sorting // before sending, as they are always sent immediately and in the order // requested. std::vector vInventoryBlockToSend; CCriticalSection cs_inventory; std::set setAskFor; std::multimap mapAskFor; int64_t nNextInvSend; // Used for headers announcements - unfiltered blocks to relay. Also // protected by cs_inventory. std::vector vBlockHashesToAnnounce; // Used for BIP35 mempool sending, also protected by cs_inventory. bool fSendMempool; // Last time a "MEMPOOL" request was serviced. std::atomic timeLastMempoolReq; // Block and TXN accept times std::atomic nLastBlockTime; std::atomic nLastTXTime; // Ping time measurement: // The pong reply we're expecting, or 0 if no pong expected. std::atomic nPingNonceSent; // Time (in usec) the last ping was sent, or 0 if no ping was ever sent. std::atomic nPingUsecStart; // Last measured round-trip time. std::atomic nPingUsecTime; // Best measured round-trip time. std::atomic nMinPingUsecTime; // Whether a ping is requested. std::atomic fPingQueued; // Minimum fee rate with which to filter inv's to this node Amount minFeeFilter; CCriticalSection cs_feeFilter; Amount lastSentFeeFilter; int64_t nextSendTimeFeeFilter; CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress &addrBindIn, const std::string &addrNameIn = "", bool fInboundIn = false); ~CNode(); private: CNode(const CNode &); void operator=(const CNode &); const NodeId id; const uint64_t nLocalHostNonce; // Services offered to this peer const ServiceFlags nLocalServices; const int nMyStartingHeight; int nSendVersion; // Used only by SocketHandler thread. std::list vRecvMsg; mutable CCriticalSection cs_addrName; std::string addrName; // Our address, as reported by the peer CService addrLocal; mutable CCriticalSection cs_addrLocal; public: NodeId GetId() const { return id; } uint64_t GetLocalNonce() const { return nLocalHostNonce; } int GetMyStartingHeight() const { return nMyStartingHeight; } int GetRefCount() const { assert(nRefCount >= 0); return nRefCount; } bool ReceiveMsgBytes(const Config &config, const char *pch, uint32_t nBytes, bool &complete); void SetRecvVersion(int nVersionIn) { nRecvVersion = nVersionIn; } int GetRecvVersion() const { return nRecvVersion; } void SetSendVersion(int nVersionIn); int GetSendVersion() const; CService GetAddrLocal() const; //! May not be called more than once void SetAddrLocal(const CService &addrLocalIn); CNode *AddRef() { nRefCount++; return this; } void Release() { nRefCount--; } void AddAddressKnown(const CAddress &_addr) { addrKnown.insert(_addr.GetKey()); } void PushAddress(const CAddress &_addr, FastRandomContext &insecure_rand) { // Known checking here is only to save space from duplicates. // SendMessages will filter it again for knowns that were added // after addresses were pushed. if (_addr.IsValid() && !addrKnown.contains(_addr.GetKey())) { if (vAddrToSend.size() >= MAX_ADDR_TO_SEND) { vAddrToSend[insecure_rand.randrange(vAddrToSend.size())] = _addr; } else { vAddrToSend.push_back(_addr); } } } void AddInventoryKnown(const CInv &inv) { LOCK(cs_inventory); filterInventoryKnown.insert(inv.hash); } void PushInventory(const CInv &inv) { LOCK(cs_inventory); if (inv.type == MSG_TX) { if (!filterInventoryKnown.contains(inv.hash)) { setInventoryTxToSend.insert(inv.hash); } } else if (inv.type == MSG_BLOCK) { vInventoryBlockToSend.push_back(inv.hash); } } void PushBlockHash(const uint256 &hash) { LOCK(cs_inventory); vBlockHashesToAnnounce.push_back(hash); } void AskFor(const CInv &inv); void CloseSocketDisconnect(); void copyStats(CNodeStats &stats); ServiceFlags GetLocalServices() const { return nLocalServices; } std::string GetAddrName() const; //! Sets the addrName only if it was not previously set void MaybeSetAddrName(const std::string &addrNameIn); }; /** * Return a timestamp in the future (in microseconds) for exponentially * distributed events. */ int64_t PoissonNextSend(int64_t nNow, int average_interval_seconds); std::string getSubVersionEB(uint64_t MaxBlockSize); std::string userAgent(const Config &config); #endif // BITCOIN_NET_H diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index 55552974f9..b2c0aea52c 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -1,204 +1,205 @@ -// Copyright (c) 2013-2016 The Bitcoin Core developers +// Copyright (c) 2013-2018 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include #include #include #include #include BOOST_FIXTURE_TEST_SUITE(hash_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(murmurhash3) { #define T(expected, seed, data) \ BOOST_CHECK_EQUAL(MurmurHash3(seed, ParseHex(data)), expected) // Test MurmurHash3 with various inputs. Of course this is retested in the // bloom filter tests - they would fail if MurmurHash3() had any problems - // but is useful for those trying to implement Bitcoin libraries as a // source of test data for their MurmurHash3() primitive during // development. // // The magic number 0xFBA4C795 comes from CBloomFilter::Hash() T(0x00000000, 0x00000000, ""); T(0x6a396f08, 0xFBA4C795, ""); T(0x81f16f39, 0xffffffff, ""); T(0x514e28b7, 0x00000000, "00"); T(0xea3f0b17, 0xFBA4C795, "00"); T(0xfd6cf10d, 0x00000000, "ff"); T(0x16c6b7ab, 0x00000000, "0011"); T(0x8eb51c3d, 0x00000000, "001122"); T(0xb4471bf8, 0x00000000, "00112233"); T(0xe2301fa8, 0x00000000, "0011223344"); T(0xfc2e4a15, 0x00000000, "001122334455"); T(0xb074502c, 0x00000000, "00112233445566"); T(0x8034d2a0, 0x00000000, "0011223344556677"); T(0xb4698def, 0x00000000, "001122334455667788"); #undef T } /** * SipHash-2-4 output with * k = 00 01 02 ... * and * in = (empty string) * in = 00 (1 byte) * in = 00 01 (2 bytes) * in = 00 01 02 (3 bytes) * ... * in = 00 01 02 ... 3e (63 bytes) * * from: https://131002.net/siphash/siphash24.c */ uint64_t siphash_4_2_testvec[] = { 0x726fdb47dd0e0e31, 0x74f839c593dc67fd, 0x0d6c8009d9a94f5a, 0x85676696d7fb7e2d, 0xcf2794e0277187b7, 0x18765564cd99a68d, 0xcbc9466e58fee3ce, 0xab0200f58b01d137, 0x93f5f5799a932462, 0x9e0082df0ba9e4b0, 0x7a5dbbc594ddb9f3, 0xf4b32f46226bada7, 0x751e8fbc860ee5fb, 0x14ea5627c0843d90, 0xf723ca908e7af2ee, 0xa129ca6149be45e5, 0x3f2acc7f57c29bdb, 0x699ae9f52cbe4794, 0x4bc1b3f0968dd39c, 0xbb6dc91da77961bd, 0xbed65cf21aa2ee98, 0xd0f2cbb02e3b67c7, 0x93536795e3a33e88, 0xa80c038ccd5ccec8, 0xb8ad50c6f649af94, 0xbce192de8a85b8ea, 0x17d835b85bbb15f3, 0x2f2e6163076bcfad, 0xde4daaaca71dc9a5, 0xa6a2506687956571, 0xad87a3535c49ef28, 0x32d892fad841c342, 0x7127512f72f27cce, 0xa7f32346f95978e3, 0x12e0b01abb051238, 0x15e034d40fa197ae, 0x314dffbe0815a3b4, 0x027990f029623981, 0xcadcd4e59ef40c4d, 0x9abfd8766a33735c, 0x0e3ea96b5304a7d0, 0xad0c42d6fc585992, 0x187306c89bc215a9, 0xd4a60abcf3792b95, 0xf935451de4f21df2, 0xa9538f0419755787, 0xdb9acddff56ca510, 0xd06c98cd5c0975eb, 0xe612a3cb9ecba951, 0xc766e62cfcadaf96, 0xee64435a9752fe72, 0xa192d576b245165a, 0x0a8787bf8ecb74b2, 0x81b3e73d20b49b6f, 0x7fa8220ba3b2ecea, 0x245731c13ca42499, 0xb78dbfaf3a8d83bd, 0xea1ad565322a1a0b, 0x60e61c23a3795013, 0x6606d7e446282b93, 0x6ca4ecb15c5f91e1, 0x9f626da15c9625f3, 0xe51b38608ef25f57, 0x958a324ceb064572}; BOOST_AUTO_TEST_CASE(siphash) { CSipHasher hasher(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL); BOOST_CHECK_EQUAL(hasher.Finalize(), 0x726fdb47dd0e0e31ull); static const uint8_t t0[1] = {0}; hasher.Write(t0, 1); BOOST_CHECK_EQUAL(hasher.Finalize(), 0x74f839c593dc67fdull); static const uint8_t t1[7] = {1, 2, 3, 4, 5, 6, 7}; hasher.Write(t1, 7); BOOST_CHECK_EQUAL(hasher.Finalize(), 0x93f5f5799a932462ull); hasher.Write(0x0F0E0D0C0B0A0908ULL); BOOST_CHECK_EQUAL(hasher.Finalize(), 0x3f2acc7f57c29bdbull); static const uint8_t t2[2] = {16, 17}; hasher.Write(t2, 2); BOOST_CHECK_EQUAL(hasher.Finalize(), 0x4bc1b3f0968dd39cull); static const uint8_t t3[9] = {18, 19, 20, 21, 22, 23, 24, 25, 26}; hasher.Write(t3, 9); BOOST_CHECK_EQUAL(hasher.Finalize(), 0x2f2e6163076bcfadull); static const uint8_t t4[5] = {27, 28, 29, 30, 31}; hasher.Write(t4, 5); BOOST_CHECK_EQUAL(hasher.Finalize(), 0x7127512f72f27cceull); hasher.Write(0x2726252423222120ULL); BOOST_CHECK_EQUAL(hasher.Finalize(), 0x0e3ea96b5304a7d0ull); hasher.Write(0x2F2E2D2C2B2A2928ULL); BOOST_CHECK_EQUAL(hasher.Finalize(), 0xe612a3cb9ecba951ull); BOOST_CHECK_EQUAL( SipHashUint256(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL, uint256S("1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09" "080706050403020100")), 0x7127512f72f27cceull); // Check test vectors from spec, one byte at a time CSipHasher hasher2(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL); for (uint8_t x = 0; x < ARRAYLEN(siphash_4_2_testvec); ++x) { BOOST_CHECK_EQUAL(hasher2.Finalize(), siphash_4_2_testvec[x]); hasher2.Write(&x, 1); } // Check test vectors from spec, eight bytes at a time CSipHasher hasher3(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL); for (uint8_t x = 0; x < ARRAYLEN(siphash_4_2_testvec); x += 8) { BOOST_CHECK_EQUAL(hasher3.Finalize(), siphash_4_2_testvec[x]); hasher3.Write(uint64_t(x) | (uint64_t(x + 1) << 8) | (uint64_t(x + 2) << 16) | (uint64_t(x + 3) << 24) | (uint64_t(x + 4) << 32) | (uint64_t(x + 5) << 40) | (uint64_t(x + 6) << 48) | (uint64_t(x + 7) << 56)); } CHashWriter ss(SER_DISK, CLIENT_VERSION); CMutableTransaction tx; // Note these tests were originally written with tx.nVersion=1 // and the test would be affected by default tx version bumps if not fixed. tx.nVersion = 1; ss << tx; BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); // Check consistency between CSipHasher and SipHashUint256[Extra]. FastRandomContext ctx; for (int i = 0; i < 16; ++i) { uint64_t k1 = ctx.rand64(); uint64_t k2 = ctx.rand64(); uint256 x = InsecureRand256(); uint32_t n = ctx.rand32(); uint8_t nb[4]; WriteLE32(nb, n); CSipHasher sip256(k1, k2); sip256.Write(x.begin(), 32); CSipHasher sip288 = sip256; sip288.Write(nb, 4); BOOST_CHECK_EQUAL(SipHashUint256(k1, k2, x), sip256.Finalize()); BOOST_CHECK_EQUAL(SipHashUint256Extra(k1, k2, x, n), sip288.Finalize()); } } namespace { class CDummyObject { uint32_t value; public: CDummyObject() : value(0) {} uint32_t GetValue() { return value; } template void Serialize(Stream &s) const { int nVersionDummy = 0; ::Serialize(s, VARINT(nVersionDummy)); ::Serialize(s, VARINT(value)); } template void Unserialize(Stream &s) { int nVersionDummy; ::Unserialize(s, VARINT(nVersionDummy)); ::Unserialize(s, VARINT(value)); } }; } // namespace BOOST_AUTO_TEST_CASE(hashverifier_tests) { std::vector data = ParseHex("4223"); CDataStream ss(data, SER_DISK, CLIENT_VERSION); CHashVerifier verifier(&ss); CDummyObject dummy; verifier >> dummy; uint256 checksum = verifier.GetHash(); BOOST_CHECK_EQUAL(dummy.GetValue(), 0x23); CHashWriter h0(SER_DISK, CLIENT_VERSION); h0 << CDataStream(data, SER_DISK, CLIENT_VERSION); BOOST_CHECK(h0.GetHash() == checksum); CHashWriter h1(SER_DISK, CLIENT_VERSION); h1 << dummy; BOOST_CHECK(h1.GetHash() != checksum); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txmempool.h b/src/txmempool.h index 6c1e44634b..fae2e149ed 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -1,944 +1,945 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_TXMEMPOOL_H #define BITCOIN_TXMEMPOOL_H #include #include +#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include class CAutoFile; class CBlockIndex; class Config; inline double AllowFreeThreshold() { return (144 * COIN) / (250 * SATOSHI); } inline bool AllowFree(double dPriority) { // Large (in bytes) low-priority (new, small-coin) transactions need a fee. return dPriority > AllowFreeThreshold(); } /** * Fake height value used in Coins to signify they are only in the memory * pool(since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; struct LockPoints { // Will be set to the blockchain height and median time past values that // would be necessary to satisfy all relative locktime constraints (BIP68) // of this tx given our view of block chain history int height; int64_t time; // As long as the current chain descends from the highest height block // containing one of the inputs used in the calculation, then the cached // values are still valid even after a reorg. CBlockIndex *maxInputBlock; LockPoints() : height(0), time(0), maxInputBlock(nullptr) {} }; class CTxMemPool; /** \class CTxMemPoolEntry * * CTxMemPoolEntry stores data about the corresponding transaction, as well as * data about all in-mempool transactions that depend on the transaction * ("descendant" transactions). * * When a new entry is added to the mempool, we update the descendant state * (nCountWithDescendants, nSizeWithDescendants, and nModFeesWithDescendants) * for all ancestors of the newly added transaction. * * If updating the descendant state is skipped, we can mark the entry as * "dirty", and set nSizeWithDescendants/nModFeesWithDescendants to equal * nTxSize/nFee+feeDelta. (This can potentially happen during a reorg, where we * limit the amount of work we're willing to do to avoid consuming too much * CPU.) */ class CTxMemPoolEntry { private: CTransactionRef tx; //!< Cached to avoid expensive parent-transaction lookups Amount nFee; //!< ... and avoid recomputing tx size size_t nTxSize; //!< ... and billable size for billing size_t nTxBillableSize; //!< ... and modified size for priority size_t nModSize; //!< ... and total memory usage size_t nUsageSize; //!< Local time when entering the mempool int64_t nTime; //!< Priority when entering the mempool double entryPriority; //!< Chain height when entering the mempool unsigned int entryHeight; //!< Sum of all txin values that are already in blockchain Amount inChainInputValue; //!< keep track of transactions that spend a coinbase bool spendsCoinbase; //!< Total sigop plus P2SH sigops count int64_t sigOpCount; //!< Used for determining the priority of the transaction for mining in a //! block Amount feeDelta; //!< Track the height and time at which tx was final LockPoints lockPoints; // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these // descendants as well. if nCountWithDescendants is 0, treat this entry as // dirty, and nSizeWithDescendants and nModFeesWithDescendants will not be // correct. //!< number of descendant transactions uint64_t nCountWithDescendants; //!< ... and size uint64_t nSizeWithDescendants; uint64_t nBillableSizeWithDescendants; //!< ... and total fees (all including us) Amount nModFeesWithDescendants; // Analogous statistics for ancestor transactions uint64_t nCountWithAncestors; uint64_t nSizeWithAncestors; uint64_t nBillableSizeWithAncestors; Amount nModFeesWithAncestors; int64_t nSigOpCountWithAncestors; public: CTxMemPoolEntry(const CTransactionRef &_tx, const Amount _nFee, int64_t _nTime, double _entryPriority, unsigned int _entryHeight, Amount _inChainInputValue, bool spendsCoinbase, int64_t nSigOpsCost, LockPoints lp); const CTransaction &GetTx() const { return *this->tx; } CTransactionRef GetSharedTx() const { return this->tx; } /** * Fast calculation of lower bound of current priority as update from entry * priority. Only inputs that were originally in-chain will age. */ double GetPriority(unsigned int currentHeight) const; const Amount GetFee() const { return nFee; } size_t GetTxSize() const { return nTxSize; } size_t GetTxBillableSize() const { return nTxBillableSize; } int64_t GetTime() const { return nTime; } unsigned int GetHeight() const { return entryHeight; } int64_t GetSigOpCount() const { return sigOpCount; } Amount GetModifiedFee() const { return nFee + feeDelta; } size_t DynamicMemoryUsage() const { return nUsageSize; } const LockPoints &GetLockPoints() const { return lockPoints; } // Adjusts the descendant state, if this entry is not dirty. void UpdateDescendantState(int64_t modifySize, int64_t modifyBillableSize, Amount modifyFee, int64_t modifyCount); // Adjusts the ancestor state void UpdateAncestorState(int64_t modifySize, int64_t modifyBillableSize, Amount modifyFee, int64_t modifyCount, int modifySigOps); // Updates the fee delta used for mining priority score, and the // modified fees with descendants. void UpdateFeeDelta(Amount feeDelta); // Update the LockPoints after a reorg void UpdateLockPoints(const LockPoints &lp); uint64_t GetCountWithDescendants() const { return nCountWithDescendants; } uint64_t GetSizeWithDescendants() const { return nSizeWithDescendants; } uint64_t GetBillableSizeWithDescendants() const { return nBillableSizeWithDescendants; } Amount GetModFeesWithDescendants() const { return nModFeesWithDescendants; } bool GetSpendsCoinbase() const { return spendsCoinbase; } uint64_t GetCountWithAncestors() const { return nCountWithAncestors; } uint64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } uint64_t GetBillableSizeWithAncestors() const { return nBillableSizeWithAncestors; } Amount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } int64_t GetSigOpCountWithAncestors() const { return nSigOpCountWithAncestors; } //!< Index in mempool's vTxHashes mutable size_t vTxHashesIdx; }; // Helpers for modifying CTxMemPool::mapTx, which is a boost multi_index. struct update_descendant_state { update_descendant_state(int64_t _modifySize, int64_t _modifyBillableSize, Amount _modifyFee, int64_t _modifyCount) : modifySize(_modifySize), modifyBillableSize(_modifyBillableSize), modifyFee(_modifyFee), modifyCount(_modifyCount) {} void operator()(CTxMemPoolEntry &e) { e.UpdateDescendantState(modifySize, modifyBillableSize, modifyFee, modifyCount); } private: int64_t modifySize; int64_t modifyBillableSize; Amount modifyFee; int64_t modifyCount; }; struct update_ancestor_state { update_ancestor_state(int64_t _modifySize, int64_t _modifyBillableSize, Amount _modifyFee, int64_t _modifyCount, int64_t _modifySigOpsCost) : modifySize(_modifySize), modifyBillableSize(_modifyBillableSize), modifyFee(_modifyFee), modifyCount(_modifyCount), modifySigOpsCost(_modifySigOpsCost) {} void operator()(CTxMemPoolEntry &e) { e.UpdateAncestorState(modifySize, modifyBillableSize, modifyFee, modifyCount, modifySigOpsCost); } private: int64_t modifySize; int64_t modifyBillableSize; Amount modifyFee; int64_t modifyCount; int64_t modifySigOpsCost; }; struct update_fee_delta { explicit update_fee_delta(Amount _feeDelta) : feeDelta(_feeDelta) {} void operator()(CTxMemPoolEntry &e) { e.UpdateFeeDelta(feeDelta); } private: Amount feeDelta; }; struct update_lock_points { explicit update_lock_points(const LockPoints &_lp) : lp(_lp) {} void operator()(CTxMemPoolEntry &e) { e.UpdateLockPoints(lp); } private: const LockPoints &lp; }; // extracts a transaction hash from CTxMempoolEntry or CTransactionRef struct mempoolentry_txid { typedef uint256 result_type; result_type operator()(const CTxMemPoolEntry &entry) const { return entry.GetTx().GetId(); } result_type operator()(const CTransactionRef &tx) const { return tx->GetId(); } }; /** \class CompareTxMemPoolEntryByDescendantScore * * Sort an entry by max(score/size of entry's tx, score/size with all * descendants). */ class CompareTxMemPoolEntryByDescendantScore { public: bool operator()(const CTxMemPoolEntry &a, const CTxMemPoolEntry &b) const { bool fUseADescendants = UseDescendantScore(a); bool fUseBDescendants = UseDescendantScore(b); double aModFee = (fUseADescendants ? a.GetModFeesWithDescendants() : a.GetModifiedFee()) / SATOSHI; double aSize = fUseADescendants ? a.GetSizeWithDescendants() : a.GetTxSize(); double bModFee = (fUseBDescendants ? b.GetModFeesWithDescendants() : b.GetModifiedFee()) / SATOSHI; double bSize = fUseBDescendants ? b.GetSizeWithDescendants() : b.GetTxSize(); // Avoid division by rewriting (a/b > c/d) as (a*d > c*b). double f1 = aModFee * bSize; double f2 = aSize * bModFee; if (f1 == f2) { return a.GetTime() >= b.GetTime(); } return f1 < f2; } // Calculate which score to use for an entry (avoiding division). bool UseDescendantScore(const CTxMemPoolEntry &a) const { double f1 = a.GetSizeWithDescendants() * (a.GetModifiedFee() / SATOSHI); double f2 = a.GetTxSize() * (a.GetModFeesWithDescendants() / SATOSHI); return f2 > f1; } }; /** \class CompareTxMemPoolEntryByScore * * Sort by score of entry ((fee+delta)/size) in descending order */ class CompareTxMemPoolEntryByScore { public: bool operator()(const CTxMemPoolEntry &a, const CTxMemPoolEntry &b) const { double f1 = b.GetTxSize() * (a.GetModifiedFee() / SATOSHI); double f2 = a.GetTxSize() * (b.GetModifiedFee() / SATOSHI); if (f1 == f2) { return b.GetTx().GetId() < a.GetTx().GetId(); } return f1 > f2; } }; class CompareTxMemPoolEntryByEntryTime { public: bool operator()(const CTxMemPoolEntry &a, const CTxMemPoolEntry &b) const { return a.GetTime() < b.GetTime(); } }; class CompareTxMemPoolEntryByAncestorFee { public: bool operator()(const CTxMemPoolEntry &a, const CTxMemPoolEntry &b) const { double aFees = a.GetModFeesWithAncestors() / SATOSHI; double aSize = a.GetSizeWithAncestors(); double bFees = b.GetModFeesWithAncestors() / SATOSHI; double bSize = b.GetSizeWithAncestors(); // Avoid division by rewriting (a/b > c/d) as (a*d > c*b). double f1 = aFees * bSize; double f2 = aSize * bFees; if (f1 == f2) { return a.GetTx().GetId() < b.GetTx().GetId(); } return f1 > f2; } }; // Multi_index tag names struct descendant_score {}; struct entry_time {}; struct mining_score {}; struct ancestor_score {}; /** * Information about a mempool transaction. */ struct TxMempoolInfo { /** The transaction itself */ CTransactionRef tx; /** Time the transaction entered the mempool. */ int64_t nTime; /** Feerate of the transaction. */ CFeeRate feeRate; /** The fee delta. */ Amount nFeeDelta; }; /** * Reason why a transaction was removed from the mempool, this is passed to the * notification signal. */ enum class MemPoolRemovalReason { //! Manually removed or unknown reason UNKNOWN = 0, //! Expired from mempool EXPIRY, //! Removed in size limiting SIZELIMIT, //! Removed for reorganization REORG, //! Removed for block BLOCK, //! Removed for conflict with in-block transaction CONFLICT, //! Removed for replacement REPLACED }; class SaltedTxidHasher { private: /** Salt */ const uint64_t k0, k1; public: SaltedTxidHasher(); size_t operator()(const uint256 &txid) const { return SipHashUint256(k0, k1, txid); } }; typedef std::pair TXModifier; /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions that * may be included in the next block. * * Transactions are added when they are seen on the network (or created by the * local node), but not all transactions seen are added to the pool. For * example, the following new transactions will not be added to the mempool: * - a transaction which doesn't meet the minimum fee requirements. * - a new transaction that double-spends an input of a transaction already in * the pool where the new transaction does not meet the Replace-By-Fee * requirements as defined in BIP 125. * - a non-standard transaction. * * CTxMemPool::mapTx, and CTxMemPoolEntry bookkeeping: * * mapTx is a boost::multi_index that sorts the mempool on 4 criteria: * - transaction hash * - feerate [we use max(feerate of tx, feerate of tx with all descendants)] * - time in mempool * - mining score (feerate modified by any fee deltas from * PrioritiseTransaction) * * Note: the term "descendant" refers to in-mempool transactions that depend on * this one, while "ancestor" refers to in-mempool transactions that a given * transaction depends on. * * In order for the feerate sort to remain correct, we must update transactions * in the mempool when new descendants arrive. To facilitate this, we track the * set of in-mempool direct parents and direct children in mapLinks. Within each * CTxMemPoolEntry, we track the size and fees of all descendants. * * Usually when a new transaction is added to the mempool, it has no in-mempool * children (because any such children would be an orphan). So in * addUnchecked(), we: * - update a new entry's setMemPoolParents to include all in-mempool parents * - update the new entry's direct parents to include the new tx as a child * - update all ancestors of the transaction to include the new tx's size/fee * * When a transaction is removed from the mempool, we must: * - update all in-mempool parents to not track the tx in setMemPoolChildren * - update all ancestors to not include the tx's size/fees in descendant state * - update all in-mempool children to not include it as a parent * * These happen in UpdateForRemoveFromMempool(). (Note that when removing a * transaction along with its descendants, we must calculate that set of * transactions to be removed before doing the removal, or else the mempool can * be in an inconsistent state where it's impossible to walk the ancestors of a * transaction.) * * In the event of a reorg, the assumption that a newly added tx has no * in-mempool children is false. In particular, the mempool is in an * inconsistent state while new transactions are being added, because there may * be descendant transactions of a tx coming from a disconnected block that are * unreachable from just looking at transactions in the mempool (the linking * transactions may also be in the disconnected block, waiting to be added). * Because of this, there's not much benefit in trying to search for in-mempool * children in addUnchecked(). Instead, in the special case of transactions * being added from a disconnected block, we require the caller to clean up the * state, to account for in-mempool, out-of-block descendants for all the * in-block transactions by calling UpdateTransactionsFromBlock(). Note that * until this is called, the mempool state is not consistent, and in particular * mapLinks may not be correct (and therefore functions like * CalculateMemPoolAncestors() and CalculateDescendants() that rely on them to * walk the mempool are not generally safe to use). * * Computational limits: * * Updating all in-mempool ancestors of a newly added transaction can be slow, * if no bound exists on how many in-mempool ancestors there may be. * CalculateMemPoolAncestors() takes configurable limits that are designed to * prevent these calculations from being too CPU intensive. * * Adding transactions from a disconnected block can be very time consuming, * because we don't have a way to limit the number of in-mempool descendants. To * bound CPU processing, we limit the amount of work we're willing to do to * properly update the descendant information for a tx being added from a * disconnected block. If we would exceed the limit, then we instead mark the * entry as "dirty", and set the feerate for sorting purposes to be equal the * feerate of the transaction without any descendants. */ class CTxMemPool { private: //!< Value n means that n times in 2^32 we check. uint32_t nCheckFrequency; unsigned int nTransactionsUpdated; //!< sum of all mempool tx's virtual sizes. uint64_t totalTxSize; //!< sum of dynamic memory usage of all the map elements (NOT the maps //! themselves) uint64_t cachedInnerUsage; mutable int64_t lastRollingFeeUpdate; mutable bool blockSinceLastRollingFeeBump; //!< minimum fee to get into the pool, decreases exponentially mutable double rollingMinimumFeeRate; void trackPackageRemoved(const CFeeRate &rate); public: // public only for testing static const int ROLLING_FEE_HALFLIFE = 60 * 60 * 12; typedef boost::multi_index_container< CTxMemPoolEntry, boost::multi_index::indexed_by< // sorted by txid boost::multi_index::hashed_unique< mempoolentry_txid, SaltedTxidHasher>, // sorted by fee rate boost::multi_index::ordered_non_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByDescendantScore>, // sorted by entry time boost::multi_index::ordered_non_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByEntryTime>, // sorted by score (for mining prioritization) boost::multi_index::ordered_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByScore>, // sorted by fee rate with ancestors boost::multi_index::ordered_non_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByAncestorFee>>> indexed_transaction_set; mutable CCriticalSection cs; indexed_transaction_set mapTx; typedef indexed_transaction_set::nth_index<0>::type::iterator txiter; //!< All tx hashes/entries in mapTx, in random order std::vector> vTxHashes; struct CompareIteratorByHash { bool operator()(const txiter &a, const txiter &b) const { return a->GetTx().GetId() < b->GetTx().GetId(); } }; typedef std::set setEntries; const setEntries &GetMemPoolParents(txiter entry) const; const setEntries &GetMemPoolChildren(txiter entry) const; private: typedef std::map cacheMap; struct TxLinks { setEntries parents; setEntries children; }; typedef std::map txlinksMap; txlinksMap mapLinks; void UpdateParent(txiter entry, txiter parent, bool add); void UpdateChild(txiter entry, txiter child, bool add); std::vector GetSortedDepthAndScore() const; public: indirectmap mapNextTx; std::map mapDeltas; /** Create a new CTxMemPool. */ CTxMemPool(); ~CTxMemPool(); /** * If sanity-checking is turned on, check makes sure the pool is consistent * (does not contain two transactions that spend the same inputs, all inputs * are in the mapNextTx array). If sanity-checking is turned off, check does * nothing. */ void check(const CCoinsViewCache *pcoins) const; void setSanityCheck(double dFrequency = 1.0) { nCheckFrequency = dFrequency * 4294967295.0; } // addUnchecked must updated state for all ancestors of a given transaction, // to track size/count of descendant transactions. First version of // addUnchecked can be used to have it call CalculateMemPoolAncestors(), and // then invoke the second version. // Note that addUnchecked is ONLY called from ATMP outside of tests // and any other callers may break wallet's in-mempool tracking (due to // lack of CValidationInterface::TransactionAddedToMempool callbacks). bool addUnchecked(const uint256 &hash, const CTxMemPoolEntry &entry); bool addUnchecked(const uint256 &hash, const CTxMemPoolEntry &entry, setEntries &setAncestors); void removeRecursive( const CTransaction &tx, MemPoolRemovalReason reason = MemPoolRemovalReason::UNKNOWN); void removeForReorg(const Config &config, const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags); void removeConflicts(const CTransaction &tx); void removeForBlock(const std::vector &vtx, unsigned int nBlockHeight); void clear(); // lock free void _clear(); bool CompareDepthAndScore(const uint256 &hasha, const uint256 &hashb); void queryHashes(std::vector &vtxid); bool isSpent(const COutPoint &outpoint); unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); /** * Check that none of this transactions inputs are in the mempool, and thus * the tx is not dependent on other mempool transactions to be included in a * block. */ bool HasNoInputsOf(const CTransaction &tx) const; /** Affect CreateNewBlock prioritisation of transactions */ void PrioritiseTransaction(const uint256 hash, const std::string strHash, double dPriorityDelta, const Amount nFeeDelta); void ApplyDeltas(const uint256 hash, double &dPriorityDelta, Amount &nFeeDelta) const; void ClearPrioritisation(const uint256 hash); public: /** * Remove a set of transactions from the mempool. If a transaction is in * this set, then all in-mempool descendants must also be in the set, unless * this transaction is being removed for being in a block. Set * updateDescendants to true when removing a tx that was in a block, so that * any in-mempool descendants have their ancestor state updated. */ void RemoveStaged(setEntries &stage, bool updateDescendants, MemPoolRemovalReason reason = MemPoolRemovalReason::UNKNOWN); /** * When adding transactions from a disconnected block back to the mempool, * new mempool entries may have children in the mempool (which is generally * not the case when otherwise adding transactions). * UpdateTransactionsFromBlock() will find child transactions and update the * descendant state for each transaction in txidsToUpdate (excluding any * child transactions present in txidsToUpdate, which are already accounted * for). * Note: txidsToUpdate should be the set of transactions from the * disconnected block that have been accepted back into the mempool. */ void UpdateTransactionsFromBlock(const std::vector &txidsToUpdate); /** * Try to calculate all in-mempool ancestors of entry. * (these are all calculated including the tx itself) * limitAncestorCount = max number of ancestors * limitAncestorSize = max size of ancestors * limitDescendantCount = max number of descendants any ancestor can have * limitDescendantSize = max size of descendants any ancestor can have * errString = populated with error reason if any limits are hit * fSearchForParents = whether to search a tx's vin for in-mempool parents, * or look up parents from mapLinks. Must be true for entries not in the * mempool */ bool CalculateMemPoolAncestors( const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents = true) const; /** * Populate setDescendants with all in-mempool descendants of hash. * Assumes that setDescendants includes all in-mempool descendants of * anything already in it. */ void CalculateDescendants(txiter it, setEntries &setDescendants) const; /** * The minimum fee to get into the mempool, which may itself not be enough * for larger-sized transactions. The incrementalRelayFee policy variable is * used to bound the time it takes the fee rate to go back down all the way * to 0. When the feerate would otherwise be half of this, it is set to 0 * instead. */ CFeeRate GetMinFee(size_t sizelimit) const; /** * Remove transactions from the mempool until its dynamic size is <= * sizelimit. pvNoSpendsRemaining, if set, will be populated with the list * of outpoints which are not in mempool which no longer have any spends in * this mempool. */ void TrimToSize(size_t sizelimit, std::vector *pvNoSpendsRemaining = nullptr); /** * Expire all transaction (and their dependencies) in the mempool older than * time. Return the number of removed transactions. */ int Expire(int64_t time); /** * Reduce the size of the mempool by expiring and then trimming the mempool. */ void LimitSize(size_t limit, unsigned long age); /** * Returns false if the transaction is in the mempool and not within the * chain limit specified. */ bool TransactionWithinChainLimit(const uint256 &txid, size_t chainLimit) const; unsigned long size() { LOCK(cs); return mapTx.size(); } uint64_t GetTotalTxSize() const { LOCK(cs); return totalTxSize; } bool exists(uint256 hash) const { LOCK(cs); return mapTx.count(hash) != 0; } bool exists(const COutPoint &outpoint) const { LOCK(cs); auto it = mapTx.find(outpoint.GetTxId()); return it != mapTx.end() && outpoint.GetN() < it->GetTx().vout.size(); } CTransactionRef get(const uint256 &hash) const; TxMempoolInfo info(const uint256 &hash) const; std::vector infoAll() const; CFeeRate estimateFee() const; size_t DynamicMemoryUsage() const; boost::signals2::signal NotifyEntryAdded; boost::signals2::signal NotifyEntryRemoved; private: /** * UpdateForDescendants is used by UpdateTransactionsFromBlock to update the * descendants for a single transaction that has been added to the mempool * but may have child transactions in the mempool, eg during a chain reorg. * setExclude is the set of descendant transactions in the mempool that must * not be accounted for (because any descendants in setExclude were added to * the mempool after the transaction being updated and hence their state is * already reflected in the parent state). * * cachedDescendants will be updated with the descendants of the transaction * being updated, so that future invocations don't need to walk the same * transaction again, if encountered in another transaction chain. */ void UpdateForDescendants(txiter updateIt, cacheMap &cachedDescendants, const std::set &setExclude); /** * Update ancestors of hash to add/remove it as a descendant transaction. */ void UpdateAncestorsOf(bool add, txiter hash, setEntries &setAncestors); /** Set ancestor state for an entry */ void UpdateEntryForAncestors(txiter it, const setEntries &setAncestors); /** * For each transaction being removed, update ancestors and any direct * children. If updateDescendants is true, then also update in-mempool * descendants' ancestor state. */ void UpdateForRemoveFromMempool(const setEntries &entriesToRemove, bool updateDescendants); /** Sever link between specified transaction and direct children. */ void UpdateChildrenForRemoval(txiter entry); /** * Before calling removeUnchecked for a given transaction, * UpdateForRemoveFromMempool must be called on the entire (dependent) set * of transactions being removed at the same time. We use each * CTxMemPoolEntry's setMemPoolParents in order to walk ancestors of a given * transaction that is removed, so we can't remove intermediate transactions * in a chain before we've updated all the state for the removal. */ void removeUnchecked(txiter entry, MemPoolRemovalReason reason = MemPoolRemovalReason::UNKNOWN); }; /** * CCoinsView that brings transactions from a memorypool into view. * It does not check for spendings by memory pool transactions. */ class CCoinsViewMemPool : public CCoinsViewBacked { protected: const CTxMemPool &mempool; public: CCoinsViewMemPool(CCoinsView *baseIn, const CTxMemPool &mempoolIn); bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; bool HaveCoin(const COutPoint &outpoint) const override; }; // We want to sort transactions by coin age priority typedef std::pair TxCoinAgePriority; struct TxCoinAgePriorityCompare { bool operator()(const TxCoinAgePriority &a, const TxCoinAgePriority &b) { if (a.first == b.first) { // Reverse order to make sort less than return CompareTxMemPoolEntryByScore()(*(b.second), *(a.second)); } return a.first < b.first; } }; /** * DisconnectedBlockTransactions * * During the reorg, it's desirable to re-add previously confirmed transactions * to the mempool, so that anything not re-confirmed in the new chain is * available to be mined. However, it's more efficient to wait until the reorg * is complete and process all still-unconfirmed transactions at that time, * since we expect most confirmed transactions to (typically) still be * confirmed in the new chain, and re-accepting to the memory pool is expensive * (and therefore better to not do in the middle of reorg-processing). * Instead, store the disconnected transactions (in order!) as we go, remove any * that are included in blocks in the new chain, and then process the remaining * still-unconfirmed transactions at the end. * * It also enables efficient reprocessing of current mempool entries, useful * when (de)activating forks that result in in-mempool transactions becoming * invalid */ // multi_index tag names struct txid_index {}; struct insertion_order {}; class DisconnectedBlockTransactions { private: typedef boost::multi_index_container< CTransactionRef, boost::multi_index::indexed_by< // sorted by txid boost::multi_index::hashed_unique< boost::multi_index::tag, mempoolentry_txid, SaltedTxidHasher>, // sorted by order in the blockchain boost::multi_index::sequenced< boost::multi_index::tag>>> indexed_disconnected_transactions; indexed_disconnected_transactions queuedTx; uint64_t cachedInnerUsage = 0; void addTransaction(const CTransactionRef &tx) { queuedTx.insert(tx); cachedInnerUsage += RecursiveDynamicUsage(tx); } public: // It's almost certainly a logic bug if we don't clear out queuedTx before // destruction, as we add to it while disconnecting blocks, and then we // need to re-process remaining transactions to ensure mempool consistency. // For now, assert() that we've emptied out this object on destruction. // This assert() can always be removed if the reorg-processing code were // to be refactored such that this assumption is no longer true (for // instance if there was some other way we cleaned up the mempool after a // reorg, besides draining this object). ~DisconnectedBlockTransactions() { assert(queuedTx.empty()); } // Estimate the overhead of queuedTx to be 6 pointers + an allocation, as // no exact formula for boost::multi_index_contained is implemented. size_t DynamicMemoryUsage() const { return memusage::MallocUsage(sizeof(CTransactionRef) + 6 * sizeof(void *)) * queuedTx.size() + cachedInnerUsage; } const indexed_disconnected_transactions &GetQueuedTx() const { return queuedTx; } // Import mempool entries in topological order into queuedTx and clear the // mempool. Caller should call updateMempoolForReorg to reprocess these // transactions void importMempool(CTxMemPool &pool); // Add entries for a block while reconstructing the topological ordering so // they can be added back to the mempool simply. void addForBlock(const std::vector &vtx); // Remove entries based on txid_index, and update memory usage. void removeForBlock(const std::vector &vtx) { // Short-circuit in the common case of a block being added to the tip if (queuedTx.empty()) { return; } for (auto const &tx : vtx) { auto it = queuedTx.find(tx->GetId()); if (it != queuedTx.end()) { cachedInnerUsage -= RecursiveDynamicUsage(*it); queuedTx.erase(it); } } } // Remove an entry by insertion_order index, and update memory usage. void removeEntry(indexed_disconnected_transactions::index< insertion_order>::type::iterator entry) { cachedInnerUsage -= RecursiveDynamicUsage(*entry); queuedTx.get().erase(entry); } bool isEmpty() const { return queuedTx.empty(); } void clear() { cachedInnerUsage = 0; queuedTx.clear(); } /** * Make mempool consistent after a reorg, by re-adding or recursively * erasing disconnected block transactions from the mempool, and also * removing any other transactions from the mempool that are no longer valid * given the new tip/height. * * Note: we assume that disconnectpool only contains transactions that are * NOT confirmed in the current chain nor already in the mempool (otherwise, * in-mempool descendants of such transactions would be removed). * * Passing fAddToMempool=false will skip trying to add the transactions * back, and instead just erase from the mempool as needed. */ void updateMempoolForReorg(const Config &config, bool fAddToMempool); }; #endif // BITCOIN_TXMEMPOOL_H diff --git a/src/undo.h b/src/undo.h index 391df8e870..62e99d080a 100644 --- a/src/undo.h +++ b/src/undo.h @@ -1,144 +1,145 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Copyright (c) 2017 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_UNDO_H #define BITCOIN_UNDO_H #include #include #include #include +#include class CBlock; class CBlockIndex; class CCoinsViewCache; class CValidationState; /** * Undo information for a CTxIn * * Contains the prevout's CTxOut being spent, and its metadata as well (coinbase * or not, height). The serialization contains a dummy value of zero. This is be * compatible with older versions which expect to see the transaction version * there. */ class TxInUndoSerializer { const Coin *pcoin; public: explicit TxInUndoSerializer(const Coin *pcoinIn) : pcoin(pcoinIn) {} template void Serialize(Stream &s) const { ::Serialize( s, VARINT(pcoin->GetHeight() * 2 + (pcoin->IsCoinBase() ? 1 : 0))); if (pcoin->GetHeight() > 0) { // Required to maintain compatibility with older undo format. ::Serialize(s, uint8_t(0)); } ::Serialize(s, CTxOutCompressor(REF(pcoin->GetTxOut()))); } }; class TxInUndoDeserializer { Coin *pcoin; public: explicit TxInUndoDeserializer(Coin *pcoinIn) : pcoin(pcoinIn) {} template void Unserialize(Stream &s) { uint32_t nCode = 0; ::Unserialize(s, VARINT(nCode)); uint32_t nHeight = nCode / 2; bool fCoinBase = nCode & 1; if (nHeight > 0) { // Old versions stored the version number for the last spend of a // transaction's outputs. Non-final spends were indicated with // height = 0. int nVersionDummy; ::Unserialize(s, VARINT(nVersionDummy)); } CTxOut txout; ::Unserialize(s, REF(CTxOutCompressor(REF(txout)))); *pcoin = Coin(std::move(txout), nHeight, fCoinBase); } }; static const size_t MAX_INPUTS_PER_TX = MAX_TX_SIZE / ::GetSerializeSize(CTxIn(), SER_NETWORK, PROTOCOL_VERSION); /** Restore the UTXO in a Coin at a given COutPoint */ class CTxUndo { public: // Undo information for all txins std::vector vprevout; template void Serialize(Stream &s) const { // TODO: avoid reimplementing vector serializer. uint64_t count = vprevout.size(); ::Serialize(s, COMPACTSIZE(REF(count))); for (const auto &prevout : vprevout) { ::Serialize(s, REF(TxInUndoSerializer(&prevout))); } } template void Unserialize(Stream &s) { // TODO: avoid reimplementing vector deserializer. uint64_t count = 0; ::Unserialize(s, COMPACTSIZE(count)); if (count > MAX_INPUTS_PER_TX) { throw std::ios_base::failure("Too many input undo records"); } vprevout.resize(count); for (auto &prevout : vprevout) { ::Unserialize(s, REF(TxInUndoDeserializer(&prevout))); } } }; /** Undo information for a CBlock */ class CBlockUndo { public: // For all but the coinbase std::vector vtxundo; ADD_SERIALIZE_METHODS; template inline void SerializationOp(Stream &s, Operation ser_action) { READWRITE(vtxundo); } }; enum DisconnectResult { // All good. DISCONNECT_OK, // Rolled back, but UTXO set was inconsistent with block. DISCONNECT_UNCLEAN, // Something else went wrong. DISCONNECT_FAILED, }; /** * Restore the UTXO in a Coin at a given COutPoint. * @param undo The Coin to be restored. * @param view The coins view to which to apply the changes. * @param out The out point that corresponds to the tx input. * @return A DisconnectResult */ DisconnectResult UndoCoinSpend(const Coin &undo, CCoinsViewCache &view, const COutPoint &out); /** * Undo a block from the block and the undoblock data. * See DisconnectBlock for more details. */ DisconnectResult ApplyBlockUndo(const CBlockUndo &blockUndo, const CBlock &block, const CBlockIndex *pindex, CCoinsViewCache &coins); #endif // BITCOIN_UNDO_H