diff --git a/doc/files.md b/doc/files.md --- a/doc/files.md +++ b/doc/files.md @@ -9,6 +9,7 @@ * database/*: BDB database environment; only used for wallet since 0.8.0; moved to wallets/ directory on new installs since 0.18.7 * db.log: wallet database log file; moved to wallets/ directory on new installs since 0.18.7 * debug.log: contains debug information and general logging generated by bitcoind or bitcoin-qt +* indexes/txindex/*: optional transaction index database (LevelDB); since 0.19.7 * mempool.dat: dump of the mempool's transactions; since 0.14.0. * peers.dat: peer IP address database (custom format); since 0.7.0 * wallet.dat: personal wallet (BDB) with keys and transactions; moved to wallets/ directory on new installs since 0.18.7 diff --git a/doc/release-notes.md b/doc/release-notes.md --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -14,3 +14,15 @@ - The `createrawtransaction` RPC will now accept an array or dictionary (kept for compatibility) for the `outputs` parameter. This means the order of transaction outputs can be specified by the client. - The new RPC `testmempoolaccept` can be used to test acceptance of a transaction to the mempool without adding it. - An `initialblockdownload` boolean has been added to the `getblockchaininfo` RPC to indicate whether the node is currently in IBD or not. + +Transaction index changes +------------------------- + +The transaction index is now built separately from the main node procedure, +meaning the `-txindex` flag can be toggled without a full reindex. If bitcoind +is run with `-txindex` on a node that is already partially or fully synced +without one, the transaction index will be built in the background and become +available once caught up. When switching from running `-txindex` to running +without the flag, the transaction index database will *not* be deleted +automatically, meaning it could be turned back on at a later time without a full +resync. diff --git a/src/Makefile.test.include b/src/Makefile.test.include --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -115,6 +115,7 @@ test/test_bitcoin_main.cpp \ test/timedata_tests.cpp \ test/transaction_tests.cpp \ + test/txindex_tests.cpp \ test/txvalidationcache_tests.cpp \ test/uint256_tests.cpp \ test/undo_tests.cpp \ diff --git a/src/index/txindex.h b/src/index/txindex.h --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -6,6 +6,7 @@ #define BITCOIN_INDEX_TXINDEX_H #include +#include #include #include #include @@ -71,8 +72,15 @@ /// this method does not block and immediately returns false. bool BlockUntilSyncedToCurrentChain(); - /// Look up the on-disk location of a transaction by hash. - bool FindTx(const uint256 &txid, CDiskTxPos &pos) const; + /// Look up a transaction by hash. + /// + /// @param[in] tx_hash The hash of the transaction to be returned. + /// @param[out] block_hash The hash of the block the transaction is found + /// in. + /// @param[out] tx The transaction itself. + /// @return true if transaction is found, false otherwise + bool FindTx(const uint256 &tx_hash, uint256 &block_hash, + CTransactionRef &tx) const; void Interrupt(); @@ -84,4 +92,7 @@ void Stop(); }; +/// The global transaction index, used in GetTransaction. May be null. +extern std::unique_ptr g_txindex; + #endif // BITCOIN_INDEX_TXINDEX_H diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -14,6 +14,8 @@ constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds +std::unique_ptr g_txindex; + template static void FatalError(const char *fmt, const Args &... args) { std::string strMessage = tfm::format(fmt, args...); @@ -251,8 +253,30 @@ return true; } -bool TxIndex::FindTx(const uint256 &txid, CDiskTxPos &pos) const { - return m_db->ReadTxPos(txid, pos); +bool TxIndex::FindTx(const uint256 &tx_hash, uint256 &block_hash, + CTransactionRef &tx) const { + CDiskTxPos postx; + if (!m_db->ReadTxPos(tx_hash, postx)) { + return false; + } + + CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + return error("%s: OpenBlockFile failed", __func__); + } + CBlockHeader header; + try { + file >> header; + fseek(file.Get(), postx.nTxOffset, SEEK_CUR); + file >> tx; + } catch (const std::exception &e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + if (tx->GetHash() != tx_hash) { + return error("%s: txid mismatch", __func__); + } + block_hash = header.GetHash(); + return true; } void TxIndex::Interrupt() { diff --git a/src/init.cpp b/src/init.cpp --- a/src/init.cpp +++ b/src/init.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -179,6 +180,9 @@ if (g_connman) { g_connman->Interrupt(); } + if (g_txindex) { + g_txindex->Interrupt(); + } } void Shutdown() { @@ -213,6 +217,9 @@ } peerLogic.reset(); g_connman.reset(); + if (g_txindex) { + g_txindex.reset(); + } StopTorControl(); @@ -2046,13 +2053,14 @@ nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be greater than nMaxDbcache nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); - int64_t nBlockTreeDBCache = nTotalCache / 8; - nBlockTreeDBCache = std::min(nBlockTreeDBCache, - (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) - ? nMaxBlockDBAndTxIndexCache - : nMaxBlockDBCache) - << 20); + int64_t nBlockTreeDBCache = + std::min(nTotalCache / 8, nMaxBlockDBCache << 20); nTotalCache -= nBlockTreeDBCache; + int64_t nTxIndexCache = + std::min(nTotalCache / 8, gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) + ? nMaxTxIndexCache << 20 + : 0); + nTotalCache -= nTxIndexCache; // use 25%-50% of the remainder for disk cache int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); @@ -2066,6 +2074,10 @@ LogPrintf("Cache configuration:\n"); LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); + if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + LogPrintf("* Using %.1fMiB for transaction index database\n", + nTxIndexCache * (1.0 / 1024 / 1024)); + } LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for in-memory UTXO set (plus up to %.1fMiB of " @@ -2106,9 +2118,8 @@ break; } - // LoadBlockIndex will load fTxIndex from the db, or set it if - // we're reindexing. It will also load fHavePruned if we've - // ever removed a block file from disk. + // LoadBlockIndex will load fHavePruned if we've ever removed a + // block file from disk. // Note that it also sets fReindex based on the disk flag! // From here on out fReindex and fReset mean something // different! @@ -2127,13 +2138,6 @@ "Wrong datadir for network?")); } - // Check for changed -txindex state - if (fTxIndex != gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - strLoadError = _("You need to rebuild the database using " - "-reindex-chainstate to change -txindex"); - break; - } - // Check for changed -prune state. What we are concerned about // is a user who has pruned blocks in the past, but is now // trying to run unpruned. @@ -2291,12 +2295,20 @@ config.SetCashAddrEncoding( gArgs.GetBoolArg("-usecashaddr", GetAdjustedTime() > 1515900000)); - // Step 8: load wallet + // Step 8: load indexers + if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + auto txindex_db = + std::make_unique(nTxIndexCache, false, fReindex); + g_txindex = std::make_unique(std::move(txindex_db)); + g_txindex->Start(); + } + + // Step 9: load wallet if (!g_wallet_init_interface.Open(chainparams)) { return false; } - // Step 9: data directory maintenance + // Step 10: data directory maintenance // if pruning, unset the service bit and perform the initial blockstore // prune after any wallet rescanning has taken place. @@ -2309,7 +2321,7 @@ } } - // Step 10: import blocks + // Step 11: import blocks if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ false)) { InitError( strprintf(_("Error: Disk space is low for %s"), GetDataDir())); @@ -2358,7 +2370,7 @@ return false; } - // Step 11: start node + // Step 12: start node int chain_active_height; @@ -2444,7 +2456,7 @@ return false; } - // Step 12: finished + // Step 13: finished SetRPCWarmupFinished(); uiInterface.InitMessage(_("Done loading")); diff --git a/src/rest.cpp b/src/rest.cpp --- a/src/rest.cpp +++ b/src/rest.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -403,6 +404,10 @@ const TxId txid(hash); + if (g_txindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + CTransactionRef tx; uint256 hashBlock = uint256(); if (!GetTransaction(config, txid, tx, hashBlock, true)) { diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,8 @@ TxToUniv(tx, uint256(), entry, true, RPCSerializationFlags()); if (!hashBlock.IsNull()) { + LOCK(cs_main); + entry.pushKV("blockhash", hashBlock.GetHex()); CBlockIndex *pindex = LookupBlockIndex(hashBlock); if (pindex) { @@ -161,8 +164,6 @@ "\"mytxid\" true \"myblockhash\"")); } - LOCK(cs_main); - bool in_active_chain = true; TxId txid = TxId(ParseHashV(request.params[0], "parameter 1")); CBlockIndex *blockindex = nullptr; @@ -183,6 +184,8 @@ } if (!request.params[2].isNull()) { + LOCK(cs_main); + uint256 blockhash = ParseHashV(request.params[2], "parameter 3"); blockindex = LookupBlockIndex(blockhash); if (!blockindex) { @@ -192,6 +195,11 @@ in_active_chain = chainActive.Contains(blockindex); } + bool f_txindex_ready = false; + if (g_txindex && !blockindex) { + f_txindex_ready = g_txindex->BlockUntilSyncedToCurrentChain(); + } + CTransactionRef tx; uint256 hash_block; if (!GetTransaction(config, txid, tx, hash_block, true, blockindex)) { @@ -201,10 +209,14 @@ throw JSONRPCError(RPC_MISC_ERROR, "Block not available"); } errmsg = "No such transaction found in the provided block"; + } else if (!g_txindex) { + errmsg = "No such mempool transaction. Use -txindex to enable " + "blockchain transaction queries"; + } else if (!f_txindex_ready) { + errmsg = "No such mempool transaction. Blockchain transactions are " + "still in the process of being indexed"; } else { - errmsg = fTxIndex ? "No such mempool or blockchain transaction" - : "No such mempool transaction. Use -txindex to " - "enable blockchain transaction queries"; + errmsg = "No such mempool or blockchain transaction"; } throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errmsg + @@ -274,18 +286,18 @@ oneTxId = txid; } - LOCK(cs_main); - CBlockIndex *pblockindex = nullptr; uint256 hashBlock; if (!request.params[1].isNull()) { + LOCK(cs_main); hashBlock = uint256S(request.params[1].get_str()); pblockindex = LookupBlockIndex(hashBlock); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } } else { + LOCK(cs_main); // Loop through txids and try to find which block they're in. Exit loop // once a block is found. for (const auto &txid : setTxIds) { @@ -297,6 +309,14 @@ } } + // Allow txindex to catch up if we need to query it and before we acquire + // cs_main. + if (g_txindex && !pblockindex) { + g_txindex->BlockUntilSyncedToCurrentChain(); + } + + LOCK(cs_main); + if (pblockindex == nullptr) { CTransactionRef tx; if (!GetTransaction(config, oneTxId, tx, hashBlock, false) || diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -128,6 +128,7 @@ test_bitcoin_main.cpp timedata_tests.cpp transaction_tests.cpp + txindex_tests.cpp txvalidationcache_tests.cpp uint256_tests.cpp undo_tests.cpp diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp new file mode 100644 --- /dev/null +++ b/src/test/txindex_tests.cpp @@ -0,0 +1,76 @@ +// Copyright (c) 2017-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include