diff --git a/contrib/devtools/utxo_snapshot.sh b/contrib/devtools/utxo_snapshot.sh new file mode 100755 --- /dev/null +++ b/contrib/devtools/utxo_snapshot.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +export LC_ALL=C + +set -ueo pipefail + +if (( $# < 3 )); then + echo 'Usage: utxo_snapshot.sh ' + echo + echo " if is '-', don't produce a snapshot file but instead print the " + echo " expected assumeutxo hash" + echo + echo 'Examples:' + echo + echo " ./contrib/devtools/utxo_snapshot.sh 570000 utxo.dat ./src/bitcoin-cli -datadir=\$(pwd)/testdata" + echo ' ./contrib/devtools/utxo_snapshot.sh 570000 - ./src/bitcoin-cli' + exit 1 +fi + +GENERATE_AT_HEIGHT="${1}"; shift; +OUTPUT_PATH="${1}"; shift; +# Most of the calls we make take a while to run, so pad with a lengthy timeout. +BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999" + +# Block we'll invalidate/reconsider to rewind/fast-forward the chain. +PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) ) + +(>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while") +${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}" + +if [[ "${OUTPUT_PATH}" = "-" ]]; then + (>&2 echo "Generating txoutset info...") + ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g' +else + (>&2 echo "Generating UTXO snapshot...") + ${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}" +fi + +(>&2 echo "Restoring chain to original height; this may take a while") +${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}" diff --git a/src/node/coinstats.h b/src/node/coinstats.h --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -15,18 +15,17 @@ class CCoinsView; struct CCoinsStats { - int nHeight; - BlockHash hashBlock; - uint64_t nTransactions; - uint64_t nTransactionOutputs; - uint64_t nBogoSize; - uint256 hashSerialized; - uint64_t nDiskSize; - Amount nTotalAmount; - - CCoinsStats() - : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), - nDiskSize(0), nTotalAmount() {} + int nHeight{0}; + BlockHash hashBlock{}; + uint64_t nTransactions{0}; + uint64_t nTransactionOutputs{0}; + uint64_t nBogoSize{0}; + uint256 hashSerialized{}; + uint64_t nDiskSize{0}; + Amount nTotalAmount{Amount::zero()}; + + //! The number of coins contained. + uint64_t coins_count{0}; }; //! Calculate statistics about the unspent transaction output set diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -39,6 +39,7 @@ //! Calculate statistics about the unspent transaction output set bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) { + stats = CCoinsStats(); std::unique_ptr pcursor(view->Cursor()); assert(pcursor); @@ -62,6 +63,7 @@ } prevkey = key.GetTxId(); outputs[key.GetN()] = std::move(coin); + stats.coins_count++; } else { return error("%s: unable to read value", __func__); } diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h new file mode 100644 --- /dev/null +++ b/src/node/utxo_snapshot.h @@ -0,0 +1,44 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H +#define BITCOIN_NODE_UTXO_SNAPSHOT_H + +#include +#include + +//! Metadata describing a serialized version of a UTXO set from which an +//! assumeutxo CChainState can be constructed. +class SnapshotMetadata { +public: + //! The hash of the block that reflects the tip of the chain for the + //! UTXO set contained in this snapshot. + BlockHash m_base_blockhash; + + //! The number of coins in the UTXO set contained in this snapshot. Used + //! during snapshot load to estimate progress of UTXO set reconstruction. + uint64_t m_coins_count = 0; + + //! Necessary to "fake" the base nChainTx so that we can estimate progress + //! during initial block download for the assumeutxo chainstate. + uint64_t m_nchaintx = 0; + + SnapshotMetadata() {} + SnapshotMetadata(const BlockHash &base_blockhash, uint64_t coins_count, + uint64_t nchaintx) + : m_base_blockhash(base_blockhash), m_coins_count(coins_count), + m_nchaintx(nchaintx) {} + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream &s, Operation ser_action) { + READWRITE(m_base_blockhash); + READWRITE(m_coins_count); + READWRITE(m_nchaintx); + } +}; + +#endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -2663,6 +2664,115 @@ return ret; } +/** + * Serialize the UTXO set to a file for loading elsewhere. + * + * @see SnapshotMetadata + */ +static UniValue dumptxoutset(const Config &config, + const JSONRPCRequest &request) { + RPCHelpMan{"dumptxoutset", + "\nWrite the serialized UTXO set to disk.\n" + "Incidentally flushes the latest coinsdb (leveldb) to disk.\n", + { + {"path", RPCArg::Type::STR, RPCArg::Optional::NO, + /* default_val */ "", + "path to the output file. If relative, will be prefixed by " + "datadir."}, + }, + RPCResult{"{\n" + " \"coins_written\": n, (numeric) the number of " + "coins written in the snapshot\n" + " \"base_hash\": \"...\", (string) the hash of the " + "base of the snapshot\n" + " \"base_height\": n, (string) the height of the " + "base of the snapshot\n" + " \"path\": \"...\" (string) the absolute " + "path that the snapshot was written to\n" + "]\n"}, + RPCExamples{HelpExampleCli("dumptxoutset", "utxo.dat")}} + .Check(request); + + fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir()); + // Write to a temporary path and then move into `path` on completion + // to avoid confusion due to an interruption. + fs::path temppath = + fs::absolute(request.params[0].get_str() + ".incomplete", GetDataDir()); + + if (fs::exists(path)) { + throw JSONRPCError( + RPC_INVALID_PARAMETER, + path.string() + + " already exists. If you are sure this is what you want, " + "move it out of the way first"); + } + + FILE *file{fsbridge::fopen(temppath, "wb")}; + CAutoFile afile{file, SER_DISK, CLIENT_VERSION}; + std::unique_ptr pcursor; + CCoinsStats stats; + CBlockIndex *tip; + + { + // We need to lock cs_main to ensure that the coinsdb isn't written to + // between (i) flushing coins cache to disk (coinsdb), (ii) getting + // stats based upon the coinsdb, and (iii) constructing a cursor to the + // coinsdb for use below this block. + // + // Cursors returned by leveldb iterate over snapshots, so the contents + // of the pcursor will not be affected by simultaneous writes during + // use below this block. + // + // See discussion here: + // https://github.com/bitcoin/bitcoin/pull/15606#discussion_r274479369 + // + LOCK(::cs_main); + + ::ChainstateActive().ForceFlushStateToDisk(); + + if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); + } + + pcursor = std::unique_ptr( + ::ChainstateActive().CoinsDB().Cursor()); + tip = LookupBlockIndex(stats.hashBlock); + CHECK_NONFATAL(tip); + } + + SnapshotMetadata metadata{tip->GetBlockHash(), stats.coins_count, + uint64_t(tip->GetChainTxCount())}; + + afile << metadata; + + COutPoint key; + Coin coin; + unsigned int iter{0}; + + while (pcursor->Valid()) { + if (iter % 5000 == 0 && !IsRPCRunning()) { + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); + } + ++iter; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + afile << key; + afile << coin; + } + + pcursor->Next(); + } + + afile.fclose(); + fs::rename(temppath, path); + + UniValue result(UniValue::VOBJ); + result.pushKV("coins_written", stats.coins_count); + result.pushKV("base_hash", tip->GetBlockHash().ToString()); + result.pushKV("base_height", tip->nHeight); + result.pushKV("path", path.string()); + return result; +} + // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -2698,6 +2808,7 @@ { "hidden", "parkblock", parkblock, {"blockhash"} }, { "hidden", "reconsiderblock", reconsiderblock, {"blockhash"} }, { "hidden", "syncwithvalidationinterfacequeue", syncwithvalidationinterfacequeue, {} }, + { "hidden", "dumptxoutset", dumptxoutset, {"path"} }, { "hidden", "unparkblock", unparkblock, {"blockhash"} }, { "hidden", "waitfornewblock", waitfornewblock, {"timeout"} }, { "hidden", "waitforblock", waitforblock, {"blockhash","timeout"} }, diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py new file mode 100755 --- /dev/null +++ b/test/functional/rpc_dumptxoutset.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the generation of UTXO snapshots using `dumptxoutset`. +""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_raises_rpc_error + +import hashlib +from pathlib import Path + + +class DumptxoutsetTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + """Test a trivial usage of the dumptxoutset RPC command.""" + node = self.nodes[0] + mocktime = node.getblockheader(node.getblockhash(0))['time'] + 1 + node.setmocktime(mocktime) + node.generate(100) + + FILENAME = 'txoutset.dat' + out = node.dumptxoutset(FILENAME) + expected_path = Path(node.datadir) / 'regtest' / FILENAME + + assert expected_path.is_file() + + assert_equal(out['coins_written'], 100) + assert_equal(out['base_height'], 100) + assert_equal(out['path'], str(expected_path)) + # Blockhash should be deterministic based on mocked time. + assert_equal( + out['base_hash'], + '65d0aec2439aae14373c153f596fb90a87b643d9bff3e65f250aa8f055e6816b') + + with open(str(expected_path), 'rb') as f: + digest = hashlib.sha256(f.read()).hexdigest() + # UTXO snapshot hash should be deterministic based on mocked time. + assert_equal( + digest, + '05957e146e38153d84e9294999cc24f0dcdb9902c4834b32c79ae8e8985babea') + + # Specifying a path to an existing file will fail. + assert_raises_rpc_error( + -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME) + + +if __name__ == '__main__': + DumptxoutsetTest().main()