diff --git a/src/txdb.h b/src/txdb.h --- a/src/txdb.h +++ b/src/txdb.h @@ -156,6 +156,11 @@ /// Write block locator of the chain that the txindex is in sync with. bool WriteBestBlock(const CBlockLocator &locator); + + /// Migrate txindex data from the block tree DB, where it may be for older + /// nodes that have not been upgraded yet to the new database. + bool MigrateData(CBlockTreeDB &block_tree_db, + const CBlockLocator &best_locator); }; #endif // BITCOIN_TXDB_H diff --git a/src/txdb.cpp b/src/txdb.cpp --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -22,6 +22,7 @@ static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; +static const char DB_TXINDEX_BLOCK = 'T'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -483,3 +484,145 @@ bool TxIndexDB::WriteBestBlock(const CBlockLocator &locator) { return Write(DB_BEST_BLOCK, locator); } + +/* + * Safely persist a transfer of data from the old txindex database to the new + * one, and compact the range of keys updated. This is used internally by + * MigrateData. + */ +static void +WriteTxIndexMigrationBatches(TxIndexDB &newdb, CBlockTreeDB &olddb, + CDBBatch &batch_newdb, CDBBatch &batch_olddb, + const std::pair &begin_key, + const std::pair &end_key) { + // Sync new DB changes to disk before deleting from old DB. + newdb.WriteBatch(batch_newdb, /*fSync=*/true); + olddb.WriteBatch(batch_olddb); + olddb.CompactRange(begin_key, end_key); + + batch_newdb.Clear(); + batch_olddb.Clear(); +} + +bool TxIndexDB::MigrateData(CBlockTreeDB &block_tree_db, + const CBlockLocator &best_locator) { + // The prior implementation of txindex was always in sync with block index + // and presence was indicated with a boolean DB flag. If the flag is set, + // this means the txindex from a previous version is valid and in sync with + // the chain tip. The first step of the migration is to unset the flag and + // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the + // index entries are copied over in batches to the new database. Finally, + // DB_TXINDEX_BLOCK is erased from the old database and the block hash is + // written to the new database. + // + // Unsetting the boolean flag ensures that if the node is downgraded to a + // previous version, it will not see a corrupted, partially migrated index + // -- it will see that the txindex is disabled. When the node is upgraded + // again, the migration will pick up where it left off and sync to the block + // with hash DB_TXINDEX_BLOCK. + bool f_legacy_flag = false; + block_tree_db.ReadFlag("txindex", f_legacy_flag); + if (f_legacy_flag) { + if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) { + return error("%s: cannot write block indicator", __func__); + } + if (!block_tree_db.WriteFlag("txindex", false)) { + return error("%s: cannot write block index db flag", __func__); + } + } + + CBlockLocator locator; + if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) { + return true; + } + + int64_t count = 0; + LogPrintf("Upgrading txindex database... [0%%]\n"); + uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true); + int report_done = 0; + const size_t batch_size = 1 << 24; // 16 MiB + + CDBBatch batch_newdb(*this); + CDBBatch batch_olddb(block_tree_db); + + std::pair key; + std::pair begin_key{DB_TXINDEX, uint256()}; + std::pair prev_key = begin_key; + + bool interrupted = false; + std::unique_ptr cursor(block_tree_db.NewIterator()); + for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) { + boost::this_thread::interruption_point(); + if (ShutdownRequested()) { + interrupted = true; + break; + } + + if (!cursor->GetKey(key)) { + return error("%s: cannot get key from valid cursor", __func__); + } + if (key.first != DB_TXINDEX) { + break; + } + + // Log progress every 10%. + if (++count % 256 == 0) { + // Since txids are uniformly random and traversed in increasing + // order, the high 16 bits of the hash can be used to estimate the + // current progress. + const uint256 &txid = key.second; + uint32_t high_nibble = + (static_cast(*(txid.begin() + 0)) << 8) + + (static_cast(*(txid.begin() + 1)) << 0); + int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5); + + uiInterface.ShowProgress(_("Upgrading txindex database"), + percentage_done, true); + if (report_done < percentage_done / 10) { + LogPrintf("Upgrading txindex database... [%d%%]\n", + percentage_done); + report_done = percentage_done / 10; + } + } + + CDiskTxPos value; + if (!cursor->GetValue(value)) { + return error("%s: cannot parse txindex record", __func__); + } + batch_newdb.Write(key, value); + batch_olddb.Erase(key); + + if (batch_newdb.SizeEstimate() > batch_size || + batch_olddb.SizeEstimate() > batch_size) { + // NOTE: it's OK to delete the key pointed at by the current DB + // cursor while iterating because LevelDB iterators are guaranteed + // to provide a consistent view of the underlying data, like a + // lightweight snapshot. + WriteTxIndexMigrationBatches(*this, block_tree_db, batch_newdb, + batch_olddb, prev_key, key); + prev_key = key; + } + } + + // If these final DB batches complete the migration, write the best block + // hash marker to the new database and delete from the old one. This signals + // that the former is fully caught up to that point in the blockchain and + // that all txindex entries have been removed from the latter. + if (!interrupted) { + batch_olddb.Erase(DB_TXINDEX_BLOCK); + batch_newdb.Write(DB_BEST_BLOCK, locator); + } + + WriteTxIndexMigrationBatches(*this, block_tree_db, batch_newdb, batch_olddb, + begin_key, key); + + if (interrupted) { + LogPrintf("[CANCELLED].\n"); + return false; + } + + uiInterface.ShowProgress("", 100, false); + + LogPrintf("[DONE].\n"); + return true; +}