diff --git a/src/dbwrapper.h b/src/dbwrapper.h index d4d91d3b58..a43e43b552 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -1,316 +1,332 @@ // Copyright (c) 2012-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_DBWRAPPER_H #define BITCOIN_DBWRAPPER_H #include "clientversion.h" #include "fs.h" #include "serialize.h" #include "streams.h" #include "util.h" #include "utilstrencodings.h" #include "version.h" #include #include static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64; static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024; class dbwrapper_error : public std::runtime_error { public: dbwrapper_error(const std::string &msg) : std::runtime_error(msg) {} }; class CDBWrapper; /** * These should be considered an implementation detail of the specific database. */ namespace dbwrapper_private { /** * Handle database error by throwing dbwrapper_error exception. */ void HandleError(const leveldb::Status &status); /** * Work around circular dependency, as well as for testing in dbwrapper_tests. * Database obfuscation should be considered an implementation detail of the * specific database. */ const std::vector &GetObfuscateKey(const CDBWrapper &w); }; // namespace dbwrapper_private /** Batch of changes queued to be written to a CDBWrapper */ class CDBBatch { friend class CDBWrapper; private: const CDBWrapper &parent; leveldb::WriteBatch batch; CDataStream ssKey; CDataStream ssValue; size_t size_estimate; public: /** * @param[in] _parent CDBWrapper that this batch is to be submitted to */ CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION), size_estimate(0){}; void Clear() { batch.Clear(); size_estimate = 0; } template void Write(const K &key, const V &value) { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey(ssKey.data(), ssKey.size()); ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE); ssValue << value; ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); leveldb::Slice slValue(ssValue.data(), ssValue.size()); batch.Put(slKey, slValue); // LevelDB serializes writes as: // - byte: header // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...) // - byte[]: key // - varint: value length // - byte[]: value // The formula below assumes the key and value are both less than 16k. size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size(); ssKey.clear(); ssValue.clear(); } template void Erase(const K &key) { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey(ssKey.data(), ssKey.size()); batch.Delete(slKey); // LevelDB serializes erases as: // - byte: header // - varint: key length // - byte[]: key // The formula below assumes the key is less than 16kB. size_estimate += 2 + (slKey.size() > 127) + slKey.size(); ssKey.clear(); } size_t SizeEstimate() const { return size_estimate; } }; class CDBIterator { private: const CDBWrapper &parent; leveldb::Iterator *piter; public: /** * @param[in] _parent Parent CDBWrapper instance. * @param[in] _piter The original leveldb iterator. */ CDBIterator(const CDBWrapper &_parent, leveldb::Iterator *_piter) : parent(_parent), piter(_piter){}; ~CDBIterator(); bool Valid(); void SeekToFirst(); template void Seek(const K &key) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey(ssKey.data(), ssKey.size()); piter->Seek(slKey); } void Next(); template bool GetKey(K &key) { leveldb::Slice slKey = piter->key(); try { CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION); ssKey >> key; } catch (const std::exception &) { return false; } return true; } unsigned int GetKeySize() { return piter->key().size(); } template bool GetValue(V &value) { leveldb::Slice slValue = piter->value(); try { CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); ssValue >> value; } catch (const std::exception &) { return false; } return true; } unsigned int GetValueSize() { return piter->value().size(); } }; class CDBWrapper { friend const std::vector & dbwrapper_private::GetObfuscateKey(const CDBWrapper &w); private: //! custom environment this database is using (may be nullptr in case of //! default environment) leveldb::Env *penv; //! database options used leveldb::Options options; //! options used when reading from the database leveldb::ReadOptions readoptions; //! options used when iterating over values of the database leveldb::ReadOptions iteroptions; //! options used when writing to the database leveldb::WriteOptions writeoptions; //! options used when sync writing to the database leveldb::WriteOptions syncoptions; //! the database itself leveldb::DB *pdb; //! a key used for optional XOR-obfuscation of the database std::vector obfuscate_key; //! the key under which the obfuscation key is stored static const std::string OBFUSCATE_KEY_KEY; //! the length of the obfuscate key in number of bytes static const unsigned int OBFUSCATE_KEY_NUM_BYTES; std::vector CreateObfuscateKey() const; public: /** * @param[in] path Location in the filesystem where leveldb data will * be stored. * @param[in] nCacheSize Configures various leveldb cache settings. * @param[in] fMemory If true, use leveldb's memory environment. * @param[in] fWipe If true, remove all existing data. * @param[in] obfuscate If true, store data obfuscated via simple XOR. If * false, XOR * with a zero'd byte array. */ CDBWrapper(const fs::path &path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); ~CDBWrapper(); template bool Read(const K &key, V &value) const { CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey(ssKey.data(), ssKey.size()); std::string strValue; leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); if (!status.ok()) { if (status.IsNotFound()) return false; LogPrintf("LevelDB read failure: %s\n", status.ToString()); dbwrapper_private::HandleError(status); } try { CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); ssValue.Xor(obfuscate_key); ssValue >> value; } catch (const std::exception &) { return false; } return true; } template bool Write(const K &key, const V &value, bool fSync = false) { CDBBatch batch(*this); batch.Write(key, value); return WriteBatch(batch, fSync); } template bool Exists(const K &key) const { CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; leveldb::Slice slKey(ssKey.data(), ssKey.size()); std::string strValue; leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); if (!status.ok()) { if (status.IsNotFound()) return false; LogPrintf("LevelDB read failure: %s\n", status.ToString()); dbwrapper_private::HandleError(status); } return true; } template bool Erase(const K &key, bool fSync = false) { CDBBatch batch(*this); batch.Erase(key); return WriteBatch(batch, fSync); } bool WriteBatch(CDBBatch &batch, bool fSync = false); // not available for LevelDB; provide for compatibility with BDB bool Flush() { return true; } bool Sync() { CDBBatch batch(*this); return WriteBatch(batch, true); } CDBIterator *NewIterator() { return new CDBIterator(*this, pdb->NewIterator(iteroptions)); } /** * Return true if the database managed by this class contains no entries. */ bool IsEmpty(); template size_t EstimateSize(const K &key_begin, const K &key_end) const { CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION); ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey1 << key_begin; ssKey2 << key_end; leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); uint64_t size = 0; leveldb::Range range(slKey1, slKey2); pdb->GetApproximateSizes(&range, 1, &size); return size; } + + /** + * Compact a certain range of keys in the database. + */ + template + void CompactRange(const K &key_begin, const K &key_end) const { + CDataStream ssKey1(SER_DISK, CLIENT_VERSION), + ssKey2(SER_DISK, CLIENT_VERSION); + ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey1 << key_begin; + ssKey2 << key_end; + leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); + leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); + pdb->CompactRange(&slKey1, &slKey2); + } }; #endif // BITCOIN_DBWRAPPER_H diff --git a/src/txdb.cpp b/src/txdb.cpp index a1bdf5d4da..8adde0c699 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -1,402 +1,406 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "txdb.h" #include "chainparams.h" #include "config.h" #include "hash.h" #include "init.h" #include "pow.h" #include "ui_interface.h" #include "uint256.h" #include #include static const char DB_COIN = 'C'; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; static const char DB_FLAG = 'F'; static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; namespace { struct CoinEntry { COutPoint *outpoint; char key; CoinEntry(const COutPoint *ptr) : outpoint(const_cast(ptr)), key(DB_COIN) {} template void Serialize(Stream &s) const { s << key; s << outpoint->hash; s << VARINT(outpoint->n); } template void Unserialize(Stream &s) { s >> key; s >> outpoint->hash; s >> VARINT(outpoint->n); } }; } // namespace CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) {} bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { return db.Read(CoinEntry(&outpoint), coin); } bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const { return db.Exists(CoinEntry(&outpoint)); } uint256 CCoinsViewDB::GetBestBlock() const { uint256 hashBestChain; if (!db.Read(DB_BEST_BLOCK, hashBestChain)) return uint256(); return hashBestChain; } bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { CDBBatch batch(db); size_t count = 0; size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { CoinEntry entry(&it->first); if (it->second.coin.IsSpent()) { batch.Erase(entry); } else { batch.Write(entry, it->second.coin); } changed++; } count++; CCoinsMap::iterator itOld = it++; mapCoins.erase(itOld); } if (!hashBlock.IsNull()) { batch.Write(DB_BEST_BLOCK, hashBlock); } bool ret = db.WriteBatch(batch); LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of " "%u) to coin database...\n", (unsigned int)changed, (unsigned int)count); return ret; } size_t CCoinsViewDB::EstimateSize() const { return db.EstimateSize(DB_COIN, char(DB_COIN + 1)); } CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) {} bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { return Read(std::make_pair(DB_BLOCK_FILES, nFile), info); } bool CBlockTreeDB::WriteReindexing(bool fReindexing) { if (fReindexing) return Write(DB_REINDEX_FLAG, '1'); else return Erase(DB_REINDEX_FLAG); } bool CBlockTreeDB::ReadReindexing(bool &fReindexing) { fReindexing = Exists(DB_REINDEX_FLAG); return true; } bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { return Read(DB_LAST_BLOCK, nFile); } CCoinsViewCursor *CCoinsViewDB::Cursor() const { CCoinsViewDBCursor *i = new CCoinsViewDBCursor( const_cast(db).NewIterator(), GetBestBlock()); /** * It seems that there are no "const iterators" for LevelDB. Since we only * need read operations on it, use a const-cast to get around that * restriction. */ i->pcursor->Seek(DB_COIN); // Cache key of first record if (i->pcursor->Valid()) { CoinEntry entry(&i->keyTmp.second); i->pcursor->GetKey(entry); i->keyTmp.first = entry.key; } else { // Make sure Valid() and GetKey() return false i->keyTmp.first = 0; } return i; } bool CCoinsViewDBCursor::GetKey(COutPoint &key) const { // Return cached key if (keyTmp.first == DB_COIN) { key = keyTmp.second; return true; } return false; } bool CCoinsViewDBCursor::GetValue(Coin &coin) const { return pcursor->GetValue(coin); } unsigned int CCoinsViewDBCursor::GetValueSize() const { return pcursor->GetValueSize(); } bool CCoinsViewDBCursor::Valid() const { return keyTmp.first == DB_COIN; } void CCoinsViewDBCursor::Next() { pcursor->Next(); CoinEntry entry(&keyTmp.second); if (!pcursor->Valid() || !pcursor->GetKey(entry)) { // Invalidate cached key after last record so that Valid() and GetKey() // return false keyTmp.first = 0; } else { keyTmp.first = entry.key; } } bool CBlockTreeDB::WriteBatchSync( const std::vector> &fileInfo, int nLastFile, const std::vector &blockinfo) { CDBBatch batch(*this); for (std::vector>::const_iterator it = fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(std::make_pair(DB_BLOCK_FILES, it->first), *it->second); } batch.Write(DB_LAST_BLOCK, nLastFile); for (std::vector::const_iterator it = blockinfo.begin(); it != blockinfo.end(); it++) { batch.Write(std::make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it)); } return WriteBatch(batch, true); } bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { return Read(std::make_pair(DB_TXINDEX, txid), pos); } bool CBlockTreeDB::WriteTxIndex( const std::vector> &vect) { CDBBatch batch(*this); for (std::vector>::const_iterator it = vect.begin(); it != vect.end(); it++) batch.Write(std::make_pair(DB_TXINDEX, it->first), it->second); return WriteBatch(batch); } bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { char ch; if (!Read(std::make_pair(DB_FLAG, name), ch)) return false; fValue = ch == '1'; return true; } bool CBlockTreeDB::LoadBlockIndexGuts( std::function insertBlockIndex) { const Config &config = GetConfig(); std::unique_ptr pcursor(NewIterator()); pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256())); // Load mapBlockIndex while (pcursor->Valid()) { boost::this_thread::interruption_point(); std::pair key; if (!pcursor->GetKey(key) || key.first != DB_BLOCK_INDEX) { break; } CDiskBlockIndex diskindex; if (!pcursor->GetValue(diskindex)) { return error("LoadBlockIndex() : failed to read value"); } // Construct block index object CBlockIndex *pindexNew = insertBlockIndex(diskindex.GetBlockHash()); pindexNew->pprev = insertBlockIndex(diskindex.hashPrev); pindexNew->nHeight = diskindex.nHeight; pindexNew->nFile = diskindex.nFile; pindexNew->nDataPos = diskindex.nDataPos; pindexNew->nUndoPos = diskindex.nUndoPos; pindexNew->nVersion = diskindex.nVersion; pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; pindexNew->nTime = diskindex.nTime; pindexNew->nBits = diskindex.nBits; pindexNew->nNonce = diskindex.nNonce; pindexNew->nStatus = diskindex.nStatus; pindexNew->nTx = diskindex.nTx; if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, config)) { return error("LoadBlockIndex(): CheckProofOfWork failed: %s", pindexNew->ToString()); } pcursor->Next(); } return true; } namespace { //! Legacy class to deserialize pre-pertxout database entries without reindex. class CCoins { public: //! whether transaction is a coinbase bool fCoinBase; //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs //! at the end of the array are dropped std::vector vout; //! at which height this transaction was included in the active block chain int nHeight; //! empty constructor CCoins() : fCoinBase(false), vout(0), nHeight(0) {} template void Unserialize(Stream &s) { uint32_t nCode = 0; // version int nVersionDummy; ::Unserialize(s, VARINT(nVersionDummy)); // header code ::Unserialize(s, VARINT(nCode)); fCoinBase = nCode & 1; std::vector vAvail(2, false); vAvail[0] = (nCode & 2) != 0; vAvail[1] = (nCode & 4) != 0; uint32_t nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); // spentness bitmask while (nMaskCode > 0) { uint8_t chAvail = 0; ::Unserialize(s, chAvail); for (unsigned int p = 0; p < 8; p++) { bool f = (chAvail & (1 << p)) != 0; vAvail.push_back(f); } if (chAvail != 0) { nMaskCode--; } } // txouts themself vout.assign(vAvail.size(), CTxOut()); for (size_t i = 0; i < vAvail.size(); i++) { if (vAvail[i]) { ::Unserialize(s, REF(CTxOutCompressor(vout[i]))); } } // coinbase height ::Unserialize(s, VARINT(nHeight)); } }; } // namespace /** * Upgrade the database from older formats. * * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout. */ bool CCoinsViewDB::Upgrade() { std::unique_ptr pcursor(db.NewIterator()); pcursor->Seek(std::make_pair(DB_COINS, uint256())); if (!pcursor->Valid()) { return true; } int64_t count = 0; LogPrintf("Upgrading utxo-set database...\n"); LogPrintf("[0%%]..."); size_t batch_size = 1 << 24; CDBBatch batch(db); uiInterface.SetProgressBreakAction(StartShutdown); int reportDone = 0; + std::pair key; + std::pair prev_key = {DB_COINS, uint256()}; while (pcursor->Valid()) { boost::this_thread::interruption_point(); if (ShutdownRequested()) { break; } - std::pair key; if (!pcursor->GetKey(key) || key.first != DB_COINS) { break; } if (count++ % 256 == 0) { uint32_t high = 0x100 * *key.second.begin() + *(key.second.begin() + 1); int percentageDone = (int)(high * 100.0 / 65536.0 + 0.5); uiInterface.ShowProgress( _("Upgrading UTXO database") + "\n" + _("(press q to shutdown and continue later)") + "\n", percentageDone); if (reportDone < percentageDone / 10) { // report max. every 10% step LogPrintf("[%d%%]...", percentageDone); reportDone = percentageDone / 10; } } CCoins old_coins; if (!pcursor->GetValue(old_coins)) { return error("%s: cannot parse CCoins record", __func__); } COutPoint outpoint(key.second, 0); for (size_t i = 0; i < old_coins.vout.size(); ++i) { if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) { Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase); outpoint.n = i; CoinEntry entry(&outpoint); batch.Write(entry, newcoin); } } batch.Erase(key); if (batch.SizeEstimate() > batch_size) { db.WriteBatch(batch); batch.Clear(); + db.CompactRange(prev_key, key); + prev_key = key; } pcursor->Next(); } db.WriteBatch(batch); + db.CompactRange({DB_COINS, uint256()}, key); uiInterface.SetProgressBreakAction(std::function()); LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE"); return !ShutdownRequested(); }