diff --git a/src/wallet/db.h b/src/wallet/db.h --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -54,8 +54,8 @@ */ enum VerifyResult { VERIFY_OK, RECOVER_OK, RECOVER_FAIL }; VerifyResult Verify(const std::string &strFile, - bool (*recoverFunc)(CDBEnv &dbenv, - const std::string &strFile)); + bool (*recoverFunc)(const std::string &strFile)); + /** * Salvage data from a file that Verify says is bad. * fAggressive sets the DB_AGGRESSIVE flag (see berkeley DB->verify() method @@ -102,6 +102,23 @@ public: void Flush(); void Close(); + static bool Recover(const std::string &filename, void *callbackDataIn, + bool (*recoverKVcallback)(void *callbackData, + CDataStream ssKey, + CDataStream ssValue)); + + /* flush the wallet passively (TRY_LOCK) + ideal to be called periodically */ + static bool PeriodicFlush(std::string strFile); + /* verifies the database environment */ + static bool VerifyEnvironment(const std::string &walletFile, + const fs::path &dataDir, + std::string &errorStr); + /* verifies the database file */ + static bool + VerifyDatabaseFile(const std::string &walletFile, const fs::path &dataDir, + std::string &warningStr, std::string &errorStr, + bool (*recoverFunc)(const std::string &strFile)); private: CDB(const CDB &); diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -145,7 +145,7 @@ CDBEnv::VerifyResult CDBEnv::Verify(const std::string &strFile, - bool (*recoverFunc)(CDBEnv &dbenv, const std::string &strFile)) { + bool (*recoverFunc)(const std::string &strFile)) { LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); @@ -158,10 +158,136 @@ } // Try to recover: - bool fRecovered = (*recoverFunc)(*this, strFile); + bool fRecovered = (*recoverFunc)(strFile); return (fRecovered ? RECOVER_OK : RECOVER_FAIL); } +bool CDB::Recover(const std::string &filename, void *callbackDataIn, + bool (*recoverKVcallback)(void *callbackData, + CDataStream ssKey, + CDataStream ssValue)) { + // Recovery procedure: + // move wallet file to wallet.timestamp.bak + // Call Salvage with fAggressive=true to get as much data as possible. + // Rewrite salvaged data to fresh wallet file. + // Set -rescan so any missing transactions will be found. + int64_t now = GetTime(); + std::string newFilename = strprintf("wallet.%d.bak", now); + + int result = bitdb.dbenv->dbrename(nullptr, filename.c_str(), nullptr, + newFilename.c_str(), DB_AUTO_COMMIT); + if (result == 0) { + LogPrintf("Renamed %s to %s\n", filename, newFilename); + } else { + LogPrintf("Failed to rename %s to %s\n", filename, newFilename); + return false; + } + + std::vector salvagedData; + bool fSuccess = bitdb.Salvage(newFilename, true, salvagedData); + if (salvagedData.empty()) { + LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); + return false; + } + LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); + + std::unique_ptr pdbCopy(new Db(bitdb.dbenv, 0)); + int ret = pdbCopy->open(nullptr, // Txn pointer + filename.c_str(), // Filename + "main", // Logical db name + DB_BTREE, // Database type + DB_CREATE, // Flags + 0); + if (ret > 0) { + LogPrintf("Cannot create database file %s\n", filename); + return false; + } + + DbTxn *ptxn = bitdb.TxnBegin(); + for (CDBEnv::KeyValPair &row : salvagedData) { + if (recoverKVcallback) { + CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); + CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); + std::string strType, strErr; + if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue)) { + continue; + } + } + Dbt datKey(&row.first[0], row.first.size()); + Dbt datValue(&row.second[0], row.second.size()); + int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); + if (ret2 > 0) { + fSuccess = false; + } + } + ptxn->commit(0); + pdbCopy->close(0); + + return fSuccess; +} + +bool CDB::VerifyEnvironment(const std::string &walletFile, + const fs::path &dataDir, std::string &errorStr) { + LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); + LogPrintf("Using wallet %s\n", walletFile); + + // Wallet file must be a plain filename without a directory + if (walletFile != fs::basename(walletFile) + fs::extension(walletFile)) { + errorStr = strprintf(_("Wallet %s resides outside data directory %s"), + walletFile, dataDir.string()); + return false; + } + + if (!bitdb.Open(dataDir)) { + // try moving the database env out of the way + fs::path pathDatabase = dataDir / "database"; + fs::path pathDatabaseBak = + dataDir / strprintf("database.%d.bak", GetTime()); + try { + fs::rename(pathDatabase, pathDatabaseBak); + LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), + pathDatabaseBak.string()); + } catch (const fs::filesystem_error &) { + // failure is ok (well, not really, but it's not worse than what we + // started with) + } + + // try again + if (!bitdb.Open(dataDir)) { + // if it still fails, it probably means we can't even create the + // database env + errorStr = strprintf( + _("Error initializing wallet database environment %s!"), + GetDataDir()); + return false; + } + } + return true; +} + +bool CDB::VerifyDatabaseFile(const std::string &walletFile, + const fs::path &dataDir, std::string &warningStr, + std::string &errorStr, + bool (*recoverFunc)(const std::string &strFile)) { + if (fs::exists(dataDir / walletFile)) { + CDBEnv::VerifyResult r = bitdb.Verify(walletFile, recoverFunc); + if (r == CDBEnv::RECOVER_OK) { + warningStr = strprintf( + _("Warning: Wallet file corrupt, data salvaged!" + " Original %s saved as %s in %s; if" + " your balance or transactions are incorrect you should" + " restore from a backup."), + walletFile, "wallet.{timestamp}.bak", dataDir); + } + if (r == CDBEnv::RECOVER_FAIL) { + errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile); + return false; + } + } + // also return true if files does not exists + return true; +} + /* End of headers, beginning of key/value data */ static const char *HEADER_END = "HEADER=END"; /* End of key/value data */ @@ -381,7 +507,8 @@ bool fSuccess = true; LogPrintf("CDB::Rewrite: Rewriting %s...\n", strFile); std::string strFileRes = strFile + ".rewrite"; - { // surround usage of db with extra {} + { + // surround usage of db with extra {} CDB db(strFile.c_str(), "r"); Db *pdbCopy = new Db(bitdb.dbenv, 0); @@ -514,3 +641,38 @@ } } } + +bool CDB::PeriodicFlush(std::string strFile) { + bool ret = false; + TRY_LOCK(bitdb.cs_db, lockDb); + if (lockDb) { + // Don't do this if any databases are in use + int nRefCount = 0; + std::map::iterator mi = bitdb.mapFileUseCount.begin(); + while (mi != bitdb.mapFileUseCount.end()) { + nRefCount += (*mi).second; + mi++; + } + + if (nRefCount == 0) { + boost::this_thread::interruption_point(); + std::map::iterator mi = + bitdb.mapFileUseCount.find(strFile); + if (mi != bitdb.mapFileUseCount.end()) { + LogPrint(BCLog::DB, "Flushing %s\n", strFile); + int64_t nStart = GetTimeMillis(); + + // Flush wallet file so it's self contained + bitdb.CloseDb(strFile); + bitdb.CheckpointLSN(strFile); + + bitdb.mapFileUseCount.erase(mi++); + LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, + GetTimeMillis() - nStart); + ret = true; + } + } + } + + return ret; +} diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -531,65 +531,34 @@ return true; } - LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); - std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT); - - LogPrintf("Using wallet %s\n", walletFile); uiInterface.InitMessage(_("Verifying wallet...")); + std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT); - // Wallet file must be a plain filename without a directory. - if (walletFile != fs::basename(walletFile) + fs::extension(walletFile)) { - return InitError( - strprintf(_("Wallet %s resides outside data directory %s"), - walletFile, GetDataDir().string())); - } - - if (!bitdb.Open(GetDataDir())) { - // Try moving the database env out of the way. - fs::path pathDatabase = GetDataDir() / "database"; - fs::path pathDatabaseBak = - GetDataDir() / strprintf("database.%d.bak", GetTime()); - try { - fs::rename(pathDatabase, pathDatabaseBak); - LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), - pathDatabaseBak.string()); - } catch (const fs::filesystem_error &) { - // Failure is ok (well, not really, but it's not worse than what we - // started with) - } - - // try again - if (!bitdb.Open(GetDataDir())) { - // If it still fails, it probably means we can't even create the - // database env. - return InitError(strprintf( - _("Error initializing wallet database environment %s!"), - GetDataDir())); - } + std::string strError; + if (!CWalletDB::VerifyEnvironment(walletFile, GetDataDir().string(), + strError)) { + return InitError(strError); } if (GetBoolArg("-salvagewallet", false)) { // Recover readable keypairs: - if (!CWalletDB::Recover(bitdb, walletFile, true)) return false; - } - - if (fs::exists(GetDataDir() / walletFile)) { - CDBEnv::VerifyResult r = bitdb.Verify(walletFile, CWalletDB::Recover); - if (r == CDBEnv::RECOVER_OK) { - InitWarning(strprintf( - _("Warning: Wallet file corrupt, data salvaged!" - " Original %s saved as %s in %s; if" - " your balance or transactions are incorrect you should" - " restore from a backup."), - walletFile, "wallet.{timestamp}.bak", GetDataDir())); - } - - if (r == CDBEnv::RECOVER_FAIL) { - return InitError( - strprintf(_("%s corrupt, salvage failed"), walletFile)); + CWallet dummyWallet; + if (!CWalletDB::Recover(walletFile, (void *)&dummyWallet, + CWalletDB::RecoverKeysOnlyFilter)) { + return false; } } + std::string strWarning; + bool dbV = CWalletDB::VerifyDatabaseFile(walletFile, GetDataDir().string(), + strWarning, strError); + if (!strWarning.empty()) { + InitWarning(strWarning); + } + if (!dbV) { + InitError(strError); + return false; + } return true; } diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -174,9 +174,31 @@ DBErrors ZapWalletTx(std::vector &vWtx); DBErrors ZapSelectTx(std::vector &vHashIn, std::vector &vHashOut); - static bool Recover(CDBEnv &dbenv, const std::string &filename, - bool fOnlyKeys); - static bool Recover(CDBEnv &dbenv, const std::string &filename); + /* Try to (very carefully!) recover wallet database (with a possible key + * type filter) */ + static bool Recover(const std::string &filename, void *callbackDataIn, + bool (*recoverKVcallback)(void *callbackData, + CDataStream ssKey, + CDataStream ssValue)); + /* Recover convenience-function to bypass the key filter callback, called + * when verify fails, recovers everything */ + static bool Recover(const std::string &filename); + /* Recover filter (used as callback), will only let keys (cryptographical + * keys) as KV/key-type pass through */ + static bool RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, + CDataStream ssValue); + /* Function to determine if a certain KV/key-type is a key (cryptographical + * key) type */ + static bool IsKeyType(const std::string &strType); + /* verifies the database environment */ + static bool VerifyEnvironment(const std::string &walletFile, + const fs::path &dataDir, + std::string &errorStr); + /* verifies the database file */ + static bool VerifyDatabaseFile(const std::string &walletFile, + const fs::path &dataDir, + std::string &warningStr, + std::string &errorStr); //! write the hdchain model (external chain child index counter) bool WriteHDChain(const CHDChain &chain); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -547,7 +547,7 @@ return true; } -static bool IsKeyType(std::string strType) { +bool CWalletDB::IsKeyType(const std::string &strType) { return (strType == "key" || strType == "wkey" || strType == "mkey" || strType == "ckey"); } @@ -814,36 +814,9 @@ if (nLastFlushed != CWalletDB::GetUpdateCounter() && GetTime() - nLastWalletUpdate >= 2) { - TRY_LOCK(bitdb.cs_db, lockDb); - if (lockDb) { - // Don't do this if any databases are in use. - int nRefCount = 0; - std::map::iterator mi = - bitdb.mapFileUseCount.begin(); - while (mi != bitdb.mapFileUseCount.end()) { - nRefCount += (*mi).second; - mi++; - } - - if (nRefCount == 0) { - boost::this_thread::interruption_point(); - const std::string &strFile = pwalletMain->strWalletFile; - std::map::iterator _mi = - bitdb.mapFileUseCount.find(strFile); - if (_mi != bitdb.mapFileUseCount.end()) { - LogPrint(BCLog::DB, "Flushing %s\n", strFile); - nLastFlushed = CWalletDB::GetUpdateCounter(); - int64_t nStart = GetTimeMillis(); - - // Flush wallet file so it's self contained. - bitdb.CloseDb(strFile); - bitdb.CheckpointLSN(strFile); - - bitdb.mapFileUseCount.erase(_mi++); - LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, - GetTimeMillis() - nStart); - } - } + const std::string &strFile = pwalletMain->strWalletFile; + if (CDB::PeriodicFlush(strFile)) { + nLastFlushed = CWalletDB::GetUpdateCounter(); } } } @@ -852,84 +825,55 @@ // // Try to (very carefully!) recover wallet file if there is a problem. // -bool CWalletDB::Recover(CDBEnv &dbenv, const std::string &filename, - bool fOnlyKeys) { - // Recovery procedure: - // move wallet file to wallet.timestamp.bak . Call Salvage with - // fAggressive=true to get as much data as possible. Rewrite salvaged data - // to fresh wallet file. Set -rescan so any missing transactions will be - // found. - int64_t now = GetTime(); - std::string newFilename = strprintf("wallet.%d.bak", now); - - int result = dbenv.dbenv->dbrename(nullptr, filename.c_str(), nullptr, - newFilename.c_str(), DB_AUTO_COMMIT); - if (result == 0) { - LogPrintf("Renamed %s to %s\n", filename, newFilename); - } else { - LogPrintf("Failed to rename %s to %s\n", filename, newFilename); +bool CWalletDB::Recover(const std::string &filename, void *callbackDataIn, + bool (*recoverKVcallback)(void *callbackData, + CDataStream ssKey, + CDataStream ssValue)) { + return CDB::Recover(filename, callbackDataIn, recoverKVcallback); +} + +bool CWalletDB::Recover(const std::string &filename) { + // recover without a key filter callback + // results in recovering all record types + return CWalletDB::Recover(filename, nullptr, nullptr); +} + +bool CWalletDB::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, + CDataStream ssValue) { + CWallet *dummyWallet = reinterpret_cast(callbackData); + CWalletScanState dummyWss; + std::string strType, strErr; + bool fReadOK; + { + // Required in LoadKeyMetadata(): + LOCK(dummyWallet->cs_wallet); + fReadOK = ReadKeyValue(dummyWallet, ssKey, ssValue, dummyWss, strType, + strErr); + } + if (!IsKeyType(strType) && strType != "hdchain") { return false; } - - std::vector salvagedData; - bool fSuccess = dbenv.Salvage(newFilename, true, salvagedData); - if (salvagedData.empty()) { - LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); + if (!fReadOK) { + LogPrintf("WARNING: CWalletDB::Recover skipping %s: %s\n", strType, + strErr); return false; } - LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); - - std::unique_ptr pdbCopy(new Db(dbenv.dbenv, 0)); - int ret = pdbCopy->open(nullptr, // Txn pointer - filename.c_str(), // Filename - "main", // Logical db name - DB_BTREE, // Database type - DB_CREATE, // Flags - 0); - if (ret > 0) { - LogPrintf("Cannot create database file %s\n", filename); - return false; - } - CWallet dummyWallet; - CWalletScanState wss; - DbTxn *ptxn = dbenv.TxnBegin(); - for (CDBEnv::KeyValPair &row : salvagedData) { - if (fOnlyKeys) { - CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); - CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); - std::string strType, strErr; - bool fReadOK; - { - // Required in LoadKeyMetadata(): - LOCK(dummyWallet.cs_wallet); - fReadOK = ReadKeyValue(&dummyWallet, ssKey, ssValue, wss, - strType, strErr); - } - if (!IsKeyType(strType) && strType != "hdchain") { - continue; - } - if (!fReadOK) { - LogPrintf("WARNING: CWalletDB::Recover skipping %s: %s\n", - strType, strErr); - continue; - } - } - Dbt datKey(&row.first[0], row.first.size()); - Dbt datValue(&row.second[0], row.second.size()); - int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); - if (ret2 > 0) { - fSuccess = false; - } - } - ptxn->commit(0); - pdbCopy->close(0); + return true; +} - return fSuccess; +bool CWalletDB::VerifyEnvironment(const std::string &walletFile, + const fs::path &dataDir, + std::string &errorStr) { + return CDB::VerifyEnvironment(walletFile, dataDir, errorStr); } -bool CWalletDB::Recover(CDBEnv &dbenv, const std::string &filename) { - return CWalletDB::Recover(dbenv, filename, false); +bool CWalletDB::VerifyDatabaseFile(const std::string &walletFile, + const fs::path &dataDir, + std::string &warningStr, + std::string &errorStr) { + return CDB::VerifyDatabaseFile(walletFile, dataDir, errorStr, warningStr, + CWalletDB::Recover); } bool CWalletDB::WriteDestData(const CTxDestination &address,