diff --git a/doc/release-notes.md b/doc/release-notes.md index 83b9aed41..698e12ed1 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,7 +1,13 @@ # Bitcoin ABC 0.22.4 Release Notes Bitcoin ABC version 0.22.4 is now available from: This release includes the following features and fixes: + +Command-line options +-------------------- + +- The `-debug=db` logging category has been renamed to `-debug=walletdb`, to distinguish it from `coindb`. + `-debug=db` has been deprecated and will be removed in a next release. diff --git a/src/logging.cpp b/src/logging.cpp index 7abf25d85..f0b00f8cc 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -1,352 +1,360 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Copyright (c) 2017-2019 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include bool fLogIPs = DEFAULT_LOGIPS; const char *const DEFAULT_DEBUGLOGFILE = "debug.log"; BCLog::Logger &LogInstance() { /** * NOTE: the logger instance is leaked on exit. This is ugly, but will be * cleaned up by the OS/libc. Defining a logger as a global object doesn't * work since the order of destruction of static/global objects is * undefined. Consider if the logger gets destroyed, and then some later * destructor calls LogPrintf, maybe indirectly, and you get a core dump at * shutdown trying to access the logger. When the shutdown sequence is fully * audited and tested, explicit destruction of these objects can be * implemented by changing this from a raw pointer to a std::unique_ptr. * Since the ~Logger() destructor is never called, the Logger class and all * its subclasses must have implicitly-defined destructors. * * This method of initialization was originally introduced in * ee3374234c60aba2cc4c5cd5cac1c0aefc2d817c. */ static BCLog::Logger *g_logger{new BCLog::Logger()}; return *g_logger; } static int FileWriteStr(const std::string &str, FILE *fp) { return fwrite(str.data(), 1, str.size(), fp); } bool BCLog::Logger::StartLogging() { LockGuard scoped_lock(m_cs); assert(m_buffering); assert(m_fileout == nullptr); if (m_print_to_file) { assert(!m_file_path.empty()); m_fileout = fsbridge::fopen(m_file_path, "a"); if (!m_fileout) { return false; } // Unbuffered. setbuf(m_fileout, nullptr); // Add newlines to the logfile to distinguish this execution from the // last one. FileWriteStr("\n\n\n\n\n", m_fileout); } // Dump buffered messages from before we opened the log. m_buffering = false; while (!m_msgs_before_open.empty()) { const std::string &s = m_msgs_before_open.front(); if (m_print_to_file) { FileWriteStr(s, m_fileout); } if (m_print_to_console) { fwrite(s.data(), 1, s.size(), stdout); } for (const auto &cb : m_print_callbacks) { cb(s); } m_msgs_before_open.pop_front(); } if (m_print_to_console) { fflush(stdout); } return true; } void BCLog::Logger::DisconnectTestLogger() { LockGuard scoped_lock(m_cs); m_buffering = true; if (m_fileout != nullptr) { fclose(m_fileout); } m_fileout = nullptr; m_print_callbacks.clear(); } struct CLogCategoryDesc { BCLog::LogFlags flag; std::string category; }; const CLogCategoryDesc LogCategories[] = { {BCLog::NONE, "0"}, {BCLog::NONE, "none"}, {BCLog::NET, "net"}, {BCLog::TOR, "tor"}, {BCLog::MEMPOOL, "mempool"}, {BCLog::HTTP, "http"}, {BCLog::BENCH, "bench"}, {BCLog::ZMQ, "zmq"}, - {BCLog::DB, "db"}, + {BCLog::WALLETDB, "walletdb"}, {BCLog::RPC, "rpc"}, {BCLog::ESTIMATEFEE, "estimatefee"}, {BCLog::ADDRMAN, "addrman"}, {BCLog::SELECTCOINS, "selectcoins"}, {BCLog::REINDEX, "reindex"}, {BCLog::CMPCTBLOCK, "cmpctblock"}, {BCLog::RAND, "rand"}, {BCLog::PRUNE, "prune"}, {BCLog::PROXY, "proxy"}, {BCLog::MEMPOOLREJ, "mempoolrej"}, {BCLog::LIBEVENT, "libevent"}, {BCLog::COINDB, "coindb"}, {BCLog::QT, "qt"}, {BCLog::LEVELDB, "leveldb"}, {BCLog::VALIDATION, "validation"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; bool GetLogCategory(BCLog::LogFlags &flag, const std::string &str) { if (str == "") { flag = BCLog::ALL; return true; } for (const CLogCategoryDesc &category_desc : LogCategories) { if (category_desc.category == str) { flag = category_desc.flag; return true; } } return false; } std::string ListLogCategories() { std::string ret; int outcount = 0; for (const CLogCategoryDesc &category_desc : LogCategories) { // Omit the special cases. if (category_desc.flag != BCLog::NONE && category_desc.flag != BCLog::ALL) { if (outcount != 0) { ret += ", "; } ret += category_desc.category; outcount++; } } return ret; } std::vector ListActiveLogCategories() { std::vector ret; for (const CLogCategoryDesc &category_desc : LogCategories) { // Omit the special cases. if (category_desc.flag != BCLog::NONE && category_desc.flag != BCLog::ALL) { CLogCategoryActive catActive; catActive.category = category_desc.category; catActive.active = LogAcceptCategory(category_desc.flag); ret.push_back(catActive); } } return ret; } BCLog::Logger::~Logger() { if (m_fileout) { fclose(m_fileout); } } std::string BCLog::Logger::LogTimestampStr(const std::string &str) { std::string strStamped; if (!m_log_timestamps) { return str; } if (m_started_new_line) { int64_t nTimeMicros = GetTimeMicros(); strStamped = FormatISO8601DateTime(nTimeMicros / 1000000); if (m_log_time_micros) { strStamped.pop_back(); strStamped += strprintf(".%06dZ", nTimeMicros % 1000000); } int64_t mocktime = GetMockTime(); if (mocktime) { strStamped += " (mocktime: " + FormatISO8601DateTime(mocktime) + ")"; } strStamped += ' ' + str; } else { strStamped = str; } return strStamped; } namespace BCLog { /** Belts and suspenders: make sure outgoing log messages don't contain * potentially suspicious characters, such as terminal control codes. * * This escapes control characters except newline ('\n') in C syntax. * It escapes instead of removes them to still allow for troubleshooting * issues where they accidentally end up in strings. */ std::string LogEscapeMessage(const std::string &str) { std::string ret; for (char ch_in : str) { uint8_t ch = (uint8_t)ch_in; if ((ch >= 32 || ch == '\n') && ch != '\x7f') { ret += ch_in; } else { ret += strprintf("\\x%02x", ch); } } return ret; } } // namespace BCLog void BCLog::Logger::LogPrintStr(const std::string &str) { LockGuard scoped_lock(m_cs); std::string str_prefixed = LogEscapeMessage(str); if (m_log_threadnames && m_started_new_line) { str_prefixed.insert(0, "[" + util::ThreadGetInternalName() + "] "); } str_prefixed = LogTimestampStr(str_prefixed); m_started_new_line = !str.empty() && str[str.size() - 1] == '\n'; if (m_buffering) { // buffer if we haven't started logging yet m_msgs_before_open.push_back(str_prefixed); return; } if (m_print_to_console) { // Print to console. fwrite(str_prefixed.data(), 1, str_prefixed.size(), stdout); fflush(stdout); } for (const auto &cb : m_print_callbacks) { cb(str_prefixed); } if (m_print_to_file) { assert(m_fileout != nullptr); // Reopen the log file, if requested. if (m_reopen_file) { m_reopen_file = false; FILE *new_fileout = fsbridge::fopen(m_file_path, "a"); if (new_fileout) { // unbuffered. setbuf(m_fileout, nullptr); fclose(m_fileout); m_fileout = new_fileout; } } FileWriteStr(str_prefixed, m_fileout); } } void BCLog::Logger::ShrinkDebugFile() { // Amount of debug.log to save at end when shrinking (must fit in memory) constexpr size_t RECENT_DEBUG_HISTORY_SIZE = 10 * 1000000; assert(!m_file_path.empty()); // Scroll debug.log if it's getting too big. FILE *file = fsbridge::fopen(m_file_path, "r"); // Special files (e.g. device nodes) may not have a size. size_t log_size = 0; try { log_size = fs::file_size(m_file_path); } catch (const fs::filesystem_error &) { } // If debug.log file is more than 10% bigger the RECENT_DEBUG_HISTORY_SIZE // trim it down by saving only the last RECENT_DEBUG_HISTORY_SIZE bytes. if (file && log_size > 11 * (RECENT_DEBUG_HISTORY_SIZE / 10)) { // Restart the file with some of the end. std::vector vch(RECENT_DEBUG_HISTORY_SIZE, 0); if (fseek(file, -((long)vch.size()), SEEK_END)) { LogPrintf("Failed to shrink debug log file: fseek(...) failed\n"); fclose(file); return; } int nBytes = fread(vch.data(), 1, vch.size(), file); fclose(file); file = fsbridge::fopen(m_file_path, "w"); if (file) { fwrite(vch.data(), 1, nBytes, file); fclose(file); } } else if (file != nullptr) { fclose(file); } } void BCLog::Logger::EnableCategory(LogFlags category) { m_categories |= category; } bool BCLog::Logger::EnableCategory(const std::string &str) { BCLog::LogFlags flag; if (!GetLogCategory(flag, str)) { + if (str == "db") { + // DEPRECATION: Added in 0.22.4, should start returning an error in + // a future release + LogPrintf("Warning: logging category 'db' is deprecated, use " + "'walletdb' instead\n"); + EnableCategory(BCLog::WALLETDB); + return true; + } return false; } EnableCategory(flag); return true; } void BCLog::Logger::DisableCategory(LogFlags category) { m_categories &= ~category; } bool BCLog::Logger::DisableCategory(const std::string &str) { BCLog::LogFlags flag; if (!GetLogCategory(flag, str)) { return false; } DisableCategory(flag); return true; } bool BCLog::Logger::WillLogCategory(LogFlags category) const { // ALL is not meant to be used as a logging category, but only as a mask // representing all categories. if (category == BCLog::NONE || category == BCLog::ALL) { LogPrintf("Error trying to log using a category mask instead of an " "explicit category.\n"); return true; } return (m_categories.load(std::memory_order_relaxed) & category) != 0; } bool BCLog::Logger::DefaultShrinkDebugFile() const { return m_categories != BCLog::NONE; } diff --git a/src/logging.h b/src/logging.h index 45f7bfb81..44b58d88f 100644 --- a/src/logging.h +++ b/src/logging.h @@ -1,204 +1,204 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Copyright (c) 2017-2019 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_LOGGING_H #define BITCOIN_LOGGING_H #include #include #include #include #include #include #include #include static const bool DEFAULT_LOGTIMEMICROS = false; static const bool DEFAULT_LOGIPS = false; static const bool DEFAULT_LOGTIMESTAMPS = true; static const bool DEFAULT_LOGTHREADNAMES = false; extern bool fLogIPs; extern const char *const DEFAULT_DEBUGLOGFILE; struct CLogCategoryActive { std::string category; bool active; }; namespace BCLog { enum LogFlags : uint32_t { NONE = 0, NET = (1 << 0), TOR = (1 << 1), MEMPOOL = (1 << 2), HTTP = (1 << 3), BENCH = (1 << 4), ZMQ = (1 << 5), - DB = (1 << 6), + WALLETDB = (1 << 6), RPC = (1 << 7), ESTIMATEFEE = (1 << 8), ADDRMAN = (1 << 9), SELECTCOINS = (1 << 10), REINDEX = (1 << 11), CMPCTBLOCK = (1 << 12), RAND = (1 << 13), PRUNE = (1 << 14), PROXY = (1 << 15), MEMPOOLREJ = (1 << 16), LIBEVENT = (1 << 17), COINDB = (1 << 18), QT = (1 << 19), LEVELDB = (1 << 20), VALIDATION = (1 << 21), ALL = ~uint32_t(0), }; class Logger { private: // Can not use Mutex from sync.h because in debug mode it would cause a // deadlock when a potential deadlock was detected mutable std::mutex m_cs; FILE *m_fileout GUARDED_BY(m_cs) = nullptr; std::list m_msgs_before_open GUARDED_BY(m_cs); //! Buffer messages before logging can be started. bool m_buffering GUARDED_BY(m_cs) = true; /** * m_started_new_line is a state variable that will suppress printing of the * timestamp when multiple calls are made that don't end in a newline. */ std::atomic_bool m_started_new_line{true}; /** * Log categories bitfield. */ std::atomic m_categories{0}; std::string LogTimestampStr(const std::string &str); /** Slots that connect to the print signal */ std::list> m_print_callbacks GUARDED_BY(m_cs){}; public: bool m_print_to_console = false; bool m_print_to_file = false; bool m_log_timestamps = DEFAULT_LOGTIMESTAMPS; bool m_log_time_micros = DEFAULT_LOGTIMEMICROS; bool m_log_threadnames = DEFAULT_LOGTHREADNAMES; fs::path m_file_path; std::atomic m_reopen_file{false}; ~Logger(); /** Send a string to the log output */ void LogPrintStr(const std::string &str); /** Returns whether logs will be written to any output */ bool Enabled() const { LockGuard scoped_lock(m_cs); return m_buffering || m_print_to_console || m_print_to_file || !m_print_callbacks.empty(); } /** Connect a slot to the print signal and return the connection */ std::list>::iterator PushBackCallback(std::function fun) { LockGuard scoped_lock(m_cs); m_print_callbacks.push_back(std::move(fun)); return --m_print_callbacks.end(); } /** Delete a connection */ void DeleteCallback( std::list>::iterator it) { LockGuard scoped_lock(m_cs); m_print_callbacks.erase(it); } /** Start logging (and flush all buffered messages) */ bool StartLogging(); /** Only for testing */ void DisconnectTestLogger(); void ShrinkDebugFile(); uint32_t GetCategoryMask() const { return m_categories.load(); } void EnableCategory(LogFlags category); bool EnableCategory(const std::string &str); void DisableCategory(LogFlags category); bool DisableCategory(const std::string &str); /** Return true if log accepts specified category */ bool WillLogCategory(LogFlags category) const; /** Default for whether ShrinkDebugFile should be run */ bool DefaultShrinkDebugFile() const; }; } // namespace BCLog BCLog::Logger &LogInstance(); /** Return true if log accepts specified category */ static inline bool LogAcceptCategory(BCLog::LogFlags category) { return LogInstance().WillLogCategory(category); } /** Returns a string with the log categories. */ std::string ListLogCategories(); /** Returns a vector of the active log categories. */ std::vector ListActiveLogCategories(); /** Return true if str parses as a log category and set the flag */ bool GetLogCategory(BCLog::LogFlags &flag, const std::string &str); // Be conservative when using LogPrintf/error or other things which // unconditionally log to debug.log! It should not be the case that an inbound // peer can fill up a user's disk with debug.log entries. template static inline void LogPrintf(const char *fmt, const Args &... args) { if (LogInstance().Enabled()) { std::string log_msg; try { log_msg = tfm::format(fmt, args...); } catch (tinyformat::format_error &fmterr) { /** * Original format string will have newline so don't add one here */ log_msg = "Error \"" + std::string(fmterr.what()) + "\" while formatting log message: " + fmt; } LogInstance().LogPrintStr(log_msg); } } // Use a macro instead of a function for conditional logging to prevent // evaluating arguments when logging for the category is not enabled. #define LogPrint(category, ...) \ do { \ if (LogAcceptCategory((category))) { \ LogPrintf(__VA_ARGS__); \ } \ } while (0) /** * These are aliases used to explicitly state that the message should not end * with a newline character. It allows for detecting the missing newlines that * could make the logs hard to read. */ #define LogPrintfToBeContinued LogPrintf #define LogPrintToBeContinued LogPrint #endif // BITCOIN_LOGGING_H diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 219728228..53f64e195 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -1,1008 +1,1006 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include // boost::this_thread::interruption_point() (mingw) #include #ifndef WIN32 #include #endif namespace { //! Make sure database has a unique fileid within the environment. If it //! doesn't, throw an error. BDB caches do not work properly when more than one //! open database has the same fileid (values written to one database may show //! up in reads to other databases). //! //! BerkeleyDB generates unique fileids by default //! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html), //! so bitcoin should never create different databases with the same fileid, but //! this error can be triggered if users manually copy database files. void CheckUniqueFileid(const BerkeleyEnvironment &env, const std::string &filename, Db &db, WalletDatabaseFileId &fileid) { if (env.IsMock()) { return; } int ret = db.get_mpf()->get_fileid(fileid.value); if (ret != 0) { throw std::runtime_error(strprintf( "BerkeleyBatch: Can't open database %s (get_fileid failed with %d)", filename, ret)); } for (const auto &item : env.m_fileids) { if (fileid == item.second && &fileid != &item.second) { throw std::runtime_error(strprintf( "BerkeleyBatch: Can't open database %s (duplicates fileid %s " "from %s)", filename, HexStr(std::begin(item.second.value), std::end(item.second.value)), item.first)); } } } RecursiveMutex cs_db; //! Map from directory name to db environment. std::map> g_dbenvs GUARDED_BY(cs_db); } // namespace bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId &rhs) const { return memcmp(value, &rhs.value, sizeof(value)) == 0; } static void SplitWalletPath(const fs::path &wallet_path, fs::path &env_directory, std::string &database_filename) { if (fs::is_regular_file(wallet_path)) { // Special case for backwards compatibility: if wallet path points to an // existing file, treat it as the path to a BDB data file in a parent // directory that also contains BDB log files. env_directory = wallet_path.parent_path(); database_filename = wallet_path.filename().string(); } else { // Normal case: Interpret wallet path as a directory path containing // data and log files. env_directory = wallet_path; database_filename = "wallet.dat"; } } bool IsWalletLoaded(const fs::path &wallet_path) { fs::path env_directory; std::string database_filename; SplitWalletPath(wallet_path, env_directory, database_filename); LOCK(cs_db); auto env = g_dbenvs.find(env_directory.string()); if (env == g_dbenvs.end()) { return false; } auto database = env->second.lock(); return database && database->IsDatabaseLoaded(database_filename); } fs::path WalletDataFilePath(const fs::path &wallet_path) { fs::path env_directory; std::string database_filename; SplitWalletPath(wallet_path, env_directory, database_filename); return env_directory / database_filename; } /** * @param[in] wallet_path Path to wallet directory. Or (for backwards * compatibility only) a path to a berkeley btree data file inside a wallet * directory. * @param[out] database_filename Filename of berkeley btree data file inside the * wallet directory. * @return A shared pointer to the BerkeleyEnvironment object for the wallet * directory, never empty because ~BerkeleyEnvironment erases the weak pointer * from the g_dbenvs map. * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the * directory path key was not already in the map. */ std::shared_ptr GetWalletEnv(const fs::path &wallet_path, std::string &database_filename) { fs::path env_directory; SplitWalletPath(wallet_path, env_directory, database_filename); LOCK(cs_db); auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr()); if (inserted.second) { auto env = std::make_shared(env_directory.string()); inserted.first->second = env; return env; } return inserted.first->second.lock(); } // // BerkeleyBatch // void BerkeleyEnvironment::Close() { if (!fDbEnvInit) { return; } fDbEnvInit = false; for (auto &db : m_databases) { auto count = mapFileUseCount.find(db.first); assert(count == mapFileUseCount.end() || count->second == 0); BerkeleyDatabase &database = db.second.get(); if (database.m_db) { database.m_db->close(0); database.m_db.reset(); } } FILE *error_file = nullptr; dbenv->get_errfile(&error_file); int ret = dbenv->close(0); if (ret != 0) { LogPrintf("BerkeleyEnvironment::Close: Error %d closing database " "environment: %s\n", ret, DbEnv::strerror(ret)); } if (!fMockDb) { DbEnv(u_int32_t(0)).remove(strPath.c_str(), 0); } if (error_file) { fclose(error_file); } UnlockDirectory(strPath, ".walletlock"); } void BerkeleyEnvironment::Reset() { dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS)); fDbEnvInit = false; fMockDb = false; } BerkeleyEnvironment::BerkeleyEnvironment(const fs::path &dir_path) : strPath(dir_path.string()) { Reset(); } BerkeleyEnvironment::~BerkeleyEnvironment() { LOCK(cs_db); g_dbenvs.erase(strPath); Close(); } bool BerkeleyEnvironment::Open(bool retry) { if (fDbEnvInit) { return true; } fs::path pathIn = strPath; TryCreateDirectories(pathIn); if (!LockDirectory(pathIn, ".walletlock")) { LogPrintf("Cannot obtain a lock on wallet directory %s. Another " "instance of bitcoin may be using it.\n", strPath); return false; } fs::path pathLogDir = pathIn / "database"; TryCreateDirectories(pathLogDir); fs::path pathErrorFile = pathIn / "db.log"; LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string()); unsigned int nEnvFlags = 0; if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) { nEnvFlags |= DB_PRIVATE; } dbenv->set_lg_dir(pathLogDir.string().c_str()); // 1 MiB should be enough for just the wallet dbenv->set_cachesize(0, 0x100000, 1); dbenv->set_lg_bsize(0x10000); dbenv->set_lg_max(1048576); dbenv->set_lk_max_locks(40000); dbenv->set_lk_max_objects(40000); /// debug dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); dbenv->set_flags(DB_AUTO_COMMIT, 1); dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1); dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1); int ret = dbenv->open(strPath.c_str(), DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD | DB_RECOVER | nEnvFlags, S_IRUSR | S_IWUSR); if (ret != 0) { LogPrintf("BerkeleyEnvironment::Open: Error %d opening database " "environment: %s\n", ret, DbEnv::strerror(ret)); int ret2 = dbenv->close(0); if (ret2 != 0) { LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed " "database environment: %s\n", ret2, DbEnv::strerror(ret2)); } Reset(); if (retry) { // try moving the database env out of the way fs::path pathDatabaseBak = pathIn / strprintf("database.%d.bak", GetTime()); try { fs::rename(pathLogDir, pathDatabaseBak); LogPrintf("Moved old %s to %s. Retrying.\n", pathLogDir.string(), pathDatabaseBak.string()); } catch (const fs::filesystem_error &) { // failure is ok (well, not really, but it's not worse than what // we started with) } // try opening it again one more time if (!Open(false /* retry */)) { // if it still fails, it probably means we can't even create the // database env return false; } } else { return false; } } fDbEnvInit = true; fMockDb = false; return true; } //! Construct an in-memory mock Berkeley environment for testing BerkeleyEnvironment::BerkeleyEnvironment() { Reset(); - LogPrint(BCLog::DB, "BerkeleyEnvironment::MakeMock\n"); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n"); dbenv->set_cachesize(1, 0, 1); dbenv->set_lg_bsize(10485760 * 4); dbenv->set_lg_max(10485760); dbenv->set_lk_max_locks(10000); dbenv->set_lk_max_objects(10000); dbenv->set_flags(DB_AUTO_COMMIT, 1); dbenv->log_set_config(DB_LOG_IN_MEMORY, 1); int ret = dbenv->open(nullptr, DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD | DB_PRIVATE, S_IRUSR | S_IWUSR); if (ret > 0) { throw std::runtime_error( strprintf("BerkeleyEnvironment::MakeMock: Error %d opening " "database environment.", ret)); } fDbEnvInit = true; fMockDb = true; } BerkeleyEnvironment::VerifyResult BerkeleyEnvironment::Verify(const std::string &strFile, recoverFunc_type recoverFunc, std::string &out_backup_filename) { LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); Db db(dbenv.get(), 0); int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); if (result == 0) { return VerifyResult::VERIFY_OK; } else if (recoverFunc == nullptr) { return VerifyResult::RECOVER_FAIL; } // Try to recover: bool fRecovered = (*recoverFunc)(fs::path(strPath) / strFile, out_backup_filename); return (fRecovered ? VerifyResult::RECOVER_OK : VerifyResult::RECOVER_FAIL); } BerkeleyBatch::SafeDbt::SafeDbt() { m_dbt.set_flags(DB_DBT_MALLOC); } BerkeleyBatch::SafeDbt::SafeDbt(void *data, size_t size) : m_dbt(data, size) {} BerkeleyBatch::SafeDbt::~SafeDbt() { if (m_dbt.get_data() != nullptr) { // Clear memory, e.g. in case it was a private key memory_cleanse(m_dbt.get_data(), m_dbt.get_size()); // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be // freed by the caller. // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html if (m_dbt.get_flags() & DB_DBT_MALLOC) { free(m_dbt.get_data()); } } } const void *BerkeleyBatch::SafeDbt::get_data() const { return m_dbt.get_data(); } u_int32_t BerkeleyBatch::SafeDbt::get_size() const { return m_dbt.get_size(); } BerkeleyBatch::SafeDbt::operator Dbt *() { return &m_dbt; } bool BerkeleyBatch::Recover(const fs::path &file_path, void *callbackDataIn, bool (*recoverKVcallback)(void *callbackData, CDataStream ssKey, CDataStream ssValue), std::string &newFilename) { std::string filename; std::shared_ptr env = GetWalletEnv(file_path, filename); // Recovery procedure: // Move wallet file to walletfilename.timestamp.bak // Call Salvage with fAggressive=true to get as much data as possible. // Rewrite salvaged data to fresh wallet file. // Set -rescan so any missing transactions will be found. int64_t now = GetTime(); newFilename = strprintf("%s.%d.bak", filename, now); int result = env->dbenv->dbrename(nullptr, filename.c_str(), nullptr, newFilename.c_str(), DB_AUTO_COMMIT); if (result == 0) { LogPrintf("Renamed %s to %s\n", filename, newFilename); } else { LogPrintf("Failed to rename %s to %s\n", filename, newFilename); return false; } std::vector salvagedData; bool fSuccess = env->Salvage(newFilename, true, salvagedData); if (salvagedData.empty()) { LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); return false; } LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); std::unique_ptr pdbCopy = std::make_unique(env->dbenv.get(), 0); int ret = pdbCopy->open(nullptr, // Txn pointer filename.c_str(), // Filename "main", // Logical db name DB_BTREE, // Database type DB_CREATE, // Flags 0); if (ret > 0) { LogPrintf("Cannot create database file %s\n", filename); pdbCopy->close(0); return false; } DbTxn *ptxn = env->TxnBegin(); for (BerkeleyEnvironment::KeyValPair &row : salvagedData) { if (recoverKVcallback) { CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue)) { continue; } } Dbt datKey(&row.first[0], row.first.size()); Dbt datValue(&row.second[0], row.second.size()); int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); if (ret2 > 0) { fSuccess = false; } } ptxn->commit(0); pdbCopy->close(0); return fSuccess; } bool BerkeleyBatch::VerifyEnvironment(const fs::path &file_path, bilingual_str &errorStr) { std::string walletFile; std::shared_ptr env = GetWalletEnv(file_path, walletFile); fs::path walletDir = env->Directory(); LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(nullptr, nullptr, nullptr)); LogPrintf("Using wallet %s\n", file_path.string()); if (!env->Open(true /* retry */)) { errorStr = strprintf( _("Error initializing wallet database environment %s!"), walletDir); return false; } return true; } bool BerkeleyBatch::VerifyDatabaseFile( const fs::path &file_path, std::vector &warnings, bilingual_str &errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc) { std::string walletFile; std::shared_ptr env = GetWalletEnv(file_path, walletFile); fs::path walletDir = env->Directory(); if (fs::exists(walletDir / walletFile)) { std::string backup_filename; BerkeleyEnvironment::VerifyResult r = env->Verify(walletFile, recoverFunc, backup_filename); if (r == BerkeleyEnvironment::VerifyResult::RECOVER_OK) { warnings.push_back( strprintf(_("Warning: Wallet file corrupt, data salvaged!" " Original %s saved as %s in %s; if your balance " "or transactions are incorrect you should restore " "from a backup."), walletFile, backup_filename, walletDir)); } if (r == BerkeleyEnvironment::VerifyResult::RECOVER_FAIL) { errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile); return false; } } // also return true if files does not exists return true; } /* End of headers, beginning of key/value data */ static const char *HEADER_END = "HEADER=END"; /* End of key/value data */ static const char *DATA_END = "DATA=END"; bool BerkeleyEnvironment::Salvage( const std::string &strFile, bool fAggressive, std::vector &vResult) { LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); u_int32_t flags = DB_SALVAGE; if (fAggressive) { flags |= DB_AGGRESSIVE; } std::stringstream strDump; Db db(dbenv.get(), 0); int result = db.verify(strFile.c_str(), nullptr, &strDump, flags); if (result == DB_VERIFY_BAD) { LogPrintf("BerkeleyEnvironment::Salvage: Database salvage found " "errors, all data may not be recoverable.\n"); if (!fAggressive) { LogPrintf("BerkeleyEnvironment::Salvage: Rerun with aggressive " "mode to ignore errors and continue.\n"); return false; } } if (result != 0 && result != DB_VERIFY_BAD) { LogPrintf("BerkeleyEnvironment::Salvage: Database salvage failed with " "result %d.\n", result); return false; } // Format of bdb dump is ascii lines: // header lines... // HEADER=END // hexadecimal key // hexadecimal value // ... repeated // DATA=END std::string strLine; while (!strDump.eof() && strLine != HEADER_END) { // Skip past header getline(strDump, strLine); } std::string keyHex, valueHex; while (!strDump.eof() && keyHex != DATA_END) { getline(strDump, keyHex); if (keyHex != DATA_END) { if (strDump.eof()) { break; } getline(strDump, valueHex); if (valueHex == DATA_END) { LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Number of " "keys in data does not match number of values.\n"); break; } vResult.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex))); } } if (keyHex != DATA_END) { LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Unexpected end of " "file while reading salvage output.\n"); return false; } return (result == 0); } void BerkeleyEnvironment::CheckpointLSN(const std::string &strFile) { dbenv->txn_checkpoint(0, 0, 0); if (fMockDb) { return; } dbenv->lsn_reset(strFile.c_str(), 0); } BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase &database, const char *pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr) { fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); fFlushOnClose = fFlushOnCloseIn; env = database.env.get(); if (database.IsDummy()) { return; } const std::string &strFilename = database.strFile; bool fCreate = strchr(pszMode, 'c') != nullptr; unsigned int nFlags = DB_THREAD; if (fCreate) { nFlags |= DB_CREATE; } { LOCK(cs_db); if (!env->Open(false /* retry */)) { throw std::runtime_error( "BerkeleyBatch: Failed to open database environment."); } pdb = database.m_db.get(); if (pdb == nullptr) { int ret; std::unique_ptr pdb_temp = std::make_unique(env->dbenv.get(), 0); bool fMockDb = env->IsMock(); if (fMockDb) { DbMpoolFile *mpf = pdb_temp->get_mpf(); ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); if (ret != 0) { throw std::runtime_error( strprintf("BerkeleyBatch: Failed to configure for no " "temp file backing for database %s", strFilename)); } } ret = pdb_temp->open( nullptr, // Txn pointer fMockDb ? nullptr : strFilename.c_str(), // Filename fMockDb ? strFilename.c_str() : "main", // Logical db name DB_BTREE, // Database type nFlags, // Flags 0); if (ret != 0) { throw std::runtime_error( strprintf("BerkeleyBatch: Error %d, can't open database %s", ret, strFilename)); } // Call CheckUniqueFileid on the containing BDB environment to // avoid BDB data consistency bugs that happen when different data // files in the same environment have the same fileid. // // Also call CheckUniqueFileid on all the other g_dbenvs to prevent // bitcoin from opening the same data file through another // environment when the file is referenced through equivalent but // not obviously identical symlinked or hard linked or bind mounted // paths. In the future a more relaxed check for equal inode and // device ids could be done instead, which would allow opening // different backup copies of a wallet at the same time. Maybe even // more ideally, an exclusive lock for accessing the database could // be implemented, so no equality checks are needed at all. (Newer // versions of BDB have an set_lk_exclusive method for this // purpose, but the older version we use does not.) for (const auto &dbenv : g_dbenvs) { CheckUniqueFileid(*dbenv.second.lock().get(), strFilename, *pdb_temp, this->env->m_fileids[strFilename]); } pdb = pdb_temp.release(); database.m_db.reset(pdb); if (fCreate && !Exists(std::string("version"))) { bool fTmp = fReadOnly; fReadOnly = false; Write(std::string("version"), CLIENT_VERSION); fReadOnly = fTmp; } } ++env->mapFileUseCount[strFilename]; strFile = strFilename; } } void BerkeleyBatch::Flush() { if (activeTxn) { return; } // Flush database activity from memory pool to disk log unsigned int nMinutes = 0; if (fReadOnly) { nMinutes = 1; } // env is nullptr for dummy databases (i.e. in tests). Don't actually flush // if env is nullptr so we don't segfault if (env) { env->dbenv->txn_checkpoint( nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); } } void BerkeleyDatabase::IncrementUpdateCounter() { ++nUpdateCounter; } void BerkeleyBatch::Close() { if (!pdb) { return; } if (activeTxn) { activeTxn->abort(); } activeTxn = nullptr; pdb = nullptr; if (fFlushOnClose) { Flush(); } { LOCK(cs_db); --env->mapFileUseCount[strFile]; } env->m_db_in_use.notify_all(); } void BerkeleyEnvironment::CloseDb(const std::string &strFile) { LOCK(cs_db); auto it = m_databases.find(strFile); assert(it != m_databases.end()); BerkeleyDatabase &database = it->second.get(); if (database.m_db) { // Close the database handle database.m_db->close(0); database.m_db.reset(); } } void BerkeleyEnvironment::ReloadDbEnv() { // Make sure that no Db's are in use AssertLockNotHeld(cs_db); std::unique_lock lock(cs_db); m_db_in_use.wait(lock, [this]() { for (auto &count : mapFileUseCount) { if (count.second > 0) { return false; } } return true; }); std::vector filenames; for (auto it : m_databases) { filenames.push_back(it.first); } // Close the individual Db's for (const std::string &filename : filenames) { CloseDb(filename); } // Reset the environment // This will flush and close the environment Flush(true); Reset(); Open(true); } bool BerkeleyBatch::Rewrite(BerkeleyDatabase &database, const char *pszSkip) { if (database.IsDummy()) { return true; } BerkeleyEnvironment *env = database.env.get(); const std::string &strFile = database.strFile; while (true) { { LOCK(cs_db); if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) { // Flush log data to the dat file env->CloseDb(strFile); env->CheckpointLSN(strFile); env->mapFileUseCount.erase(strFile); bool fSuccess = true; LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile); std::string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} BerkeleyBatch db(database, "r"); std::unique_ptr pdbCopy = std::make_unique(env->dbenv.get(), 0); int ret = pdbCopy->open(nullptr, // Txn pointer strFileRes.c_str(), // Filename "main", // Logical db name DB_BTREE, // Database type DB_CREATE, // Flags 0); if (ret > 0) { LogPrintf("BerkeleyBatch::Rewrite: Can't create " "database file %s\n", strFileRes); fSuccess = false; } Dbc *pcursor = db.GetCursor(); if (pcursor) { while (fSuccess) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue); if (ret1 == DB_NOTFOUND) { pcursor->close(); break; } if (ret1 != 0) { pcursor->close(); fSuccess = false; break; } if (pszSkip && strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) { continue; } if (strncmp(ssKey.data(), "\x07version", 8) == 0) { // Update version: ssValue.clear(); ssValue << CLIENT_VERSION; } Dbt datKey(ssKey.data(), ssKey.size()); Dbt datValue(ssValue.data(), ssValue.size()); int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE); if (ret2 > 0) { fSuccess = false; } } } if (fSuccess) { db.Close(); env->CloseDb(strFile); if (pdbCopy->close(0)) { fSuccess = false; } } else { pdbCopy->close(0); } } if (fSuccess) { Db dbA(env->dbenv.get(), 0); if (dbA.remove(strFile.c_str(), nullptr, 0)) { fSuccess = false; } Db dbB(env->dbenv.get(), 0); if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0)) { fSuccess = false; } } if (!fSuccess) { LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite " "database file %s\n", strFileRes); } return fSuccess; } } UninterruptibleSleep(std::chrono::milliseconds{100}); } } void BerkeleyEnvironment::Flush(bool fShutdown) { int64_t nStart = GetTimeMillis(); // Flush log data to the actual data file on all files that are not in use - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); if (!fDbEnvInit) { return; } { LOCK(cs_db); std::map::iterator mi = mapFileUseCount.begin(); while (mi != mapFileUseCount.end()) { std::string strFile = (*mi).first; int nRefCount = (*mi).second; LogPrint( - BCLog::DB, + BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); if (nRefCount == 0) { // Move log data to the dat file CloseDb(strFile); - LogPrint(BCLog::DB, + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); dbenv->txn_checkpoint(0, 0, 0); - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s detach\n", - strFile); - if (!fMockDb) { - dbenv->lsn_reset(strFile.c_str(), 0); - } - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s closed\n", - strFile); + LogPrint(BCLog::WALLETDB, + "BerkeleyEnvironment::Flush: %s detach\n", strFile); + if (!fMockDb) dbenv->lsn_reset(strFile.c_str(), 0); + LogPrint(BCLog::WALLETDB, + "BerkeleyEnvironment::Flush: %s closed\n", strFile); mapFileUseCount.erase(mi++); } else { mi++; } } - LogPrint(BCLog::DB, + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); if (fShutdown) { char **listp; if (mapFileUseCount.empty()) { dbenv->log_archive(&listp, DB_ARCH_REMOVE); Close(); if (!fMockDb) { fs::remove_all(fs::path(strPath) / "database"); } } } } } bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase &database) { if (database.IsDummy()) { return true; } bool ret = false; BerkeleyEnvironment *env = database.env.get(); const std::string &strFile = database.strFile; TRY_LOCK(cs_db, lockDb); if (lockDb) { // Don't do this if any databases are in use int nRefCount = 0; std::map::iterator mit = env->mapFileUseCount.begin(); while (mit != env->mapFileUseCount.end()) { nRefCount += (*mit).second; mit++; } if (nRefCount == 0) { boost::this_thread::interruption_point(); std::map::iterator mi = env->mapFileUseCount.find(strFile); if (mi != env->mapFileUseCount.end()) { - LogPrint(BCLog::DB, "Flushing %s\n", strFile); + LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); int64_t nStart = GetTimeMillis(); // Flush wallet file so it's self contained env->CloseDb(strFile); env->CheckpointLSN(strFile); env->mapFileUseCount.erase(mi++); - LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, + LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); ret = true; } } } return ret; } bool BerkeleyDatabase::Rewrite(const char *pszSkip) { return BerkeleyBatch::Rewrite(*this, pszSkip); } bool BerkeleyDatabase::Backup(const std::string &strDest) { if (IsDummy()) { return false; } while (true) { { LOCK(cs_db); if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) { // Flush log data to the dat file env->CloseDb(strFile); env->CheckpointLSN(strFile); env->mapFileUseCount.erase(strFile); // Copy wallet file. fs::path pathSrc = env->Directory() / strFile; fs::path pathDest(strDest); if (fs::is_directory(pathDest)) { pathDest /= strFile; } try { if (fs::equivalent(pathSrc, pathDest)) { LogPrintf("cannot backup to wallet source file %s\n", pathDest.string()); return false; } fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); LogPrintf("copied %s to %s\n", strFile, pathDest.string()); return true; } catch (const fs::filesystem_error &e) { LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), fsbridge::get_filesystem_error_message(e)); return false; } } } UninterruptibleSleep(std::chrono::milliseconds{100}); } } void BerkeleyDatabase::Flush(bool shutdown) { if (!IsDummy()) { env->Flush(shutdown); if (shutdown) { LOCK(cs_db); g_dbenvs.erase(env->Directory().string()); env = nullptr; } else { // TODO: To avoid g_dbenvs.erase erasing the environment prematurely // after the first database shutdown when multiple databases are // open in the same environment, should replace raw database `env` // pointers with shared or weak pointers, or else separate the // database and environment shutdowns so environments can be shut // down after databases. env->m_fileids.erase(strFile); } } } void BerkeleyDatabase::ReloadDbEnv() { if (!IsDummy()) { env->ReloadDbEnv(); } } diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index d449bb0f8..2164a52d0 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -1,836 +1,836 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Copyright (c) 2017-2020 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include #include #include #include namespace DBKeys { const std::string ACENTRY{"acentry"}; const std::string BESTBLOCK_NOMERKLE{"bestblock_nomerkle"}; const std::string BESTBLOCK{"bestblock"}; const std::string CRYPTED_KEY{"ckey"}; const std::string CSCRIPT{"cscript"}; const std::string DEFAULTKEY{"defaultkey"}; const std::string DESTDATA{"destdata"}; const std::string FLAGS{"flags"}; const std::string HDCHAIN{"hdchain"}; const std::string KEYMETA{"keymeta"}; const std::string KEY{"key"}; const std::string MASTER_KEY{"mkey"}; const std::string MINVERSION{"minversion"}; const std::string NAME{"name"}; const std::string OLD_KEY{"wkey"}; const std::string ORDERPOSNEXT{"orderposnext"}; const std::string POOL{"pool"}; const std::string PURPOSE{"purpose"}; const std::string SETTINGS{"settings"}; const std::string TX{"tx"}; const std::string VERSION{"version"}; const std::string WATCHMETA{"watchmeta"}; const std::string WATCHS{"watchs"}; } // namespace DBKeys // // WalletBatch // bool WalletBatch::WriteName(const CTxDestination &address, const std::string &strName) { if (!IsValidDestination(address)) { return false; } return WriteIC( std::make_pair(DBKeys::NAME, EncodeLegacyAddr(address, Params())), strName); } bool WalletBatch::EraseName(const CTxDestination &address) { // This should only be used for sending addresses, never for receiving // addresses, receiving addresses must always have an address book entry if // they're not change return. if (!IsValidDestination(address)) { return false; } return EraseIC( std::make_pair(DBKeys::NAME, EncodeLegacyAddr(address, Params()))); } bool WalletBatch::WritePurpose(const CTxDestination &address, const std::string &strPurpose) { if (!IsValidDestination(address)) { return false; } return WriteIC( std::make_pair(DBKeys::PURPOSE, EncodeLegacyAddr(address, Params())), strPurpose); } bool WalletBatch::ErasePurpose(const CTxDestination &address) { if (!IsValidDestination(address)) { return false; } return EraseIC( std::make_pair(DBKeys::PURPOSE, EncodeLegacyAddr(address, Params()))); } bool WalletBatch::WriteTx(const CWalletTx &wtx) { return WriteIC(std::make_pair(DBKeys::TX, wtx.GetId()), wtx); } bool WalletBatch::EraseTx(uint256 hash) { return EraseIC(std::make_pair(DBKeys::TX, hash)); } bool WalletBatch::WriteKeyMetadata(const CKeyMetadata &meta, const CPubKey &pubkey, const bool overwrite) { return WriteIC(std::make_pair(DBKeys::KEYMETA, pubkey), meta, overwrite); } bool WalletBatch::WriteKey(const CPubKey &vchPubKey, const CPrivKey &vchPrivKey, const CKeyMetadata &keyMeta) { if (!WriteKeyMetadata(keyMeta, vchPubKey, false)) { return false; } // hash pubkey/privkey to accelerate wallet load std::vector vchKey; vchKey.reserve(vchPubKey.size() + vchPrivKey.size()); vchKey.insert(vchKey.end(), vchPubKey.begin(), vchPubKey.end()); vchKey.insert(vchKey.end(), vchPrivKey.begin(), vchPrivKey.end()); return WriteIC( std::make_pair(DBKeys::KEY, vchPubKey), std::make_pair(vchPrivKey, Hash(vchKey.begin(), vchKey.end())), false); } bool WalletBatch::WriteCryptedKey(const CPubKey &vchPubKey, const std::vector &vchCryptedSecret, const CKeyMetadata &keyMeta) { if (!WriteKeyMetadata(keyMeta, vchPubKey, true)) { return false; } if (!WriteIC(std::make_pair(DBKeys::CRYPTED_KEY, vchPubKey), vchCryptedSecret, false)) { return false; } EraseIC(std::make_pair(DBKeys::KEY, vchPubKey)); return true; } bool WalletBatch::WriteMasterKey(unsigned int nID, const CMasterKey &kMasterKey) { return WriteIC(std::make_pair(DBKeys::MASTER_KEY, nID), kMasterKey, true); } bool WalletBatch::WriteCScript(const uint160 &hash, const CScript &redeemScript) { return WriteIC(std::make_pair(DBKeys::CSCRIPT, hash), redeemScript, false); } bool WalletBatch::WriteWatchOnly(const CScript &dest, const CKeyMetadata &keyMeta) { if (!WriteIC(std::make_pair(DBKeys::WATCHMETA, dest), keyMeta)) { return false; } return WriteIC(std::make_pair(DBKeys::WATCHS, dest), '1'); } bool WalletBatch::EraseWatchOnly(const CScript &dest) { if (!EraseIC(std::make_pair(DBKeys::WATCHMETA, dest))) { return false; } return EraseIC(std::make_pair(DBKeys::WATCHS, dest)); } bool WalletBatch::WriteBestBlock(const CBlockLocator &locator) { // Write empty block locator so versions that require a merkle branch // automatically rescan WriteIC(DBKeys::BESTBLOCK, CBlockLocator()); return WriteIC(DBKeys::BESTBLOCK_NOMERKLE, locator); } bool WalletBatch::ReadBestBlock(CBlockLocator &locator) { if (m_batch.Read(DBKeys::BESTBLOCK, locator) && !locator.vHave.empty()) { return true; } return m_batch.Read(DBKeys::BESTBLOCK_NOMERKLE, locator); } bool WalletBatch::WriteOrderPosNext(int64_t nOrderPosNext) { return WriteIC(DBKeys::ORDERPOSNEXT, nOrderPosNext); } bool WalletBatch::ReadPool(int64_t nPool, CKeyPool &keypool) { return m_batch.Read(std::make_pair(DBKeys::POOL, nPool), keypool); } bool WalletBatch::WritePool(int64_t nPool, const CKeyPool &keypool) { return WriteIC(std::make_pair(DBKeys::POOL, nPool), keypool); } bool WalletBatch::ErasePool(int64_t nPool) { return EraseIC(std::make_pair(DBKeys::POOL, nPool)); } bool WalletBatch::WriteMinVersion(int nVersion) { return WriteIC(DBKeys::MINVERSION, nVersion); } class CWalletScanState { public: unsigned int nKeys{0}; unsigned int nCKeys{0}; unsigned int nWatchKeys{0}; unsigned int nKeyMeta{0}; unsigned int m_unknown_records{0}; bool fIsEncrypted{false}; bool fAnyUnordered{false}; std::vector vWalletUpgrade; CWalletScanState() {} }; static bool ReadKeyValue(CWallet *pwallet, CDataStream &ssKey, CDataStream &ssValue, CWalletScanState &wss, std::string &strType, std::string &strErr) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet, pwallet->GetLegacyScriptPubKeyMan()->cs_wallet) { try { // Unserialize // Taking advantage of the fact that pair serialization is just the two // items serialized one after the other. ssKey >> strType; if (strType == DBKeys::NAME) { std::string strAddress; ssKey >> strAddress; ssValue >> pwallet ->mapAddressBook[DecodeDestination( strAddress, pwallet->chainParams)] .name; } else if (strType == DBKeys::PURPOSE) { std::string strAddress; ssKey >> strAddress; ssValue >> pwallet ->mapAddressBook[DecodeDestination( strAddress, pwallet->chainParams)] .purpose; } else if (strType == DBKeys::TX) { TxId txid; ssKey >> txid; CWalletTx wtx(nullptr /* pwallet */, MakeTransactionRef()); ssValue >> wtx; if (wtx.GetId() != txid) { return false; } // Undo serialize changes in 31600 if (31404 <= wtx.fTimeReceivedIsTxTime && wtx.fTimeReceivedIsTxTime <= 31703) { if (!ssValue.empty()) { char fTmp; char fUnused; std::string unused_string; ssValue >> fTmp >> fUnused >> unused_string; strErr = strprintf("LoadWallet() upgrading tx ver=%d %d %s", wtx.fTimeReceivedIsTxTime, fTmp, txid.ToString()); wtx.fTimeReceivedIsTxTime = fTmp; } else { strErr = strprintf("LoadWallet() repairing tx ver=%d %s", wtx.fTimeReceivedIsTxTime, txid.ToString()); wtx.fTimeReceivedIsTxTime = 0; } wss.vWalletUpgrade.push_back(txid); } if (wtx.nOrderPos == -1) { wss.fAnyUnordered = true; } pwallet->LoadToWallet(wtx); } else if (strType == DBKeys::WATCHS) { wss.nWatchKeys++; CScript script; ssKey >> script; char fYes; ssValue >> fYes; if (fYes == '1') { pwallet->GetLegacyScriptPubKeyMan()->LoadWatchOnly(script); } } else if (strType == DBKeys::KEY) { CPubKey vchPubKey; ssKey >> vchPubKey; if (!vchPubKey.IsValid()) { strErr = "Error reading wallet database: CPubKey corrupt"; return false; } CKey key; CPrivKey pkey; uint256 hash; wss.nKeys++; ssValue >> pkey; // Old wallets store keys as DBKeys::KEY [pubkey] => [privkey] ... // which was slow for wallets with lots of keys, because the public // key is re-derived from the private key using EC operations as a // checksum. Newer wallets store keys as DBKeys::KEY [pubkey] => // [privkey][hash(pubkey,privkey)], which is much faster while // remaining backwards-compatible. try { ssValue >> hash; } catch (...) { } bool fSkipCheck = false; if (!hash.IsNull()) { // hash pubkey/privkey to accelerate wallet load std::vector vchKey; vchKey.reserve(vchPubKey.size() + pkey.size()); vchKey.insert(vchKey.end(), vchPubKey.begin(), vchPubKey.end()); vchKey.insert(vchKey.end(), pkey.begin(), pkey.end()); if (Hash(vchKey.begin(), vchKey.end()) != hash) { strErr = "Error reading wallet database: CPubKey/CPrivKey " "corrupt"; return false; } fSkipCheck = true; } if (!key.Load(pkey, vchPubKey, fSkipCheck)) { strErr = "Error reading wallet database: CPrivKey corrupt"; return false; } if (!pwallet->GetLegacyScriptPubKeyMan()->LoadKey(key, vchPubKey)) { strErr = "Error reading wallet database: " "LegacyScriptPubKeyMan::LoadKey failed"; return false; } } else if (strType == DBKeys::MASTER_KEY) { // Master encryption key is loaded into only the wallet and not any // of the ScriptPubKeyMans. unsigned int nID; ssKey >> nID; CMasterKey kMasterKey; ssValue >> kMasterKey; if (pwallet->mapMasterKeys.count(nID) != 0) { strErr = strprintf( "Error reading wallet database: duplicate CMasterKey id %u", nID); return false; } pwallet->mapMasterKeys[nID] = kMasterKey; if (pwallet->nMasterKeyMaxID < nID) { pwallet->nMasterKeyMaxID = nID; } } else if (strType == DBKeys::CRYPTED_KEY) { CPubKey vchPubKey; ssKey >> vchPubKey; if (!vchPubKey.IsValid()) { strErr = "Error reading wallet database: CPubKey corrupt"; return false; } std::vector vchPrivKey; ssValue >> vchPrivKey; wss.nCKeys++; if (!pwallet->GetLegacyScriptPubKeyMan()->LoadCryptedKey( vchPubKey, vchPrivKey)) { strErr = "Error reading wallet database: " "LegacyScriptPubKeyMan::LoadCryptedKey failed"; return false; } wss.fIsEncrypted = true; } else if (strType == DBKeys::KEYMETA) { CPubKey vchPubKey; ssKey >> vchPubKey; CKeyMetadata keyMeta; ssValue >> keyMeta; wss.nKeyMeta++; pwallet->GetLegacyScriptPubKeyMan()->LoadKeyMetadata( vchPubKey.GetID(), keyMeta); } else if (strType == DBKeys::WATCHMETA) { CScript script; ssKey >> script; CKeyMetadata keyMeta; ssValue >> keyMeta; wss.nKeyMeta++; pwallet->GetLegacyScriptPubKeyMan()->LoadScriptMetadata( CScriptID(script), keyMeta); } else if (strType == DBKeys::DEFAULTKEY) { // We don't want or need the default key, but if there is one set, // we want to make sure that it is valid so that we can detect // corruption CPubKey vchPubKey; ssValue >> vchPubKey; if (!vchPubKey.IsValid()) { strErr = "Error reading wallet database: Default Key corrupt"; return false; } } else if (strType == DBKeys::POOL) { int64_t nIndex; ssKey >> nIndex; CKeyPool keypool; ssValue >> keypool; pwallet->GetLegacyScriptPubKeyMan()->LoadKeyPool(nIndex, keypool); } else if (strType == DBKeys::CSCRIPT) { uint160 hash; ssKey >> hash; CScript script; ssValue >> script; if (!pwallet->GetLegacyScriptPubKeyMan()->LoadCScript(script)) { strErr = "Error reading wallet database: " "LegacyScriptPubKeyMan::LoadCScript failed"; return false; } } else if (strType == DBKeys::ORDERPOSNEXT) { ssValue >> pwallet->nOrderPosNext; } else if (strType == DBKeys::DESTDATA) { std::string strAddress, strKey, strValue; ssKey >> strAddress; ssKey >> strKey; ssValue >> strValue; pwallet->LoadDestData( DecodeDestination(strAddress, pwallet->chainParams), strKey, strValue); } else if (strType == DBKeys::HDCHAIN) { CHDChain chain; ssValue >> chain; pwallet->GetLegacyScriptPubKeyMan()->SetHDChain(chain, true); } else if (strType == DBKeys::FLAGS) { uint64_t flags; ssValue >> flags; if (!pwallet->SetWalletFlags(flags, true)) { strErr = "Error reading wallet database: Unknown non-tolerable " "wallet flags found"; return false; } } else if (strType == DBKeys::OLD_KEY) { strErr = "Found unsupported 'wkey' record, try loading with " "version 0.20"; return false; } else if (strType != DBKeys::BESTBLOCK && strType != DBKeys::BESTBLOCK_NOMERKLE && strType != DBKeys::MINVERSION && strType != DBKeys::ACENTRY && strType != DBKeys::VERSION && strType != DBKeys::SETTINGS) { wss.m_unknown_records++; } } catch (const std::exception &e) { if (strErr.empty()) { strErr = e.what(); } return false; } catch (...) { if (strErr.empty()) { strErr = "Caught unknown exception in ReadKeyValue"; } return false; } return true; } bool WalletBatch::IsKeyType(const std::string &strType) { return (strType == DBKeys::KEY || strType == DBKeys::MASTER_KEY || strType == DBKeys::CRYPTED_KEY); } DBErrors WalletBatch::LoadWallet(CWallet *pwallet) { CWalletScanState wss; bool fNoncriticalErrors = false; DBErrors result = DBErrors::LOAD_OK; LOCK(pwallet->cs_wallet); AssertLockHeld(pwallet->GetLegacyScriptPubKeyMan()->cs_wallet); try { int nMinVersion = 0; if (m_batch.Read(DBKeys::MINVERSION, nMinVersion)) { if (nMinVersion > FEATURE_LATEST) { return DBErrors::TOO_NEW; } pwallet->LoadMinVersion(nMinVersion); } // Get cursor Dbc *pcursor = m_batch.GetCursor(); if (!pcursor) { pwallet->WalletLogPrintf("Error getting wallet database cursor\n"); return DBErrors::CORRUPT; } while (true) { // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); int ret = m_batch.ReadAtCursor(pcursor, ssKey, ssValue); if (ret == DB_NOTFOUND) { break; } if (ret != 0) { pwallet->WalletLogPrintf( "Error reading next record from wallet database\n"); return DBErrors::CORRUPT; } // Try to be tolerant of single corrupt records: std::string strType, strErr; if (!ReadKeyValue(pwallet, ssKey, ssValue, wss, strType, strErr)) { // losing keys is considered a catastrophic error, anything else // we assume the user can live with: if (IsKeyType(strType) || strType == DBKeys::DEFAULTKEY) { result = DBErrors::CORRUPT; } else if (strType == DBKeys::FLAGS) { // Reading the wallet flags can only fail if unknown flags // are present. result = DBErrors::TOO_NEW; } else { // Leave other errors alone, if we try to fix them we might // make things worse. But do warn the user there is // something wrong. fNoncriticalErrors = true; if (strType == DBKeys::TX) { // Rescan if there is a bad transaction record: gArgs.SoftSetBoolArg("-rescan", true); } } } if (!strErr.empty()) { pwallet->WalletLogPrintf("%s\n", strErr); } } pcursor->close(); } catch (const boost::thread_interrupted &) { throw; } catch (...) { result = DBErrors::CORRUPT; } if (fNoncriticalErrors && result == DBErrors::LOAD_OK) { result = DBErrors::NONCRITICAL_ERROR; } // Any wallet corruption at all: skip any rewriting or upgrading, we don't // want to make it worse. if (result != DBErrors::LOAD_OK) { return result; } // Last client version to open this wallet, was previously the file version // number int last_client = CLIENT_VERSION; m_batch.Read(DBKeys::VERSION, last_client); int wallet_version = pwallet->GetVersion(); pwallet->WalletLogPrintf("Wallet File Version = %d\n", wallet_version > 0 ? wallet_version : last_client); pwallet->WalletLogPrintf("Keys: %u plaintext, %u encrypted, %u w/ " "metadata, %u total. Unknown wallet records: %u\n", wss.nKeys, wss.nCKeys, wss.nKeyMeta, wss.nKeys + wss.nCKeys, wss.m_unknown_records); // nTimeFirstKey is only reliable if all keys have metadata if ((wss.nKeys + wss.nCKeys + wss.nWatchKeys) != wss.nKeyMeta) { auto spk_man = pwallet->GetLegacyScriptPubKeyMan(); if (spk_man) { spk_man->UpdateTimeFirstKey(1); } } for (const TxId &txid : wss.vWalletUpgrade) { WriteTx(pwallet->mapWallet.at(txid)); } // Rewrite encrypted wallets of versions 0.4.0 and 0.5.0rc: if (wss.fIsEncrypted && (last_client == 40000 || last_client == 50000)) { return DBErrors::NEED_REWRITE; } if (last_client < CLIENT_VERSION) { // Update m_batch.Write(DBKeys::VERSION, CLIENT_VERSION); } if (wss.fAnyUnordered) { result = pwallet->ReorderTransactions(); } // Upgrade all of the wallet keymetadata to have the hd master key id // This operation is not atomic, but if it fails, updated entries are still // backwards compatible with older software try { pwallet->UpgradeKeyMetadata(); } catch (...) { result = DBErrors::CORRUPT; } return result; } DBErrors WalletBatch::FindWalletTx(std::vector &txIds, std::vector &vWtx) { DBErrors result = DBErrors::LOAD_OK; try { int nMinVersion = 0; if (m_batch.Read(DBKeys::MINVERSION, nMinVersion)) { if (nMinVersion > FEATURE_LATEST) { return DBErrors::TOO_NEW; } } // Get cursor Dbc *pcursor = m_batch.GetCursor(); if (!pcursor) { LogPrintf("Error getting wallet database cursor\n"); return DBErrors::CORRUPT; } while (true) { // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); int ret = m_batch.ReadAtCursor(pcursor, ssKey, ssValue); if (ret == DB_NOTFOUND) { break; } if (ret != 0) { LogPrintf("Error reading next record from wallet database\n"); return DBErrors::CORRUPT; } std::string strType; ssKey >> strType; if (strType == DBKeys::TX) { TxId txid; ssKey >> txid; CWalletTx wtx(nullptr /* pwallet */, MakeTransactionRef()); ssValue >> wtx; txIds.push_back(txid); vWtx.push_back(wtx); } } pcursor->close(); } catch (const boost::thread_interrupted &) { throw; } catch (...) { result = DBErrors::CORRUPT; } return result; } DBErrors WalletBatch::ZapSelectTx(std::vector &txIdsIn, std::vector &txIdsOut) { // Build list of wallet TXs and hashes. std::vector txIds; std::vector vWtx; DBErrors err = FindWalletTx(txIds, vWtx); if (err != DBErrors::LOAD_OK) { return err; } std::sort(txIds.begin(), txIds.end()); std::sort(txIdsIn.begin(), txIdsIn.end()); // Erase each matching wallet TX. bool delerror = false; std::vector::iterator it = txIdsIn.begin(); for (const TxId &txid : txIds) { while (it < txIdsIn.end() && (*it) < txid) { it++; } if (it == txIdsIn.end()) { break; } if ((*it) == txid) { if (!EraseTx(txid)) { - LogPrint(BCLog::DB, + LogPrint(BCLog::WALLETDB, "Transaction was found for deletion but returned " "database error: %s\n", txid.GetHex()); delerror = true; } txIdsOut.push_back(txid); } } if (delerror) { return DBErrors::CORRUPT; } return DBErrors::LOAD_OK; } DBErrors WalletBatch::ZapWalletTx(std::vector &vWtx) { // Build list of wallet TXs. std::vector txIds; DBErrors err = FindWalletTx(txIds, vWtx); if (err != DBErrors::LOAD_OK) { return err; } // Erase each wallet TX. for (const TxId &txid : txIds) { if (!EraseTx(txid)) { return DBErrors::CORRUPT; } } return DBErrors::LOAD_OK; } void MaybeCompactWalletDB() { static std::atomic fOneThread; if (fOneThread.exchange(true)) { return; } if (!gArgs.GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) { return; } for (const std::shared_ptr &pwallet : GetWallets()) { WalletDatabase &dbh = pwallet->GetDBHandle(); unsigned int nUpdateCounter = dbh.nUpdateCounter; if (dbh.nLastSeen != nUpdateCounter) { dbh.nLastSeen = nUpdateCounter; dbh.nLastWalletUpdate = GetTime(); } if (dbh.nLastFlushed != nUpdateCounter && GetTime() - dbh.nLastWalletUpdate >= 2) { if (BerkeleyBatch::PeriodicFlush(dbh)) { dbh.nLastFlushed = nUpdateCounter; } } } fOneThread = false; } // // Try to (very carefully!) recover wallet file if there is a problem. // bool WalletBatch::Recover(const fs::path &wallet_path, void *callbackDataIn, bool (*recoverKVcallback)(void *callbackData, CDataStream ssKey, CDataStream ssValue), std::string &out_backup_filename) { return BerkeleyBatch::Recover(wallet_path, callbackDataIn, recoverKVcallback, out_backup_filename); } bool WalletBatch::Recover(const fs::path &wallet_path, std::string &out_backup_filename) { // recover without a key filter callback // results in recovering all record types return WalletBatch::Recover(wallet_path, nullptr, nullptr, out_backup_filename); } bool WalletBatch::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDataStream ssValue) { CWallet *dummyWallet = reinterpret_cast(callbackData); CWalletScanState dummyWss; std::string strType, strErr; bool fReadOK; { // Required in LoadKeyMetadata(): LOCK(dummyWallet->cs_wallet); AssertLockHeld(dummyWallet->GetLegacyScriptPubKeyMan()->cs_wallet); fReadOK = ReadKeyValue(dummyWallet, ssKey, ssValue, dummyWss, strType, strErr); } if (!IsKeyType(strType) && strType != DBKeys::HDCHAIN) { return false; } if (!fReadOK) { LogPrintf("WARNING: WalletBatch::Recover skipping %s: %s\n", strType, strErr); return false; } return true; } bool WalletBatch::VerifyEnvironment(const fs::path &wallet_path, bilingual_str &errorStr) { return BerkeleyBatch::VerifyEnvironment(wallet_path, errorStr); } bool WalletBatch::VerifyDatabaseFile(const fs::path &wallet_path, std::vector &warnings, bilingual_str &errorStr) { return BerkeleyBatch::VerifyDatabaseFile(wallet_path, warnings, errorStr, WalletBatch::Recover); } bool WalletBatch::WriteDestData(const CTxDestination &address, const std::string &key, const std::string &value) { if (!IsValidDestination(address)) { return false; } return WriteIC( std::make_pair( DBKeys::DESTDATA, std::make_pair(EncodeLegacyAddr(address, Params()), key)), value); } bool WalletBatch::EraseDestData(const CTxDestination &address, const std::string &key) { if (!IsValidDestination(address)) { return false; } return EraseIC(std::make_pair( DBKeys::DESTDATA, std::make_pair(EncodeLegacyAddr(address, Params()), key))); } bool WalletBatch::WriteHDChain(const CHDChain &chain) { return WriteIC(DBKeys::HDCHAIN, chain); } bool WalletBatch::WriteWalletFlags(const uint64_t flags) { return WriteIC(DBKeys::FLAGS, flags); } bool WalletBatch::TxnBegin() { return m_batch.TxnBegin(); } bool WalletBatch::TxnCommit() { return m_batch.TxnCommit(); } bool WalletBatch::TxnAbort() { return m_batch.TxnAbort(); }