diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -23,6 +23,7 @@ #include #ifdef ENABLE_WALLET +#include #include #include diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -12,6 +12,7 @@ add_library(wallet ../interfaces/wallet.cpp + bdb.cpp coincontrol.cpp coinselection.cpp context.cpp diff --git a/src/wallet/db.h b/src/wallet/bdb.h copy from src/wallet/db.h copy to src/wallet/bdb.h --- a/src/wallet/db.h +++ b/src/wallet/bdb.h @@ -1,16 +1,17 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_WALLET_DB_H -#define BITCOIN_WALLET_DB_H +#ifndef BITCOIN_WALLET_BDB_H +#define BITCOIN_WALLET_BDB_H #include #include #include #include #include +#include #include @@ -75,25 +76,20 @@ DbTxn *TxnBegin(int flags = DB_TXN_WRITE_NOSYNC) { DbTxn *ptxn = nullptr; int ret = dbenv->txn_begin(nullptr, &ptxn, flags); - if (!ptxn || ret != 0) return nullptr; + if (!ptxn || ret != 0) { + return nullptr; + } return ptxn; } }; -/** Return whether a wallet database is currently loaded. */ -bool IsBDBWalletLoaded(const fs::path &wallet_path); - -/** - * Given a wallet directory path or legacy file path, return path to main data - * file in the wallet database. */ -fs::path WalletDataFilePath(const fs::path &wallet_path); -void SplitWalletPath(const fs::path &wallet_path, fs::path &env_directory, - std::string &database_filename); - /** Get BerkeleyEnvironment and database filename given a wallet path. */ std::shared_ptr GetWalletEnv(const fs::path &wallet_path, std::string &database_filename); +/** Return whether a BDB wallet database is currently loaded. */ +bool IsBDBWalletLoaded(const fs::path &wallet_path); + /** * An instance of this class represents one database. * For BerkeleyDB this is just a (env, strFile) tuple. @@ -242,8 +238,10 @@ void Flush(); void Close(); - /* flush the wallet passively (TRY_LOCK) - ideal to be called periodically */ + /* + * flush the wallet passively (TRY_LOCK)i + * ideal to be called periodically + */ static bool PeriodicFlush(BerkeleyDatabase &database); /* verifies the database environment */ static bool VerifyEnvironment(const fs::path &file_path, @@ -355,4 +353,4 @@ const char *pszSkip = nullptr); }; -#endif // BITCOIN_WALLET_DB_H +#endif // BITCOIN_WALLET_BDB_H diff --git a/src/wallet/db.cpp b/src/wallet/bdb.cpp copy from src/wallet/db.cpp copy to src/wallet/bdb.cpp --- a/src/wallet/db.cpp +++ b/src/wallet/bdb.cpp @@ -1,8 +1,9 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include #include @@ -63,22 +64,6 @@ return memcmp(value, &rhs.value, sizeof(value)) == 0; } -void SplitWalletPath(const fs::path &wallet_path, fs::path &env_directory, - std::string &database_filename) { - if (fs::is_regular_file(wallet_path)) { - // Special case for backwards compatibility: if wallet path points to an - // existing file, treat it as the path to a BDB data file in a parent - // directory that also contains BDB log files. - env_directory = wallet_path.parent_path(); - database_filename = wallet_path.filename().string(); - } else { - // Normal case: Interpret wallet path as a directory path containing - // data and log files. - env_directory = wallet_path; - database_filename = "wallet.dat"; - } -} - bool IsBDBWalletLoaded(const fs::path &wallet_path) { fs::path env_directory; std::string database_filename; @@ -94,13 +79,6 @@ return database && database->IsDatabaseLoaded(database_filename); } -fs::path WalletDataFilePath(const fs::path &wallet_path) { - fs::path env_directory; - std::string database_filename; - SplitWalletPath(wallet_path, env_directory, database_filename); - return env_directory / database_filename; -} - /** * @param[in] wallet_path Path to wallet directory. Or (for backwards * compatibility only) a path to a berkeley btree data file inside a wallet diff --git a/src/wallet/db.h b/src/wallet/db.h --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -6,82 +6,9 @@ #ifndef BITCOIN_WALLET_DB_H #define BITCOIN_WALLET_DB_H -#include #include -#include -#include -#include -#include - -#include -#include -#include #include -#include -#include - -struct bilingual_str; - -static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100; -static const bool DEFAULT_WALLET_PRIVDB = true; - -struct WalletDatabaseFileId { - u_int8_t value[DB_FILE_ID_LEN]; - bool operator==(const WalletDatabaseFileId &rhs) const; -}; - -class BerkeleyDatabase; - -class BerkeleyEnvironment { -private: - bool fDbEnvInit; - bool fMockDb; - // Don't change into fs::path, as that can result in - // shutdown problems/crashes caused by a static initialized internal - // pointer. - std::string strPath; - -public: - std::unique_ptr dbenv; - std::map mapFileUseCount; - std::map> m_databases; - std::unordered_map m_fileids; - std::condition_variable_any m_db_in_use; - - BerkeleyEnvironment(const fs::path &env_directory); - BerkeleyEnvironment(); - ~BerkeleyEnvironment(); - void Reset(); - - void MakeMock(); - bool IsMock() const { return fMockDb; } - bool IsInitialized() const { return fDbEnvInit; } - bool IsDatabaseLoaded(const std::string &db_filename) const { - return m_databases.find(db_filename) != m_databases.end(); - } - fs::path Directory() const { return strPath; } - - bool Verify(const std::string &strFile); - - bool Open(bool retry); - void Close(); - void Flush(bool fShutdown); - void CheckpointLSN(const std::string &strFile); - - void CloseDb(const std::string &strFile); - void ReloadDbEnv(); - - DbTxn *TxnBegin(int flags = DB_TXN_WRITE_NOSYNC) { - DbTxn *ptxn = nullptr; - int ret = dbenv->txn_begin(nullptr, &ptxn, flags); - if (!ptxn || ret != 0) return nullptr; - return ptxn; - } -}; - -/** Return whether a wallet database is currently loaded. */ -bool IsBDBWalletLoaded(const fs::path &wallet_path); /** * Given a wallet directory path or legacy file path, return path to main data @@ -90,269 +17,4 @@ void SplitWalletPath(const fs::path &wallet_path, fs::path &env_directory, std::string &database_filename); -/** Get BerkeleyEnvironment and database filename given a wallet path. */ -std::shared_ptr -GetWalletEnv(const fs::path &wallet_path, std::string &database_filename); - -/** - * An instance of this class represents one database. - * For BerkeleyDB this is just a (env, strFile) tuple. - */ -class BerkeleyDatabase { - friend class BerkeleyBatch; - -public: - /** Create dummy DB handle */ - BerkeleyDatabase() - : nUpdateCounter(0), nLastSeen(0), nLastFlushed(0), - nLastWalletUpdate(0), env(nullptr) {} - - /** Create DB handle to real database */ - BerkeleyDatabase(std::shared_ptr envIn, - std::string filename) - : nUpdateCounter(0), nLastSeen(0), nLastFlushed(0), - nLastWalletUpdate(0), env(std::move(envIn)), - strFile(std::move(filename)) { - auto inserted = - this->env->m_databases.emplace(strFile, std::ref(*this)); - assert(inserted.second); - } - - ~BerkeleyDatabase() { - if (env) { - size_t erased = env->m_databases.erase(strFile); - assert(erased == 1); - } - } - - /** Return object for accessing database at specified path. */ - static std::unique_ptr Create(const fs::path &path) { - std::string filename; - return std::make_unique(GetWalletEnv(path, filename), - std::move(filename)); - } - - /** - * Return object for accessing dummy database with no read/write - * capabilities. - */ - static std::unique_ptr CreateDummy() { - return std::make_unique(); - } - - /** - * Return object for accessing temporary in-memory database. - */ - static std::unique_ptr CreateMock() { - return std::make_unique( - std::make_shared(), ""); - } - - /** - * Rewrite the entire database on disk, with the exception of key pszSkip if - * non-zero - */ - bool Rewrite(const char *pszSkip = nullptr); - - /** - * Back up the entire database to a file. - */ - bool Backup(const std::string &strDest) const; - - /** - * Make sure all changes are flushed to disk. - */ - void Flush(bool shutdown); - - void IncrementUpdateCounter(); - - void ReloadDbEnv(); - - std::atomic nUpdateCounter; - unsigned int nLastSeen; - unsigned int nLastFlushed; - int64_t nLastWalletUpdate; - - /** - * Pointer to shared database environment. - * - * Normally there is only one BerkeleyDatabase object per - * BerkeleyEnvivonment, but in the special, backwards compatible case where - * multiple wallet BDB data files are loaded from the same directory, this - * will point to a shared instance that gets freed when the last data file - * is closed. - */ - std::shared_ptr env; - - /** - * Database pointer. This is initialized lazily and reset during flushes, - * so it can be null. - */ - std::unique_ptr m_db; - -private: - std::string strFile; - - /** - * Return whether this database handle is a dummy for testing. - * Only to be used at a low level, application should ideally not care - * about this. - */ - bool IsDummy() const { return env == nullptr; } -}; - -/** RAII class that provides access to a Berkeley database */ -class BerkeleyBatch { - /** RAII class that automatically cleanses its data on destruction */ - class SafeDbt final { - Dbt m_dbt; - - public: - // construct Dbt with internally-managed data - SafeDbt(); - // construct Dbt with provided data - SafeDbt(void *data, size_t size); - ~SafeDbt(); - - // delegate to Dbt - const void *get_data() const; - u_int32_t get_size() const; - - // conversion operator to access the underlying Dbt - operator Dbt *(); - }; - -protected: - Db *pdb; - std::string strFile; - DbTxn *activeTxn; - bool fReadOnly; - bool fFlushOnClose; - BerkeleyEnvironment *env; - -public: - explicit BerkeleyBatch(BerkeleyDatabase &database, - const char *pszMode = "r+", - bool fFlushOnCloseIn = true); - ~BerkeleyBatch() { Close(); } - - BerkeleyBatch(const BerkeleyBatch &) = delete; - BerkeleyBatch &operator=(const BerkeleyBatch &) = delete; - - void Flush(); - void Close(); - - /* flush the wallet passively (TRY_LOCK) - ideal to be called periodically */ - static bool PeriodicFlush(BerkeleyDatabase &database); - /* verifies the database environment */ - static bool VerifyEnvironment(const fs::path &file_path, - bilingual_str &errorStr); - /* verifies the database file */ - static bool VerifyDatabaseFile(const fs::path &file_path, - bilingual_str &errorStr); - - template bool Read(const K &key, T &value) { - if (!pdb) { - return false; - } - - // Key - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - - // Read - SafeDbt datValue; - int ret = pdb->get(activeTxn, datKey, datValue, 0); - bool success = false; - if (datValue.get_data() != nullptr) { - // Unserialize value - try { - CDataStream ssValue((char *)datValue.get_data(), - (char *)datValue.get_data() + - datValue.get_size(), - SER_DISK, CLIENT_VERSION); - ssValue >> value; - success = true; - } catch (const std::exception &) { - // In this case success remains 'false' - } - } - return ret == 0 && success; - } - - template - bool Write(const K &key, const T &value, bool fOverwrite = true) { - if (!pdb) { - return true; - } - if (fReadOnly) { - assert(!"Write called on database in read-only mode"); - } - - // Key - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - - // Value - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - ssValue.reserve(10000); - ssValue << value; - SafeDbt datValue(ssValue.data(), ssValue.size()); - - // Write - int ret = pdb->put(activeTxn, datKey, datValue, - (fOverwrite ? 0 : DB_NOOVERWRITE)); - return (ret == 0); - } - - template bool Erase(const K &key) { - if (!pdb) { - return false; - } - if (fReadOnly) { - assert(!"Erase called on database in read-only mode"); - } - - // Key - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - - // Erase - int ret = pdb->del(activeTxn, datKey, 0); - return (ret == 0 || ret == DB_NOTFOUND); - } - - template bool Exists(const K &key) { - if (!pdb) { - return false; - } - - // Key - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - - // Exists - int ret = pdb->exists(activeTxn, datKey, 0); - return (ret == 0); - } - - Dbc *GetCursor(); - int ReadAtCursor(Dbc *pcursor, CDataStream &ssKey, CDataStream &ssValue); - bool TxnBegin(); - bool TxnCommit(); - bool TxnAbort(); - - static bool Rewrite(BerkeleyDatabase &database, - const char *pszSkip = nullptr); -}; - #endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -3,65 +3,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include -#include -#include - -#include // boost::this_thread::interruption_point() (mingw) - -#include -#ifndef WIN32 -#include -#endif - -namespace { -//! Make sure database has a unique fileid within the environment. If it -//! doesn't, throw an error. BDB caches do not work properly when more than one -//! open database has the same fileid (values written to one database may show -//! up in reads to other databases). -//! -//! BerkeleyDB generates unique fileids by default -//! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html), -//! so bitcoin should never create different databases with the same fileid, but -//! this error can be triggered if users manually copy database files. -void CheckUniqueFileid(const BerkeleyEnvironment &env, - const std::string &filename, Db &db, - WalletDatabaseFileId &fileid) { - if (env.IsMock()) { - return; - } - - int ret = db.get_mpf()->get_fileid(fileid.value); - if (ret != 0) { - throw std::runtime_error(strprintf( - "BerkeleyBatch: Can't open database %s (get_fileid failed with %d)", - filename, ret)); - } - - for (const auto &item : env.m_fileids) { - if (fileid == item.second && &fileid != &item.second) { - throw std::runtime_error(strprintf( - "BerkeleyBatch: Can't open database %s (duplicates fileid %s " - "from %s)", - filename, - HexStr(std::begin(item.second.value), - std::end(item.second.value)), - item.first)); - } - } -} - -RecursiveMutex cs_db; - -//! Map from directory name to db environment. -std::map> - g_dbenvs GUARDED_BY(cs_db); -} // namespace - -bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId &rhs) const { - return memcmp(value, &rhs.value, sizeof(value)) == 0; -} +#include void SplitWalletPath(const fs::path &wallet_path, fs::path &env_directory, std::string &database_filename) { @@ -79,826 +24,9 @@ } } -bool IsBDBWalletLoaded(const fs::path &wallet_path) { - fs::path env_directory; - std::string database_filename; - SplitWalletPath(wallet_path, env_directory, database_filename); - - LOCK(cs_db); - auto env = g_dbenvs.find(env_directory.string()); - if (env == g_dbenvs.end()) { - return false; - } - - auto database = env->second.lock(); - return database && database->IsDatabaseLoaded(database_filename); -} - fs::path WalletDataFilePath(const fs::path &wallet_path) { fs::path env_directory; std::string database_filename; SplitWalletPath(wallet_path, env_directory, database_filename); return env_directory / database_filename; } - -/** - * @param[in] wallet_path Path to wallet directory. Or (for backwards - * compatibility only) a path to a berkeley btree data file inside a wallet - * directory. - * @param[out] database_filename Filename of berkeley btree data file inside the - * wallet directory. - * @return A shared pointer to the BerkeleyEnvironment object for the wallet - * directory, never empty because ~BerkeleyEnvironment erases the weak pointer - * from the g_dbenvs map. - * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the - * directory path key was not already in the map. - */ -std::shared_ptr -GetWalletEnv(const fs::path &wallet_path, std::string &database_filename) { - fs::path env_directory; - SplitWalletPath(wallet_path, env_directory, database_filename); - LOCK(cs_db); - auto inserted = g_dbenvs.emplace(env_directory.string(), - std::weak_ptr()); - if (inserted.second) { - auto env = - std::make_shared(env_directory.string()); - inserted.first->second = env; - return env; - } - return inserted.first->second.lock(); -} - -// -// BerkeleyBatch -// - -void BerkeleyEnvironment::Close() { - if (!fDbEnvInit) { - return; - } - - fDbEnvInit = false; - - for (auto &db : m_databases) { - auto count = mapFileUseCount.find(db.first); - assert(count == mapFileUseCount.end() || count->second == 0); - BerkeleyDatabase &database = db.second.get(); - if (database.m_db) { - database.m_db->close(0); - database.m_db.reset(); - } - } - - FILE *error_file = nullptr; - dbenv->get_errfile(&error_file); - - int ret = dbenv->close(0); - if (ret != 0) { - LogPrintf("BerkeleyEnvironment::Close: Error %d closing database " - "environment: %s\n", - ret, DbEnv::strerror(ret)); - } - if (!fMockDb) { - DbEnv(u_int32_t(0)).remove(strPath.c_str(), 0); - } - - if (error_file) { - fclose(error_file); - } - - UnlockDirectory(strPath, ".walletlock"); -} - -void BerkeleyEnvironment::Reset() { - dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS)); - fDbEnvInit = false; - fMockDb = false; -} - -BerkeleyEnvironment::BerkeleyEnvironment(const fs::path &dir_path) - : strPath(dir_path.string()) { - Reset(); -} - -BerkeleyEnvironment::~BerkeleyEnvironment() { - LOCK(cs_db); - g_dbenvs.erase(strPath); - Close(); -} - -bool BerkeleyEnvironment::Open(bool retry) { - if (fDbEnvInit) { - return true; - } - - fs::path pathIn = strPath; - TryCreateDirectories(pathIn); - if (!LockDirectory(pathIn, ".walletlock")) { - LogPrintf("Cannot obtain a lock on wallet directory %s. Another " - "instance of bitcoin may be using it.\n", - strPath); - return false; - } - - fs::path pathLogDir = pathIn / "database"; - TryCreateDirectories(pathLogDir); - fs::path pathErrorFile = pathIn / "db.log"; - LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", - pathLogDir.string(), pathErrorFile.string()); - - unsigned int nEnvFlags = 0; - if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) { - nEnvFlags |= DB_PRIVATE; - } - - dbenv->set_lg_dir(pathLogDir.string().c_str()); - // 1 MiB should be enough for just the wallet - dbenv->set_cachesize(0, 0x100000, 1); - dbenv->set_lg_bsize(0x10000); - dbenv->set_lg_max(1048576); - dbenv->set_lk_max_locks(40000); - dbenv->set_lk_max_objects(40000); - /// debug - dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); - dbenv->set_flags(DB_AUTO_COMMIT, 1); - dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1); - dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1); - int ret = - dbenv->open(strPath.c_str(), - DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | - DB_INIT_TXN | DB_THREAD | DB_RECOVER | nEnvFlags, - S_IRUSR | S_IWUSR); - if (ret != 0) { - LogPrintf("BerkeleyEnvironment::Open: Error %d opening database " - "environment: %s\n", - ret, DbEnv::strerror(ret)); - int ret2 = dbenv->close(0); - if (ret2 != 0) { - LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed " - "database environment: %s\n", - ret2, DbEnv::strerror(ret2)); - } - Reset(); - if (retry) { - // try moving the database env out of the way - fs::path pathDatabaseBak = - pathIn / strprintf("database.%d.bak", GetTime()); - try { - fs::rename(pathLogDir, pathDatabaseBak); - LogPrintf("Moved old %s to %s. Retrying.\n", - pathLogDir.string(), pathDatabaseBak.string()); - } catch (const fs::filesystem_error &) { - // failure is ok (well, not really, but it's not worse than what - // we started with) - } - // try opening it again one more time - if (!Open(false /* retry */)) { - // if it still fails, it probably means we can't even create the - // database env - return false; - } - } else { - return false; - } - } - - fDbEnvInit = true; - fMockDb = false; - return true; -} - -//! Construct an in-memory mock Berkeley environment for testing -BerkeleyEnvironment::BerkeleyEnvironment() { - Reset(); - - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n"); - - dbenv->set_cachesize(1, 0, 1); - dbenv->set_lg_bsize(10485760 * 4); - dbenv->set_lg_max(10485760); - dbenv->set_lk_max_locks(10000); - dbenv->set_lk_max_objects(10000); - dbenv->set_flags(DB_AUTO_COMMIT, 1); - dbenv->log_set_config(DB_LOG_IN_MEMORY, 1); - int ret = - dbenv->open(nullptr, - DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | - DB_INIT_TXN | DB_THREAD | DB_PRIVATE, - S_IRUSR | S_IWUSR); - if (ret > 0) { - throw std::runtime_error( - strprintf("BerkeleyEnvironment::MakeMock: Error %d opening " - "database environment.", - ret)); - } - - fDbEnvInit = true; - fMockDb = true; -} - -bool BerkeleyEnvironment::Verify(const std::string &strFile) { - LOCK(cs_db); - assert(mapFileUseCount.count(strFile) == 0); - - Db db(dbenv.get(), 0); - int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); - return result == 0; -} - -BerkeleyBatch::SafeDbt::SafeDbt() { - m_dbt.set_flags(DB_DBT_MALLOC); -} - -BerkeleyBatch::SafeDbt::SafeDbt(void *data, size_t size) : m_dbt(data, size) {} - -BerkeleyBatch::SafeDbt::~SafeDbt() { - if (m_dbt.get_data() != nullptr) { - // Clear memory, e.g. in case it was a private key - memory_cleanse(m_dbt.get_data(), m_dbt.get_size()); - // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be - // freed by the caller. - // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html - if (m_dbt.get_flags() & DB_DBT_MALLOC) { - free(m_dbt.get_data()); - } - } -} - -const void *BerkeleyBatch::SafeDbt::get_data() const { - return m_dbt.get_data(); -} - -u_int32_t BerkeleyBatch::SafeDbt::get_size() const { - return m_dbt.get_size(); -} - -BerkeleyBatch::SafeDbt::operator Dbt *() { - return &m_dbt; -} - -bool BerkeleyBatch::VerifyEnvironment(const fs::path &file_path, - bilingual_str &errorStr) { - std::string walletFile; - std::shared_ptr env = - GetWalletEnv(file_path, walletFile); - fs::path walletDir = env->Directory(); - - LogPrintf("Using BerkeleyDB version %s\n", - DbEnv::version(nullptr, nullptr, nullptr)); - LogPrintf("Using wallet %s\n", file_path.string()); - - if (!env->Open(true /* retry */)) { - errorStr = strprintf( - _("Error initializing wallet database environment %s!"), walletDir); - return false; - } - - return true; -} - -bool BerkeleyBatch::VerifyDatabaseFile(const fs::path &file_path, - bilingual_str &errorStr) { - std::string walletFile; - std::shared_ptr env = - GetWalletEnv(file_path, walletFile); - fs::path walletDir = env->Directory(); - - if (fs::exists(walletDir / walletFile)) { - if (!env->Verify(walletFile)) { - errorStr = - strprintf(_("%s corrupt. Try using the wallet tool " - "bitcoin-wallet to salvage or restoring a backup."), - walletFile); - return false; - } - } - // also return true if files does not exists - return true; -} - -void BerkeleyEnvironment::CheckpointLSN(const std::string &strFile) { - dbenv->txn_checkpoint(0, 0, 0); - if (fMockDb) { - return; - } - dbenv->lsn_reset(strFile.c_str(), 0); -} - -BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase &database, const char *pszMode, - bool fFlushOnCloseIn) - : pdb(nullptr), activeTxn(nullptr) { - fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); - fFlushOnClose = fFlushOnCloseIn; - env = database.env.get(); - if (database.IsDummy()) { - return; - } - - const std::string &strFilename = database.strFile; - - bool fCreate = strchr(pszMode, 'c') != nullptr; - unsigned int nFlags = DB_THREAD; - if (fCreate) { - nFlags |= DB_CREATE; - } - - { - LOCK(cs_db); - if (!env->Open(false /* retry */)) { - throw std::runtime_error( - "BerkeleyBatch: Failed to open database environment."); - } - - pdb = database.m_db.get(); - if (pdb == nullptr) { - int ret; - std::unique_ptr pdb_temp = - std::make_unique(env->dbenv.get(), 0); - - bool fMockDb = env->IsMock(); - if (fMockDb) { - DbMpoolFile *mpf = pdb_temp->get_mpf(); - ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); - if (ret != 0) { - throw std::runtime_error( - strprintf("BerkeleyBatch: Failed to configure for no " - "temp file backing for database %s", - strFilename)); - } - } - - ret = pdb_temp->open( - nullptr, // Txn pointer - fMockDb ? nullptr : strFilename.c_str(), // Filename - fMockDb ? strFilename.c_str() : "main", // Logical db name - DB_BTREE, // Database type - nFlags, // Flags - 0); - - if (ret != 0) { - throw std::runtime_error( - strprintf("BerkeleyBatch: Error %d, can't open database %s", - ret, strFilename)); - } - - // Call CheckUniqueFileid on the containing BDB environment to - // avoid BDB data consistency bugs that happen when different data - // files in the same environment have the same fileid. - // - // Also call CheckUniqueFileid on all the other g_dbenvs to prevent - // bitcoin from opening the same data file through another - // environment when the file is referenced through equivalent but - // not obviously identical symlinked or hard linked or bind mounted - // paths. In the future a more relaxed check for equal inode and - // device ids could be done instead, which would allow opening - // different backup copies of a wallet at the same time. Maybe even - // more ideally, an exclusive lock for accessing the database could - // be implemented, so no equality checks are needed at all. (Newer - // versions of BDB have an set_lk_exclusive method for this - // purpose, but the older version we use does not.) - for (const auto &dbenv : g_dbenvs) { - CheckUniqueFileid(*dbenv.second.lock().get(), strFilename, - *pdb_temp, this->env->m_fileids[strFilename]); - } - - pdb = pdb_temp.release(); - database.m_db.reset(pdb); - - if (fCreate && !Exists(std::string("version"))) { - bool fTmp = fReadOnly; - fReadOnly = false; - Write(std::string("version"), CLIENT_VERSION); - fReadOnly = fTmp; - } - } - ++env->mapFileUseCount[strFilename]; - strFile = strFilename; - } -} - -void BerkeleyBatch::Flush() { - if (activeTxn) { - return; - } - - // Flush database activity from memory pool to disk log - unsigned int nMinutes = 0; - if (fReadOnly) { - nMinutes = 1; - } - - // env is nullptr for dummy databases (i.e. in tests). Don't actually flush - // if env is nullptr so we don't segfault - if (env) { - env->dbenv->txn_checkpoint( - nMinutes - ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 - : 0, - nMinutes, 0); - } -} - -void BerkeleyDatabase::IncrementUpdateCounter() { - ++nUpdateCounter; -} - -void BerkeleyBatch::Close() { - if (!pdb) { - return; - } - if (activeTxn) { - activeTxn->abort(); - } - activeTxn = nullptr; - pdb = nullptr; - - if (fFlushOnClose) { - Flush(); - } - - { - LOCK(cs_db); - --env->mapFileUseCount[strFile]; - } - env->m_db_in_use.notify_all(); -} - -void BerkeleyEnvironment::CloseDb(const std::string &strFile) { - LOCK(cs_db); - auto it = m_databases.find(strFile); - assert(it != m_databases.end()); - BerkeleyDatabase &database = it->second.get(); - if (database.m_db) { - // Close the database handle - database.m_db->close(0); - database.m_db.reset(); - } -} - -void BerkeleyEnvironment::ReloadDbEnv() { - // Make sure that no Db's are in use - AssertLockNotHeld(cs_db); - std::unique_lock lock(cs_db); - m_db_in_use.wait(lock, [this]() { - for (auto &count : mapFileUseCount) { - if (count.second > 0) { - return false; - } - } - return true; - }); - - std::vector filenames; - for (auto it : m_databases) { - filenames.push_back(it.first); - } - // Close the individual Db's - for (const std::string &filename : filenames) { - CloseDb(filename); - } - // Reset the environment - // This will flush and close the environment - Flush(true); - Reset(); - Open(true); -} - -bool BerkeleyBatch::Rewrite(BerkeleyDatabase &database, const char *pszSkip) { - if (database.IsDummy()) { - return true; - } - BerkeleyEnvironment *env = database.env.get(); - const std::string &strFile = database.strFile; - while (true) { - { - LOCK(cs_db); - if (!env->mapFileUseCount.count(strFile) || - env->mapFileUseCount[strFile] == 0) { - // Flush log data to the dat file - env->CloseDb(strFile); - env->CheckpointLSN(strFile); - env->mapFileUseCount.erase(strFile); - - bool fSuccess = true; - LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile); - std::string strFileRes = strFile + ".rewrite"; - { - // surround usage of db with extra {} - BerkeleyBatch db(database, "r"); - std::unique_ptr pdbCopy = - std::make_unique(env->dbenv.get(), 0); - - int ret = pdbCopy->open(nullptr, // Txn pointer - strFileRes.c_str(), // Filename - "main", // Logical db name - DB_BTREE, // Database type - DB_CREATE, // Flags - 0); - if (ret > 0) { - LogPrintf("BerkeleyBatch::Rewrite: Can't create " - "database file %s\n", - strFileRes); - fSuccess = false; - } - - Dbc *pcursor = db.GetCursor(); - if (pcursor) { - while (fSuccess) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue); - if (ret1 == DB_NOTFOUND) { - pcursor->close(); - break; - } - if (ret1 != 0) { - pcursor->close(); - fSuccess = false; - break; - } - if (pszSkip && - strncmp(ssKey.data(), pszSkip, - std::min(ssKey.size(), - strlen(pszSkip))) == 0) { - continue; - } - if (strncmp(ssKey.data(), "\x07version", 8) == 0) { - // Update version: - ssValue.clear(); - ssValue << CLIENT_VERSION; - } - Dbt datKey(ssKey.data(), ssKey.size()); - Dbt datValue(ssValue.data(), ssValue.size()); - int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, - DB_NOOVERWRITE); - if (ret2 > 0) { - fSuccess = false; - } - } - } - if (fSuccess) { - db.Close(); - env->CloseDb(strFile); - if (pdbCopy->close(0)) { - fSuccess = false; - } - } else { - pdbCopy->close(0); - } - } - if (fSuccess) { - Db dbA(env->dbenv.get(), 0); - if (dbA.remove(strFile.c_str(), nullptr, 0)) { - fSuccess = false; - } - Db dbB(env->dbenv.get(), 0); - if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), - 0)) { - fSuccess = false; - } - } - if (!fSuccess) { - LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite " - "database file %s\n", - strFileRes); - } - return fSuccess; - } - } - UninterruptibleSleep(std::chrono::milliseconds{100}); - } -} - -void BerkeleyEnvironment::Flush(bool fShutdown) { - int64_t nStart = GetTimeMillis(); - // Flush log data to the actual data file on all files that are not in use - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", - strPath, fShutdown ? "true" : "false", - fDbEnvInit ? "" : " database not started"); - if (!fDbEnvInit) { - return; - } - { - LOCK(cs_db); - std::map::iterator mi = mapFileUseCount.begin(); - while (mi != mapFileUseCount.end()) { - std::string strFile = (*mi).first; - int nRefCount = (*mi).second; - LogPrint( - BCLog::WALLETDB, - "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", - strFile, nRefCount); - if (nRefCount == 0) { - // Move log data to the dat file - CloseDb(strFile); - LogPrint(BCLog::WALLETDB, - "BerkeleyEnvironment::Flush: %s checkpoint\n", - strFile); - dbenv->txn_checkpoint(0, 0, 0); - LogPrint(BCLog::WALLETDB, - "BerkeleyEnvironment::Flush: %s detach\n", strFile); - if (!fMockDb) { - dbenv->lsn_reset(strFile.c_str(), 0); - } - LogPrint(BCLog::WALLETDB, - "BerkeleyEnvironment::Flush: %s closed\n", strFile); - mapFileUseCount.erase(mi++); - } else { - mi++; - } - } - LogPrint(BCLog::WALLETDB, - "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", - fShutdown ? "true" : "false", - fDbEnvInit ? "" : " database not started", - GetTimeMillis() - nStart); - if (fShutdown) { - char **listp; - if (mapFileUseCount.empty()) { - dbenv->log_archive(&listp, DB_ARCH_REMOVE); - Close(); - if (!fMockDb) { - fs::remove_all(fs::path(strPath) / "database"); - } - } - } - } -} - -bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase &database) { - if (database.IsDummy()) { - return true; - } - bool ret = false; - BerkeleyEnvironment *env = database.env.get(); - const std::string &strFile = database.strFile; - TRY_LOCK(cs_db, lockDb); - if (lockDb) { - // Don't do this if any databases are in use - int nRefCount = 0; - std::map::iterator mit = env->mapFileUseCount.begin(); - while (mit != env->mapFileUseCount.end()) { - nRefCount += (*mit).second; - mit++; - } - - if (nRefCount == 0) { - boost::this_thread::interruption_point(); - std::map::iterator mi = - env->mapFileUseCount.find(strFile); - if (mi != env->mapFileUseCount.end()) { - LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); - int64_t nStart = GetTimeMillis(); - - // Flush wallet file so it's self contained - env->CloseDb(strFile); - env->CheckpointLSN(strFile); - - env->mapFileUseCount.erase(mi++); - LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, - GetTimeMillis() - nStart); - ret = true; - } - } - } - - return ret; -} - -bool BerkeleyDatabase::Rewrite(const char *pszSkip) { - return BerkeleyBatch::Rewrite(*this, pszSkip); -} - -bool BerkeleyDatabase::Backup(const std::string &strDest) const { - if (IsDummy()) { - return false; - } - while (true) { - { - LOCK(cs_db); - if (!env->mapFileUseCount.count(strFile) || - env->mapFileUseCount[strFile] == 0) { - // Flush log data to the dat file - env->CloseDb(strFile); - env->CheckpointLSN(strFile); - env->mapFileUseCount.erase(strFile); - - // Copy wallet file. - fs::path pathSrc = env->Directory() / strFile; - fs::path pathDest(strDest); - if (fs::is_directory(pathDest)) { - pathDest /= strFile; - } - - try { - if (fs::equivalent(pathSrc, pathDest)) { - LogPrintf("cannot backup to wallet source file %s\n", - pathDest.string()); - return false; - } - - fs::copy_file(pathSrc, pathDest, - fs::copy_option::overwrite_if_exists); - LogPrintf("copied %s to %s\n", strFile, pathDest.string()); - return true; - } catch (const fs::filesystem_error &e) { - LogPrintf("error copying %s to %s - %s\n", strFile, - pathDest.string(), - fsbridge::get_filesystem_error_message(e)); - return false; - } - } - } - UninterruptibleSleep(std::chrono::milliseconds{100}); - } -} - -void BerkeleyDatabase::Flush(bool shutdown) { - if (!IsDummy()) { - env->Flush(shutdown); - if (shutdown) { - LOCK(cs_db); - g_dbenvs.erase(env->Directory().string()); - env = nullptr; - } else { - // TODO: To avoid g_dbenvs.erase erasing the environment prematurely - // after the first database shutdown when multiple databases are - // open in the same environment, should replace raw database `env` - // pointers with shared or weak pointers, or else separate the - // database and environment shutdowns so environments can be shut - // down after databases. - env->m_fileids.erase(strFile); - } - } -} - -void BerkeleyDatabase::ReloadDbEnv() { - if (!IsDummy()) { - env->ReloadDbEnv(); - } -} - -Dbc *BerkeleyBatch::GetCursor() { - if (!pdb) { - return nullptr; - } - Dbc *pcursor = nullptr; - int ret = pdb->cursor(nullptr, &pcursor, 0); - if (ret != 0) { - return nullptr; - } - return pcursor; -} - -int BerkeleyBatch::ReadAtCursor(Dbc *pcursor, CDataStream &ssKey, - CDataStream &ssValue) { - // Read at cursor - SafeDbt datKey; - SafeDbt datValue; - int ret = pcursor->get(datKey, datValue, DB_NEXT); - if (ret != 0) { - return ret; - } else if (datKey.get_data() == nullptr || datValue.get_data() == nullptr) { - return 99999; - } - - // Convert to streams - ssKey.SetType(SER_DISK); - ssKey.clear(); - ssKey.write((char *)datKey.get_data(), datKey.get_size()); - ssValue.SetType(SER_DISK); - ssValue.clear(); - ssValue.write((char *)datValue.get_data(), datValue.get_size()); - return 0; -} - -bool BerkeleyBatch::TxnBegin() { - if (!pdb || activeTxn) { - return false; - } - DbTxn *ptxn = env->TxnBegin(); - if (!ptxn) { - return false; - } - activeTxn = ptxn; - return true; -} - -bool BerkeleyBatch::TxnCommit() { - if (!pdb || !activeTxn) { - return false; - } - int ret = activeTxn->commit(0); - activeTxn = nullptr; - return (ret == 0); -} - -bool BerkeleyBatch::TxnAbort() { - if (!pdb || !activeTxn) { - return false; - } - int ret = activeTxn->abort(); - activeTxn = nullptr; - return (ret == 0); -} diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include #include diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -11,7 +11,7 @@ #include #include