diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index d1ff4dcd1..224ccf64a 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -1,938 +1,929 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #ifndef WIN32 #include #endif namespace { //! Make sure database has a unique fileid within the environment. If it //! doesn't, throw an error. BDB caches do not work properly when more than one //! open database has the same fileid (values written to one database may show //! up in reads to other databases). //! //! BerkeleyDB generates unique fileids by default //! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html), //! so bitcoin should never create different databases with the same fileid, but //! this error can be triggered if users manually copy database files. void CheckUniqueFileid(const BerkeleyEnvironment &env, const std::string &filename, Db &db, WalletDatabaseFileId &fileid) { if (env.IsMock()) { return; } int ret = db.get_mpf()->get_fileid(fileid.value); if (ret != 0) { - throw std::runtime_error(strprintf( - "BerkeleyBatch: Can't open database %s (get_fileid failed with %d)", - filename, ret)); + throw std::runtime_error( + strprintf("BerkeleyDatabase: Can't open database %s (get_fileid " + "failed with %d)", + filename, ret)); } for (const auto &item : env.m_fileids) { if (fileid == item.second && &fileid != &item.second) { - throw std::runtime_error(strprintf( - "BerkeleyBatch: Can't open database %s (duplicates fileid %s " - "from %s)", - filename, HexStr(item.second.value), item.first)); + throw std::runtime_error( + strprintf("BerkeleyDatabase: Can't open database %s " + "(duplicates fileid %s " + "from %s)", + filename, HexStr(item.second.value), item.first)); } } } RecursiveMutex cs_db; //! Map from directory name to db environment. std::map> g_dbenvs GUARDED_BY(cs_db); } // namespace bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId &rhs) const { return memcmp(value, &rhs.value, sizeof(value)) == 0; } bool IsBDBWalletLoaded(const fs::path &wallet_path) { fs::path env_directory; std::string database_filename; SplitWalletPath(wallet_path, env_directory, database_filename); LOCK(cs_db); auto env = g_dbenvs.find(env_directory.string()); if (env == g_dbenvs.end()) { return false; } auto database = env->second.lock(); return database && database->IsDatabaseLoaded(database_filename); } /** * @param[in] wallet_path Path to wallet directory. Or (for backwards * compatibility only) a path to a berkeley btree data file inside a wallet * directory. * @param[out] database_filename Filename of berkeley btree data file inside the * wallet directory. * @return A shared pointer to the BerkeleyEnvironment object for the wallet * directory, never empty because ~BerkeleyEnvironment erases the weak pointer * from the g_dbenvs map. * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the * directory path key was not already in the map. */ std::shared_ptr GetWalletEnv(const fs::path &wallet_path, std::string &database_filename) { fs::path env_directory; SplitWalletPath(wallet_path, env_directory, database_filename); LOCK(cs_db); auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr()); if (inserted.second) { auto env = std::make_shared(env_directory.string()); inserted.first->second = env; return env; } return inserted.first->second.lock(); } // // BerkeleyBatch // void BerkeleyEnvironment::Close() { if (!fDbEnvInit) { return; } fDbEnvInit = false; for (auto &db : m_databases) { BerkeleyDatabase &database = db.second.get(); assert(database.m_refcount <= 0); if (database.m_db) { database.m_db->close(0); database.m_db.reset(); } } FILE *error_file = nullptr; dbenv->get_errfile(&error_file); int ret = dbenv->close(0); if (ret != 0) { LogPrintf("BerkeleyEnvironment::Close: Error %d closing database " "environment: %s\n", ret, DbEnv::strerror(ret)); } if (!fMockDb) { DbEnv(u_int32_t(0)).remove(strPath.c_str(), 0); } if (error_file) { fclose(error_file); } UnlockDirectory(strPath, ".walletlock"); } void BerkeleyEnvironment::Reset() { dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS)); fDbEnvInit = false; fMockDb = false; } BerkeleyEnvironment::BerkeleyEnvironment(const fs::path &dir_path) : strPath(dir_path.string()) { Reset(); } BerkeleyEnvironment::~BerkeleyEnvironment() { LOCK(cs_db); g_dbenvs.erase(strPath); Close(); } bool BerkeleyEnvironment::Open(bilingual_str &err) { if (fDbEnvInit) { return true; } fs::path pathIn = strPath; TryCreateDirectories(pathIn); if (!LockDirectory(pathIn, ".walletlock")) { LogPrintf("Cannot obtain a lock on wallet directory %s. Another " "instance of bitcoin may be using it.\n", strPath); err = strprintf(_("Error initializing wallet database environment %s!"), Directory()); return false; } fs::path pathLogDir = pathIn / "database"; TryCreateDirectories(pathLogDir); fs::path pathErrorFile = pathIn / "db.log"; LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string()); unsigned int nEnvFlags = 0; if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) { nEnvFlags |= DB_PRIVATE; } dbenv->set_lg_dir(pathLogDir.string().c_str()); // 1 MiB should be enough for just the wallet dbenv->set_cachesize(0, 0x100000, 1); dbenv->set_lg_bsize(0x10000); dbenv->set_lg_max(1048576); dbenv->set_lk_max_locks(40000); dbenv->set_lk_max_objects(40000); /// debug dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); dbenv->set_flags(DB_AUTO_COMMIT, 1); dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1); dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1); int ret = dbenv->open(strPath.c_str(), DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD | DB_RECOVER | nEnvFlags, S_IRUSR | S_IWUSR); if (ret != 0) { LogPrintf("BerkeleyEnvironment::Open: Error %d opening database " "environment: %s\n", ret, DbEnv::strerror(ret)); int ret2 = dbenv->close(0); if (ret2 != 0) { LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed " "database environment: %s\n", ret2, DbEnv::strerror(ret2)); } Reset(); err = strprintf(_("Error initializing wallet database environment %s!"), Directory()); if (ret == DB_RUNRECOVERY) { err += Untranslated(" ") + _("This error could occur if this wallet was not shutdown " "cleanly and was last loaded using a build with a newer " "version of Berkeley DB. If so, please use the software " "that last loaded this wallet"); } return false; } fDbEnvInit = true; fMockDb = false; return true; } //! Construct an in-memory mock Berkeley environment for testing BerkeleyEnvironment::BerkeleyEnvironment() { Reset(); LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n"); dbenv->set_cachesize(1, 0, 1); dbenv->set_lg_bsize(10485760 * 4); dbenv->set_lg_max(10485760); dbenv->set_lk_max_locks(10000); dbenv->set_lk_max_objects(10000); dbenv->set_flags(DB_AUTO_COMMIT, 1); dbenv->log_set_config(DB_LOG_IN_MEMORY, 1); int ret = dbenv->open(nullptr, DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD | DB_PRIVATE, S_IRUSR | S_IWUSR); if (ret > 0) { throw std::runtime_error( strprintf("BerkeleyEnvironment::MakeMock: Error %d opening " "database environment.", ret)); } fDbEnvInit = true; fMockDb = true; } BerkeleyBatch::SafeDbt::SafeDbt() { m_dbt.set_flags(DB_DBT_MALLOC); } BerkeleyBatch::SafeDbt::SafeDbt(void *data, size_t size) : m_dbt(data, size) {} BerkeleyBatch::SafeDbt::~SafeDbt() { if (m_dbt.get_data() != nullptr) { // Clear memory, e.g. in case it was a private key memory_cleanse(m_dbt.get_data(), m_dbt.get_size()); // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be // freed by the caller. // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html if (m_dbt.get_flags() & DB_DBT_MALLOC) { free(m_dbt.get_data()); } } } const void *BerkeleyBatch::SafeDbt::get_data() const { return m_dbt.get_data(); } u_int32_t BerkeleyBatch::SafeDbt::get_size() const { return m_dbt.get_size(); } BerkeleyBatch::SafeDbt::operator Dbt *() { return &m_dbt; } bool BerkeleyDatabase::Verify(bilingual_str &errorStr) { fs::path walletDir = env->Directory(); fs::path file_path = walletDir / strFile; LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(nullptr, nullptr, nullptr)); LogPrintf("Using wallet %s\n", file_path.string()); if (!env->Open(errorStr)) { return false; } if (fs::exists(file_path)) { assert(m_refcount == 0); Db db(env->dbenv.get(), 0); int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); if (result != 0) { errorStr = strprintf(_("%s corrupt. Try using the wallet tool " "bitcoin-wallet to salvage or restoring a backup."), file_path); return false; } } // also return true if files does not exists return true; } void BerkeleyEnvironment::CheckpointLSN(const std::string &strFile) { dbenv->txn_checkpoint(0, 0, 0); if (fMockDb) { return; } dbenv->lsn_reset(strFile.c_str(), 0); } BerkeleyDatabase::~BerkeleyDatabase() { if (env) { LOCK(cs_db); + env->CloseDb(strFile); + assert(!m_db); size_t erased = env->m_databases.erase(strFile); assert(erased == 1); env->m_fileids.erase(strFile); } } BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase &database, const char *pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr), m_cursor(nullptr), m_database(database) { database.AddRef(); database.Open(pszMode); fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); fFlushOnClose = fFlushOnCloseIn; env = database.env.get(); pdb = database.m_db.get(); strFile = database.strFile; bool fCreate = strchr(pszMode, 'c') != nullptr; if (fCreate && !Exists(std::string("version"))) { bool fTmp = fReadOnly; fReadOnly = false; Write(std::string("version"), CLIENT_VERSION); fReadOnly = fTmp; } } void BerkeleyDatabase::Open(const char *pszMode) { if (IsDummy()) { return; } bool fCreate = strchr(pszMode, 'c') != nullptr; unsigned int nFlags = DB_THREAD; if (fCreate) { nFlags |= DB_CREATE; } { LOCK(cs_db); bilingual_str open_err; if (!env->Open(open_err)) { throw std::runtime_error( "BerkeleyDatabase: Failed to open database environment."); } if (m_db == nullptr) { int ret; std::unique_ptr pdb_temp = std::make_unique(env->dbenv.get(), 0); bool fMockDb = env->IsMock(); if (fMockDb) { DbMpoolFile *mpf = pdb_temp->get_mpf(); ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); if (ret != 0) { throw std::runtime_error(strprintf( "BerkeleyDatabase: Failed to configure for no " "temp file backing for database %s", strFile)); } } ret = pdb_temp->open( nullptr, // Txn pointer fMockDb ? nullptr : strFile.c_str(), // Filename fMockDb ? strFile.c_str() : "main", // Logical db name DB_BTREE, // Database type nFlags, // Flags 0); if (ret != 0) { throw std::runtime_error(strprintf( "BerkeleyDatabase: Error %d, can't open database %s", ret, strFile)); } + m_file_path = (env->Directory() / strFile).string(); // Call CheckUniqueFileid on the containing BDB environment to // avoid BDB data consistency bugs that happen when different data // files in the same environment have the same fileid. - // - // Also call CheckUniqueFileid on all the other g_dbenvs to prevent - // bitcoin from opening the same data file through another - // environment when the file is referenced through equivalent but - // not obviously identical symlinked or hard linked or bind mounted - // paths. In the future a more relaxed check for equal inode and - // device ids could be done instead, which would allow opening - // different backup copies of a wallet at the same time. Maybe even - // more ideally, an exclusive lock for accessing the database could - // be implemented, so no equality checks are needed at all. (Newer - // versions of BDB have an set_lk_exclusive method for this - // purpose, but the older version we use does not.) - for (const auto &dbenv : g_dbenvs) { - CheckUniqueFileid(*dbenv.second.lock().get(), strFile, - *pdb_temp, this->env->m_fileids[strFile]); - } + CheckUniqueFileid(*env, strFile, *pdb_temp, + this->env->m_fileids[strFile]); m_db.reset(pdb_temp.release()); } } } void BerkeleyBatch::Flush() { if (activeTxn) { return; } // Flush database activity from memory pool to disk log unsigned int nMinutes = 0; if (fReadOnly) { nMinutes = 1; } // env is nullptr for dummy databases (i.e. in tests). Don't actually flush // if env is nullptr so we don't segfault if (env) { env->dbenv->txn_checkpoint( nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); } } void BerkeleyDatabase::IncrementUpdateCounter() { ++nUpdateCounter; } BerkeleyBatch::~BerkeleyBatch() { Close(); m_database.RemoveRef(); } void BerkeleyBatch::Close() { if (!pdb) { return; } if (activeTxn) { activeTxn->abort(); } activeTxn = nullptr; pdb = nullptr; CloseCursor(); if (fFlushOnClose) { Flush(); } } void BerkeleyEnvironment::CloseDb(const std::string &strFile) { LOCK(cs_db); auto it = m_databases.find(strFile); assert(it != m_databases.end()); BerkeleyDatabase &database = it->second.get(); if (database.m_db) { // Close the database handle database.m_db->close(0); database.m_db.reset(); } } void BerkeleyEnvironment::ReloadDbEnv() { // Make sure that no Db's are in use AssertLockNotHeld(cs_db); std::unique_lock lock(cs_db); m_db_in_use.wait(lock, [this]() { for (auto &db : m_databases) { if (db.second.get().m_refcount > 0) { return false; } } return true; }); std::vector filenames; for (auto it : m_databases) { filenames.push_back(it.first); } // Close the individual Db's for (const std::string &filename : filenames) { CloseDb(filename); } // Reset the environment // This will flush and close the environment Flush(true); Reset(); bilingual_str open_err; Open(open_err); } bool BerkeleyDatabase::Rewrite(const char *pszSkip) { if (IsDummy()) { return true; } while (true) { { LOCK(cs_db); if (m_refcount <= 0) { // Flush log data to the dat file env->CloseDb(strFile); env->CheckpointLSN(strFile); m_refcount = -1; bool fSuccess = true; LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile); std::string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} BerkeleyBatch db(*this, "r"); std::unique_ptr pdbCopy = std::make_unique(env->dbenv.get(), 0); int ret = pdbCopy->open(nullptr, // Txn pointer strFileRes.c_str(), // Filename "main", // Logical db name DB_BTREE, // Database type DB_CREATE, // Flags 0); if (ret > 0) { LogPrintf("BerkeleyBatch::Rewrite: Can't create " "database file %s\n", strFileRes); fSuccess = false; } if (db.StartCursor()) { while (fSuccess) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); bool complete; bool ret1 = db.ReadAtCursor(ssKey, ssValue, complete); if (complete) { break; } if (!ret1) { fSuccess = false; break; } if (pszSkip && strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) { continue; } if (strncmp(ssKey.data(), "\x07version", 8) == 0) { // Update version: ssValue.clear(); ssValue << CLIENT_VERSION; } Dbt datKey(ssKey.data(), ssKey.size()); Dbt datValue(ssValue.data(), ssValue.size()); int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE); if (ret2 > 0) { fSuccess = false; } } db.CloseCursor(); } if (fSuccess) { db.Close(); env->CloseDb(strFile); if (pdbCopy->close(0)) { fSuccess = false; } } else { pdbCopy->close(0); } } if (fSuccess) { Db dbA(env->dbenv.get(), 0); if (dbA.remove(strFile.c_str(), nullptr, 0)) { fSuccess = false; } Db dbB(env->dbenv.get(), 0); if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0)) { fSuccess = false; } } if (!fSuccess) { LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite " "database file %s\n", strFileRes); } return fSuccess; } } UninterruptibleSleep(std::chrono::milliseconds{100}); } } void BerkeleyEnvironment::Flush(bool fShutdown) { int64_t nStart = GetTimeMillis(); // Flush log data to the actual data file on all files that are not in use LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); if (!fDbEnvInit) { return; } { LOCK(cs_db); bool no_dbs_accessed = true; for (auto &db_it : m_databases) { std::string strFile = db_it.first; int nRefCount = db_it.second.get().m_refcount; if (nRefCount < 0) { continue; } LogPrint( BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); if (nRefCount == 0) { // Move log data to the dat file CloseDb(strFile); LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); dbenv->txn_checkpoint(0, 0, 0); LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s detach\n", strFile); if (!fMockDb) { dbenv->lsn_reset(strFile.c_str(), 0); } LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s closed\n", strFile); nRefCount = -1; } else { no_dbs_accessed = false; } } LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); if (fShutdown) { char **listp; if (no_dbs_accessed) { dbenv->log_archive(&listp, DB_ARCH_REMOVE); Close(); if (!fMockDb) { fs::remove_all(fs::path(strPath) / "database"); } } } } } bool BerkeleyDatabase::PeriodicFlush() { // There's nothing to do for dummy databases. Return true. if (IsDummy()) { return true; } // Don't flush if we can't acquire the lock. TRY_LOCK(cs_db, lockDb); if (!lockDb) { return false; } // Don't flush if any databases are in use for (auto &it : env->m_databases) { if (it.second.get().m_refcount > 0) { return false; } } // Don't flush if there haven't been any batch writes for this database. if (m_refcount < 0) { return false; } LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); int64_t nStart = GetTimeMillis(); // Flush wallet file so it's self contained env->CloseDb(strFile); env->CheckpointLSN(strFile); m_refcount = -1; LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); return true; } bool BerkeleyDatabase::Backup(const std::string &strDest) const { if (IsDummy()) { return false; } while (true) { { LOCK(cs_db); if (m_refcount <= 0) { // Flush log data to the dat file env->CloseDb(strFile); env->CheckpointLSN(strFile); // Copy wallet file. fs::path pathSrc = env->Directory() / strFile; fs::path pathDest(strDest); if (fs::is_directory(pathDest)) { pathDest /= strFile; } try { if (fs::equivalent(pathSrc, pathDest)) { LogPrintf("cannot backup to wallet source file %s\n", pathDest.string()); return false; } fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); LogPrintf("copied %s to %s\n", strFile, pathDest.string()); return true; } catch (const fs::filesystem_error &e) { LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), fsbridge::get_filesystem_error_message(e)); return false; } } } UninterruptibleSleep(std::chrono::milliseconds{100}); } } void BerkeleyDatabase::Flush() { if (!IsDummy()) { env->Flush(false); } } void BerkeleyDatabase::Close() { if (!IsDummy()) { env->Flush(true); } } void BerkeleyDatabase::ReloadDbEnv() { if (!IsDummy()) { env->ReloadDbEnv(); } } bool BerkeleyBatch::StartCursor() { assert(!m_cursor); if (!pdb) { return false; } int ret = pdb->cursor(nullptr, &m_cursor, 0); return ret == 0; } bool BerkeleyBatch::ReadAtCursor(CDataStream &ssKey, CDataStream &ssValue, bool &complete) { complete = false; if (m_cursor == nullptr) { return false; } // Read at cursor SafeDbt datKey; SafeDbt datValue; int ret = m_cursor->get(datKey, datValue, DB_NEXT); if (ret == DB_NOTFOUND) { complete = true; } if (ret != 0) { return false; } else if (datKey.get_data() == nullptr || datValue.get_data() == nullptr) { return false; } // Convert to streams ssKey.SetType(SER_DISK); ssKey.clear(); ssKey.write((char *)datKey.get_data(), datKey.get_size()); ssValue.SetType(SER_DISK); ssValue.clear(); ssValue.write((char *)datValue.get_data(), datValue.get_size()); return true; } void BerkeleyBatch::CloseCursor() { if (!m_cursor) { return; } m_cursor->close(); m_cursor = nullptr; } bool BerkeleyBatch::TxnBegin() { if (!pdb || activeTxn) { return false; } DbTxn *ptxn = env->TxnBegin(); if (!ptxn) { return false; } activeTxn = ptxn; return true; } bool BerkeleyBatch::TxnCommit() { if (!pdb || !activeTxn) { return false; } int ret = activeTxn->commit(0); activeTxn = nullptr; return (ret == 0); } bool BerkeleyBatch::TxnAbort() { if (!pdb || !activeTxn) { return false; } int ret = activeTxn->abort(); activeTxn = nullptr; return (ret == 0); } bool BerkeleyBatch::ReadKey(CDataStream &&key, CDataStream &value) { if (!pdb) { return false; } SafeDbt datKey(key.data(), key.size()); SafeDbt datValue; int ret = pdb->get(activeTxn, datKey, datValue, 0); if (ret == 0 && datValue.get_data() != nullptr) { value.write((char *)datValue.get_data(), datValue.get_size()); return true; } return false; } bool BerkeleyBatch::WriteKey(CDataStream &&key, CDataStream &&value, bool overwrite) { if (!pdb) { return true; } if (fReadOnly) { assert(!"Write called on database in read-only mode"); } SafeDbt datKey(key.data(), key.size()); SafeDbt datValue(value.data(), value.size()); int ret = pdb->put(activeTxn, datKey, datValue, (overwrite ? 0 : DB_NOOVERWRITE)); return (ret == 0); } bool BerkeleyBatch::EraseKey(CDataStream &&key) { if (!pdb) { return false; } if (fReadOnly) { assert(!"Erase called on database in read-only mode"); } SafeDbt datKey(key.data(), key.size()); int ret = pdb->del(activeTxn, datKey, 0); return (ret == 0 || ret == DB_NOTFOUND); } bool BerkeleyBatch::HasKey(CDataStream &&key) { if (!pdb) { return false; } SafeDbt datKey(key.data(), key.size()); int ret = pdb->exists(activeTxn, datKey, 0); return ret == 0; } void BerkeleyDatabase::AddRef() { LOCK(cs_db); if (m_refcount < 0) { m_refcount = 1; } else { m_refcount++; } } void BerkeleyDatabase::RemoveRef() { LOCK(cs_db); m_refcount--; if (env) { env->m_db_in_use.notify_all(); } } std::unique_ptr BerkeleyDatabase::MakeBatch(const char *mode, bool flush_on_close) { return std::make_unique(*this, mode, flush_on_close); } diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index c98181152..1f3a1a722 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -1,436 +1,436 @@ #!/usr/bin/env python3 # Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multiwallet. Verify that a bitcoind node can load multiple wallet files """ from decimal import Decimal from threading import Thread import os import shutil import time from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework from test_framework.test_node import ErrorMatch from test_framework.util import ( assert_equal, assert_raises_rpc_error, get_rpc_proxy, ) FEATURE_LATEST = 200300 got_loading_error = False def test_load_unload(node, name): global got_loading_error for i in range(10): if got_loading_error: return try: node.loadwallet(name) node.unloadwallet(name) except JSONRPCException as e: if e.error['code'] == - \ 4 and 'Wallet already being loading' in e.error['message']: got_loading_error = True return class MultiWalletTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 def skip_test_if_missing_module(self): self.skip_if_no_wallet() def add_options(self, parser): parser.add_argument( '--data_wallets_dir', default=os.path.join( os.path.dirname( os.path.realpath(__file__)), 'data/wallets/'), help='Test data with wallet directories (default: %(default)s)', ) def run_test(self): node = self.nodes[0] def data_dir(*p): return os.path.join(node.datadir, self.chain, *p) def wallet_dir(*p): return data_dir('wallets', *p) def wallet(name): return node.get_wallet_rpc(name) def wallet_file(name): if os.path.isdir(wallet_dir(name)): return wallet_dir(name, "wallet.dat") return wallet_dir(name) assert_equal(self.nodes[0].listwalletdir(), {'wallets': [{'name': ''}]}) # check wallet.dat is created self.stop_nodes() assert_equal(os.path.isfile(wallet_dir('wallet.dat')), True) # create symlink to verify wallet directory path can be referenced # through symlink if os.name != 'nt': os.mkdir(wallet_dir('w7')) os.symlink('w7', wallet_dir('w7_symlink')) # rename wallet.dat to make sure plain wallet file paths (as opposed to # directory paths) can be loaded os.rename(wallet_dir("wallet.dat"), wallet_dir("w8")) # create another dummy wallet for use in testing backups later self.start_node(0, []) self.stop_nodes() empty_wallet = os.path.join(self.options.tmpdir, 'empty.dat') os.rename(wallet_dir("wallet.dat"), empty_wallet) # restart node with a mix of wallet names: # w1, w2, w3 - to verify new wallets created when non-existing paths specified # w - to verify wallet name matching works when one wallet path is prefix of another # sub/w5 - to verify relative wallet path is created correctly # extern/w6 - to verify absolute wallet path is created correctly # w7_symlink - to verify symlinked wallet path is initialized correctly # w8 - to verify existing wallet file is loaded correctly # '' - to verify default wallet file is created correctly wallet_names = ['w1', 'w2', 'w3', 'w', 'sub/w5', os.path.join(self.options.tmpdir, 'extern/w6'), 'w7_symlink', 'w8', ''] if os.name == 'nt': wallet_names.remove('w7_symlink') extra_args = ['-wallet={}'.format(n) for n in wallet_names] self.start_node(0, extra_args) assert_equal(sorted(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), [ '', os.path.join('sub', 'w5'), 'w', 'w1', 'w2', 'w3', 'w7', 'w7_symlink', 'w8']) assert_equal(set(node.listwallets()), set(wallet_names)) # check that all requested wallets were created self.stop_node(0) for wallet_name in wallet_names: assert_equal(os.path.isfile(wallet_file(wallet_name)), True) # should not initialize if wallet path can't be created exp_stderr = "boost::filesystem::create_directory:" self.nodes[0].assert_start_raises_init_error( ['-wallet=wallet.dat/bad'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) self.nodes[0].assert_start_raises_init_error( ['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" does not exist') self.nodes[0].assert_start_raises_init_error( ['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" is a relative path', cwd=data_dir()) self.nodes[0].assert_start_raises_init_error( ['-walletdir=debug.log'], 'Error: Specified -walletdir "debug.log" is not a directory', cwd=data_dir()) # should not initialize if there are duplicate wallets self.nodes[0].assert_start_raises_init_error( ['-wallet=w1', '-wallet=w1'], 'Error: Error loading wallet w1. Duplicate -wallet filename specified.') # should not initialize if one wallet is a copy of another shutil.copyfile(wallet_dir('w8'), wallet_dir('w8_copy')) - exp_stderr = r"BerkeleyBatch: Can't open database w8_copy \(duplicates fileid \w+ from w8\)" + exp_stderr = r"BerkeleyDatabase: Can't open database w8_copy \(duplicates fileid \w+ from w8\)" self.nodes[0].assert_start_raises_init_error( ['-wallet=w8', '-wallet=w8_copy'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) # should not initialize if wallet file is a symlink if os.name != 'nt': os.symlink('w8', wallet_dir('w8_symlink')) self.nodes[0].assert_start_raises_init_error( ['-wallet=w8_symlink'], r'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX) # should not initialize if the specified walletdir does not exist self.nodes[0].assert_start_raises_init_error( ['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist') # should not initialize if the specified walletdir is not a directory not_a_dir = wallet_dir('notadir') open(not_a_dir, 'a', encoding="utf8").close() self.nodes[0].assert_start_raises_init_error( ['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory') self.log.info("Do not allow -zapwallettxes with multiwallet") self.nodes[0].assert_start_raises_init_error( [ '-zapwallettxes', '-wallet=w1', '-wallet=w2'], "Error: -zapwallettxes is only allowed with a single wallet file") self.nodes[0].assert_start_raises_init_error( [ '-zapwallettxes=1', '-wallet=w1', '-wallet=w2'], "Error: -zapwallettxes is only allowed with a single wallet file") self.nodes[0].assert_start_raises_init_error( [ '-zapwallettxes=2', '-wallet=w1', '-wallet=w2'], "Error: -zapwallettxes is only allowed with a single wallet file") # if wallets/ doesn't exist, datadir should be the default wallet dir wallet_dir2 = data_dir('walletdir') os.rename(wallet_dir(), wallet_dir2) self.start_node(0, ['-wallet=w4', '-wallet=w5']) assert_equal(set(node.listwallets()), {"w4", "w5"}) w5 = wallet("w5") node.generatetoaddress(nblocks=1, address=w5.getnewaddress()) # now if wallets/ exists again, but the rootdir is specified as the # walletdir, w4 and w5 should still be loaded os.rename(wallet_dir2, wallet_dir()) self.restart_node(0, ['-wallet=w4', '-wallet=w5', '-walletdir=' + data_dir()]) assert_equal(set(node.listwallets()), {"w4", "w5"}) w5 = wallet("w5") w5_info = w5.getwalletinfo() assert_equal(w5_info['immature_balance'], 50000000) competing_wallet_dir = os.path.join( self.options.tmpdir, 'competing_walletdir') os.mkdir(competing_wallet_dir) self.restart_node(0, ['-walletdir=' + competing_wallet_dir]) exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\"!" self.nodes[1].assert_start_raises_init_error( ['-walletdir=' + competing_wallet_dir], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) self.restart_node(0, extra_args) assert_equal(sorted(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), [ '', os.path.join('sub', 'w5'), 'w', 'w1', 'w2', 'w3', 'w7', 'w7_symlink', 'w8', 'w8_copy']) wallets = [wallet(w) for w in wallet_names] wallet_bad = wallet("bad") # check wallet names and balances node.generatetoaddress(nblocks=1, address=wallets[0].getnewaddress()) for wallet_name, wallet in zip(wallet_names, wallets): info = wallet.getwalletinfo() assert_equal(info['immature_balance'], 50000000 if wallet is wallets[0] else 0) assert_equal(info['walletname'], wallet_name) # accessing invalid wallet fails assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", wallet_bad.getwalletinfo) # accessing wallet RPC without using wallet endpoint fails assert_raises_rpc_error(-19, "Wallet file not specified (must request wallet RPC through /wallet/ uri-path).", node.getwalletinfo) w1, w2, w3, w4, *_ = wallets node.generatetoaddress(nblocks=101, address=w1.getnewaddress()) assert_equal(w1.getbalance(), 100000000) assert_equal(w2.getbalance(), 0) assert_equal(w3.getbalance(), 0) assert_equal(w4.getbalance(), 0) w1.sendtoaddress(w2.getnewaddress(), 1000000) w1.sendtoaddress(w3.getnewaddress(), 2000000) w1.sendtoaddress(w4.getnewaddress(), 3000000) node.generatetoaddress(nblocks=1, address=w1.getnewaddress()) assert_equal(w2.getbalance(), 1000000) assert_equal(w3.getbalance(), 2000000) assert_equal(w4.getbalance(), 3000000) batch = w1.batch([w1.getblockchaininfo.get_request(), w1.getwalletinfo.get_request()]) assert_equal(batch[0]["result"]["chain"], self.chain) assert_equal(batch[1]["result"]["walletname"], "w1") self.log.info('Check for per-wallet settxfee call') assert_equal(w1.getwalletinfo()['paytxfee'], 0) assert_equal(w2.getwalletinfo()['paytxfee'], 0) w2.settxfee(1000) assert_equal(w1.getwalletinfo()['paytxfee'], 0) assert_equal(w2.getwalletinfo()['paytxfee'], Decimal('1000.00')) self.log.info("Test dynamic wallet loading") self.restart_node(0, ['-nowallet']) assert_equal(node.listwallets(), []) assert_raises_rpc_error(-32601, "Method not found", node.getwalletinfo) self.log.info("Load first wallet") loadwallet_name = node.loadwallet(wallet_names[0]) assert_equal(loadwallet_name['name'], wallet_names[0]) assert_equal(node.listwallets(), wallet_names[0:1]) node.getwalletinfo() w1 = node.get_wallet_rpc(wallet_names[0]) w1.getwalletinfo() self.log.info("Load second wallet") loadwallet_name = node.loadwallet(wallet_names[1]) assert_equal(loadwallet_name['name'], wallet_names[1]) assert_equal(node.listwallets(), wallet_names[0:2]) assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo) w2 = node.get_wallet_rpc(wallet_names[1]) w2.getwalletinfo() self.log.info("Concurrent wallet loading") threads = [] for _ in range(3): n = node.cli if self.options.usecli else get_rpc_proxy( node.url, 1, timeout=600, coveragedir=node.coverage_dir) t = Thread(target=test_load_unload, args=(n, wallet_names[2], )) t.start() threads.append(t) for t in threads: t.join() global got_loading_error assert_equal(got_loading_error, True) self.log.info("Load remaining wallets") for wallet_name in wallet_names[2:]: loadwallet_name = self.nodes[0].loadwallet(wallet_name) assert_equal(loadwallet_name['name'], wallet_name) assert_equal(set(self.nodes[0].listwallets()), set(wallet_names)) # Fail to load if wallet doesn't exist assert_raises_rpc_error(-18, 'Wallet wallets not found.', self.nodes[0].loadwallet, 'wallets') # Fail to load duplicate wallets assert_raises_rpc_error( -4, 'Wallet file verification failed. Error loading wallet w1. Duplicate -wallet filename specified.', self.nodes[0].loadwallet, wallet_names[0]) # Fail to load duplicate wallets by different ways (directory and # filepath) assert_raises_rpc_error( -4, "Wallet file verification failed. Error loading wallet wallet.dat. Duplicate -wallet filename specified.", self.nodes[0].loadwallet, 'wallet.dat') # Fail to load if one wallet is a copy of another - assert_raises_rpc_error(-4, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", + assert_raises_rpc_error(-4, "BerkeleyDatabase: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') # Fail to load if one wallet is a copy of another. # Test this twice to make sure that we don't re-introduce # https://github.com/bitcoin/bitcoin/issues/14304 - assert_raises_rpc_error(-4, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", + assert_raises_rpc_error(-4, "BerkeleyDatabase: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') # Fail to load if wallet file is a symlink if os.name != 'nt': assert_raises_rpc_error( -4, "Wallet file verification failed. Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink') # Fail to load if a directory is specified that doesn't contain a # wallet os.mkdir(wallet_dir('empty_wallet_dir')) assert_raises_rpc_error(-18, "Directory empty_wallet_dir does not contain a wallet.dat file", self.nodes[0].loadwallet, 'empty_wallet_dir') self.log.info("Test dynamic wallet creation.") # Fail to create a wallet if it already exists. assert_raises_rpc_error(-4, "Wallet w2 already exists.", self.nodes[0].createwallet, 'w2') # Successfully create a wallet with a new name loadwallet_name = self.nodes[0].createwallet('w9') assert_equal(loadwallet_name['name'], 'w9') w9 = node.get_wallet_rpc('w9') assert_equal(w9.getwalletinfo()['walletname'], 'w9') assert 'w9' in self.nodes[0].listwallets() # Successfully create a wallet using a full path new_wallet_dir = os.path.join(self.options.tmpdir, 'new_walletdir') new_wallet_name = os.path.join(new_wallet_dir, 'w10') loadwallet_name = self.nodes[0].createwallet(new_wallet_name) assert_equal(loadwallet_name['name'], new_wallet_name) w10 = node.get_wallet_rpc(new_wallet_name) assert_equal(w10.getwalletinfo()['walletname'], new_wallet_name) assert new_wallet_name in self.nodes[0].listwallets() self.log.info("Test dynamic wallet unloading") # Test `unloadwallet` errors assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].unloadwallet) assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", self.nodes[0].unloadwallet, "dummy") assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", node.get_wallet_rpc("dummy").unloadwallet) assert_raises_rpc_error(-8, "Cannot unload the requested wallet", w1.unloadwallet, "w2"), # Successfully unload the specified wallet name self.nodes[0].unloadwallet("w1") assert 'w1' not in self.nodes[0].listwallets() # Successfully unload the wallet referenced by the request endpoint # Also ensure unload works during walletpassphrase timeout w2.encryptwallet('test') w2.walletpassphrase('test', 1) w2.unloadwallet() time.sleep(1.1) assert 'w2' not in self.nodes[0].listwallets() # Successfully unload all wallets for wallet_name in self.nodes[0].listwallets(): self.nodes[0].unloadwallet(wallet_name) assert_equal(self.nodes[0].listwallets(), []) assert_raises_rpc_error(-32601, "Method not found (wallet method is disabled because no wallet is loaded)", self.nodes[0].getwalletinfo) # Successfully load a previously unloaded wallet self.nodes[0].loadwallet('w1') assert_equal(self.nodes[0].listwallets(), ['w1']) assert_equal(w1.getwalletinfo()['walletname'], 'w1') assert_equal(sorted(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), [ '', os.path.join('sub', 'w5'), 'w', 'w1', 'w2', 'w3', 'w7', 'w7_symlink', 'w8', 'w8_copy', 'w9']) # Test backing up and restoring wallets self.log.info("Test wallet backup") self.restart_node(0, ['-nowallet']) for wallet_name in wallet_names: self.nodes[0].loadwallet(wallet_name) for wallet_name in wallet_names: rpc = self.nodes[0].get_wallet_rpc(wallet_name) addr = rpc.getnewaddress() backup = os.path.join(self.options.tmpdir, 'backup.dat') rpc.backupwallet(backup) self.nodes[0].unloadwallet(wallet_name) shutil.copyfile(empty_wallet, wallet_file(wallet_name)) self.nodes[0].loadwallet(wallet_name) assert_equal(rpc.getaddressinfo(addr)['ismine'], False) self.nodes[0].unloadwallet(wallet_name) shutil.copyfile(backup, wallet_file(wallet_name)) self.nodes[0].loadwallet(wallet_name) assert_equal(rpc.getaddressinfo(addr)['ismine'], True) # Test .walletlock file is closed self.start_node(1) wallet = os.path.join(self.options.tmpdir, 'my_wallet') self.nodes[0].createwallet(wallet) assert_raises_rpc_error(-4, "Error initializing wallet database environment", self.nodes[1].loadwallet, wallet) self.nodes[0].unloadwallet(wallet) self.nodes[1].loadwallet(wallet) if __name__ == '__main__': MultiWalletTest().main()