diff --git a/doc/release-notes.md b/doc/release-notes.md
index 41b1aa623..24683594d 100644
--- a/doc/release-notes.md
+++ b/doc/release-notes.md
@@ -1,5 +1,8 @@
Bitcoin ABC version 0.20.9 is now available from:
This release includes the following features and fixes:
+ - Fixed a bug with Multiwallets that have their own directories (i.e. cases
+ such as `DATADIR/wallets/mywallet/wallet.dat`). Backups of these wallets
+ will now take each wallet's specific directory into account.
diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp
index 7711b002e..c26aa6465 100644
--- a/src/wallet/db.cpp
+++ b/src/wallet/db.cpp
@@ -1,876 +1,876 @@
// Copyright (c) 2009-2010 Satoshi Nakamoto
// Copyright (c) 2009-2016 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include
#include
#include
#include
#include
#include
#include
#include
#include // boost::this_thread::interruption_point() (mingw)
#include
#ifndef WIN32
#include
#endif
namespace {
//! Make sure database has a unique fileid within the environment. If it
//! doesn't, throw an error. BDB caches do not work properly when more than one
//! open database has the same fileid (values written to one database may show
//! up in reads to other databases).
//!
//! BerkeleyDB generates unique fileids by default
//! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html),
//! so bitcoin should never create different databases with the same fileid, but
//! this error can be triggered if users manually copy database files.
void CheckUniqueFileid(const BerkeleyEnvironment &env,
const std::string &filename, Db &db) {
if (env.IsMock()) {
return;
}
u_int8_t fileid[DB_FILE_ID_LEN];
int ret = db.get_mpf()->get_fileid(fileid);
if (ret != 0) {
throw std::runtime_error(strprintf(
"BerkeleyBatch: Can't open database %s (get_fileid failed with %d)",
filename, ret));
}
for (const auto &item : env.mapDb) {
u_int8_t item_fileid[DB_FILE_ID_LEN];
if (item.second &&
item.second->get_mpf()->get_fileid(item_fileid) == 0 &&
memcmp(fileid, item_fileid, sizeof(fileid)) == 0) {
const char *item_filename = nullptr;
item.second->get_dbname(&item_filename, nullptr);
throw std::runtime_error(strprintf(
"BerkeleyBatch: Can't open database %s (duplicates fileid %s "
"from %s)",
filename,
HexStr(std::begin(item_fileid), std::end(item_fileid)),
item_filename ? item_filename : "(unknown database)"));
}
}
}
CCriticalSection cs_db;
//!< Map from directory name to open db environment.
std::map g_dbenvs;
} // namespace
BerkeleyEnvironment *GetWalletEnv(const fs::path &wallet_path,
std::string &database_filename) {
fs::path env_directory;
if (fs::is_regular_file(wallet_path)) {
// Special case for backwards compatibility: if wallet path points to an
// existing file, treat it as the path to a BDB data file in a parent
// directory that also contains BDB log files.
env_directory = wallet_path.parent_path();
database_filename = wallet_path.filename().string();
} else {
// Normal case: Interpret wallet path as a directory path containing
// data and log files.
env_directory = wallet_path;
database_filename = "wallet.dat";
}
LOCK(cs_db);
// Note: An ununsed temporary BerkeleyEnvironment object may be created
// inside the emplace function if the key already exists. This is a little
// inefficient, but not a big concern since the map will be changed in the
// future to hold pointers instead of objects, anyway.
return &g_dbenvs
.emplace(std::piecewise_construct,
std::forward_as_tuple(env_directory.string()),
std::forward_as_tuple(env_directory))
.first->second;
}
//
// BerkeleyBatch
//
void BerkeleyEnvironment::Close() {
if (!fDbEnvInit) {
return;
}
fDbEnvInit = false;
for (auto &db : mapDb) {
auto count = mapFileUseCount.find(db.first);
assert(count == mapFileUseCount.end() || count->second == 0);
if (db.second) {
db.second->close(0);
delete db.second;
db.second = nullptr;
}
}
int ret = dbenv->close(0);
if (ret != 0) {
LogPrintf("BerkeleyEnvironment::EnvShutdown: Error %d shutting down "
"database environment: %s\n",
ret, DbEnv::strerror(ret));
}
if (!fMockDb) {
DbEnv(u_int32_t(0)).remove(strPath.c_str(), 0);
}
}
void BerkeleyEnvironment::Reset() {
dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS));
fDbEnvInit = false;
fMockDb = false;
}
BerkeleyEnvironment::BerkeleyEnvironment(const fs::path &dir_path)
: strPath(dir_path.string()) {
Reset();
}
BerkeleyEnvironment::~BerkeleyEnvironment() {
Close();
}
bool BerkeleyEnvironment::Open(bool retry) {
if (fDbEnvInit) {
return true;
}
boost::this_thread::interruption_point();
fs::path pathIn = strPath;
TryCreateDirectories(pathIn);
if (!LockDirectory(pathIn, ".walletlock")) {
LogPrintf("Cannot obtain a lock on wallet directory %s. Another "
"instance of bitcoin may be using it.\n",
strPath);
return false;
}
fs::path pathLogDir = pathIn / "database";
TryCreateDirectories(pathLogDir);
fs::path pathErrorFile = pathIn / "db.log";
LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n",
pathLogDir.string(), pathErrorFile.string());
unsigned int nEnvFlags = 0;
if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) {
nEnvFlags |= DB_PRIVATE;
}
dbenv->set_lg_dir(pathLogDir.string().c_str());
// 1 MiB should be enough for just the wallet
dbenv->set_cachesize(0, 0x100000, 1);
dbenv->set_lg_bsize(0x10000);
dbenv->set_lg_max(1048576);
dbenv->set_lk_max_locks(40000);
dbenv->set_lk_max_objects(40000);
/// debug
dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a"));
dbenv->set_flags(DB_AUTO_COMMIT, 1);
dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1);
dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1);
int ret =
dbenv->open(strPath.c_str(),
DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL |
DB_INIT_TXN | DB_THREAD | DB_RECOVER | nEnvFlags,
S_IRUSR | S_IWUSR);
if (ret != 0) {
dbenv->close(0);
LogPrintf("BerkeleyEnvironment::Open: Error %d opening database "
"environment: %s\n",
ret, DbEnv::strerror(ret));
if (retry) {
// try moving the database env out of the way
fs::path pathDatabaseBak =
pathIn / strprintf("database.%d.bak", GetTime());
try {
fs::rename(pathLogDir, pathDatabaseBak);
LogPrintf("Moved old %s to %s. Retrying.\n",
pathLogDir.string(), pathDatabaseBak.string());
} catch (const fs::filesystem_error &) {
// failure is ok (well, not really, but it's not worse than what
// we started with)
}
// try opening it again one more time
if (!Open(false /* retry */)) {
// if it still fails, it probably means we can't even create the
// database env
return false;
}
} else {
return false;
}
}
fDbEnvInit = true;
fMockDb = false;
return true;
}
void BerkeleyEnvironment::MakeMock() {
if (fDbEnvInit) {
throw std::runtime_error(
"BerkeleyEnvironment::MakeMock: Already initialized");
}
boost::this_thread::interruption_point();
LogPrint(BCLog::DB, "BerkeleyEnvironment::MakeMock\n");
dbenv->set_cachesize(1, 0, 1);
dbenv->set_lg_bsize(10485760 * 4);
dbenv->set_lg_max(10485760);
dbenv->set_lk_max_locks(10000);
dbenv->set_lk_max_objects(10000);
dbenv->set_flags(DB_AUTO_COMMIT, 1);
dbenv->log_set_config(DB_LOG_IN_MEMORY, 1);
int ret =
dbenv->open(nullptr,
DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL |
DB_INIT_TXN | DB_THREAD | DB_PRIVATE,
S_IRUSR | S_IWUSR);
if (ret > 0) {
throw std::runtime_error(
strprintf("BerkeleyEnvironment::MakeMock: Error %d opening "
"database environment.",
ret));
}
fDbEnvInit = true;
fMockDb = true;
}
BerkeleyEnvironment::VerifyResult
BerkeleyEnvironment::Verify(const std::string &strFile,
recoverFunc_type recoverFunc,
std::string &out_backup_filename) {
LOCK(cs_db);
assert(mapFileUseCount.count(strFile) == 0);
Db db(dbenv.get(), 0);
int result = db.verify(strFile.c_str(), nullptr, nullptr, 0);
if (result == 0) {
return VerifyResult::VERIFY_OK;
} else if (recoverFunc == nullptr) {
return VerifyResult::RECOVER_FAIL;
}
// Try to recover:
bool fRecovered =
(*recoverFunc)(fs::path(strPath) / strFile, out_backup_filename);
return (fRecovered ? VerifyResult::RECOVER_OK : VerifyResult::RECOVER_FAIL);
}
bool BerkeleyBatch::Recover(const fs::path &file_path, void *callbackDataIn,
bool (*recoverKVcallback)(void *callbackData,
CDataStream ssKey,
CDataStream ssValue),
std::string &newFilename) {
std::string filename;
BerkeleyEnvironment *env = GetWalletEnv(file_path, filename);
// Recovery procedure:
// Move wallet file to walletfilename.timestamp.bak
// Call Salvage with fAggressive=true to get as much data as possible.
// Rewrite salvaged data to fresh wallet file.
// Set -rescan so any missing transactions will be found.
int64_t now = GetTime();
newFilename = strprintf("%s.%d.bak", filename, now);
int result = env->dbenv->dbrename(nullptr, filename.c_str(), nullptr,
newFilename.c_str(), DB_AUTO_COMMIT);
if (result == 0) {
LogPrintf("Renamed %s to %s\n", filename, newFilename);
} else {
LogPrintf("Failed to rename %s to %s\n", filename, newFilename);
return false;
}
std::vector salvagedData;
bool fSuccess = env->Salvage(newFilename, true, salvagedData);
if (salvagedData.empty()) {
LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename);
return false;
}
LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size());
std::unique_ptr pdbCopy = std::make_unique(env->dbenv.get(), 0);
int ret = pdbCopy->open(nullptr, // Txn pointer
filename.c_str(), // Filename
"main", // Logical db name
DB_BTREE, // Database type
DB_CREATE, // Flags
0);
if (ret > 0) {
LogPrintf("Cannot create database file %s\n", filename);
pdbCopy->close(0);
return false;
}
DbTxn *ptxn = env->TxnBegin();
for (BerkeleyEnvironment::KeyValPair &row : salvagedData) {
if (recoverKVcallback) {
CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION);
CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION);
if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue)) {
continue;
}
}
Dbt datKey(&row.first[0], row.first.size());
Dbt datValue(&row.second[0], row.second.size());
int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE);
if (ret2 > 0) {
fSuccess = false;
}
}
ptxn->commit(0);
pdbCopy->close(0);
return fSuccess;
}
bool BerkeleyBatch::VerifyEnvironment(const fs::path &file_path,
std::string &errorStr) {
std::string walletFile;
BerkeleyEnvironment *env = GetWalletEnv(file_path, walletFile);
fs::path walletDir = env->Directory();
LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0));
LogPrintf("Using wallet %s\n", walletFile);
// Wallet file must be a plain filename without a directory
if (walletFile != fs::basename(walletFile) + fs::extension(walletFile)) {
errorStr = strprintf(_("Wallet %s resides outside wallet directory %s"),
walletFile, walletDir.string());
return false;
}
if (!env->Open(true /* retry */)) {
errorStr = strprintf(
_("Error initializing wallet database environment %s!"), walletDir);
return false;
}
return true;
}
bool BerkeleyBatch::VerifyDatabaseFile(
const fs::path &file_path, std::string &warningStr, std::string &errorStr,
BerkeleyEnvironment::recoverFunc_type recoverFunc) {
std::string walletFile;
BerkeleyEnvironment *env = GetWalletEnv(file_path, walletFile);
fs::path walletDir = env->Directory();
if (fs::exists(walletDir / walletFile)) {
std::string backup_filename;
BerkeleyEnvironment::VerifyResult r =
env->Verify(walletFile, recoverFunc, backup_filename);
if (r == BerkeleyEnvironment::VerifyResult::RECOVER_OK) {
warningStr = strprintf(
_("Warning: Wallet file corrupt, data salvaged! Original %s "
"saved as %s in %s; if your balance or transactions are "
"incorrect you should restore from a backup."),
walletFile, backup_filename, walletDir);
}
if (r == BerkeleyEnvironment::VerifyResult::RECOVER_FAIL) {
errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile);
return false;
}
}
// also return true if files does not exists
return true;
}
/* End of headers, beginning of key/value data */
static const char *HEADER_END = "HEADER=END";
/* End of key/value data */
static const char *DATA_END = "DATA=END";
bool BerkeleyEnvironment::Salvage(
const std::string &strFile, bool fAggressive,
std::vector &vResult) {
LOCK(cs_db);
assert(mapFileUseCount.count(strFile) == 0);
u_int32_t flags = DB_SALVAGE;
if (fAggressive) {
flags |= DB_AGGRESSIVE;
}
std::stringstream strDump;
Db db(dbenv.get(), 0);
int result = db.verify(strFile.c_str(), nullptr, &strDump, flags);
if (result == DB_VERIFY_BAD) {
LogPrintf("BerkeleyEnvironment::Salvage: Database salvage found "
"errors, all data may not be recoverable.\n");
if (!fAggressive) {
LogPrintf("BerkeleyEnvironment::Salvage: Rerun with aggressive "
"mode to ignore errors and continue.\n");
return false;
}
}
if (result != 0 && result != DB_VERIFY_BAD) {
LogPrintf("BerkeleyEnvironment::Salvage: Database salvage failed with "
"result %d.\n",
result);
return false;
}
// Format of bdb dump is ascii lines:
// header lines...
// HEADER=END
// hexadecimal key
// hexadecimal value
// ... repeated
// DATA=END
std::string strLine;
while (!strDump.eof() && strLine != HEADER_END) {
// Skip past header
getline(strDump, strLine);
}
std::string keyHex, valueHex;
while (!strDump.eof() && keyHex != DATA_END) {
getline(strDump, keyHex);
if (keyHex != DATA_END) {
if (strDump.eof()) {
break;
}
getline(strDump, valueHex);
if (valueHex == DATA_END) {
LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Number of "
"keys in data does not match number of values.\n");
break;
}
vResult.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex)));
}
}
if (keyHex != DATA_END) {
LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Unexpected end of "
"file while reading salvage output.\n");
return false;
}
return (result == 0);
}
void BerkeleyEnvironment::CheckpointLSN(const std::string &strFile) {
dbenv->txn_checkpoint(0, 0, 0);
if (fMockDb) {
return;
}
dbenv->lsn_reset(strFile.c_str(), 0);
}
BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase &database, const char *pszMode,
bool fFlushOnCloseIn)
: pdb(nullptr), activeTxn(nullptr) {
fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
fFlushOnClose = fFlushOnCloseIn;
env = database.env;
if (database.IsDummy()) {
return;
}
const std::string &strFilename = database.strFile;
bool fCreate = strchr(pszMode, 'c') != nullptr;
unsigned int nFlags = DB_THREAD;
if (fCreate) {
nFlags |= DB_CREATE;
}
{
LOCK(cs_db);
if (!env->Open(false /* retry */)) {
throw std::runtime_error(
"BerkeleyBatch: Failed to open database environment.");
}
pdb = env->mapDb[strFilename];
if (pdb == nullptr) {
int ret;
std::unique_ptr pdb_temp =
std::make_unique(env->dbenv.get(), 0);
bool fMockDb = env->IsMock();
if (fMockDb) {
DbMpoolFile *mpf = pdb_temp->get_mpf();
ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
if (ret != 0) {
throw std::runtime_error(
strprintf("BerkeleyBatch: Failed to configure for no "
"temp file backing for database %s",
strFilename));
}
}
ret = pdb_temp->open(
nullptr, // Txn pointer
fMockDb ? nullptr : strFilename.c_str(), // Filename
fMockDb ? strFilename.c_str() : "main", // Logical db name
DB_BTREE, // Database type
nFlags, // Flags
0);
if (ret != 0) {
throw std::runtime_error(
strprintf("BerkeleyBatch: Error %d, can't open database %s",
ret, strFilename));
}
// Call CheckUniqueFileid on the containing BDB environment to
// avoid BDB data consistency bugs that happen when different data
// files in the same environment have the same fileid.
//
// Also call CheckUniqueFileid on all the other g_dbenvs to prevent
// bitcoin from opening the same data file through another
// environment when the file is referenced through equivalent but
// not obviously identical symlinked or hard linked or bind mounted
// paths. In the future a more relaxed check for equal inode and
// device ids could be done instead, which would allow opening
// different backup copies of a wallet at the same time. Maybe even
// more ideally, an exclusive lock for accessing the database could
// be implemented, so no equality checks are needed at all. (Newer
// versions of BDB have an set_lk_exclusive method for this
// purpose, but the older version we use does not.)
for (const auto &dbenv : g_dbenvs) {
CheckUniqueFileid(dbenv.second, strFilename, *pdb_temp);
}
pdb = pdb_temp.release();
env->mapDb[strFilename] = pdb;
if (fCreate && !Exists(std::string("version"))) {
bool fTmp = fReadOnly;
fReadOnly = false;
WriteVersion(CLIENT_VERSION);
fReadOnly = fTmp;
}
}
++env->mapFileUseCount[strFilename];
strFile = strFilename;
}
}
void BerkeleyBatch::Flush() {
if (activeTxn) {
return;
}
// Flush database activity from memory pool to disk log
unsigned int nMinutes = 0;
if (fReadOnly) {
nMinutes = 1;
}
env->dbenv->txn_checkpoint(
nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024
: 0,
nMinutes, 0);
}
void BerkeleyDatabase::IncrementUpdateCounter() {
++nUpdateCounter;
}
void BerkeleyBatch::Close() {
if (!pdb) {
return;
}
if (activeTxn) {
activeTxn->abort();
}
activeTxn = nullptr;
pdb = nullptr;
if (fFlushOnClose) {
Flush();
}
LOCK(cs_db);
--env->mapFileUseCount[strFile];
}
void BerkeleyEnvironment::CloseDb(const std::string &strFile) {
LOCK(cs_db);
if (mapDb[strFile] != nullptr) {
// Close the database handle
Db *pdb = mapDb[strFile];
pdb->close(0);
delete pdb;
mapDb[strFile] = nullptr;
}
}
bool BerkeleyBatch::Rewrite(BerkeleyDatabase &database, const char *pszSkip) {
if (database.IsDummy()) {
return true;
}
BerkeleyEnvironment *env = database.env;
const std::string &strFile = database.strFile;
while (true) {
{
LOCK(cs_db);
if (!env->mapFileUseCount.count(strFile) ||
env->mapFileUseCount[strFile] == 0) {
// Flush log data to the dat file
env->CloseDb(strFile);
env->CheckpointLSN(strFile);
env->mapFileUseCount.erase(strFile);
bool fSuccess = true;
LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile);
std::string strFileRes = strFile + ".rewrite";
{
// surround usage of db with extra {}
BerkeleyBatch db(database, "r");
std::unique_ptr pdbCopy =
std::make_unique(env->dbenv.get(), 0);
int ret = pdbCopy->open(nullptr, // Txn pointer
strFileRes.c_str(), // Filename
"main", // Logical db name
DB_BTREE, // Database type
DB_CREATE, // Flags
0);
if (ret > 0) {
LogPrintf("BerkeleyBatch::Rewrite: Can't create "
"database file %s\n",
strFileRes);
fSuccess = false;
}
Dbc *pcursor = db.GetCursor();
if (pcursor)
while (fSuccess) {
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
CDataStream ssValue(SER_DISK, CLIENT_VERSION);
int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue);
if (ret1 == DB_NOTFOUND) {
pcursor->close();
break;
}
if (ret1 != 0) {
pcursor->close();
fSuccess = false;
break;
}
if (pszSkip &&
strncmp(ssKey.data(), pszSkip,
std::min(ssKey.size(),
strlen(pszSkip))) == 0) {
continue;
}
if (strncmp(ssKey.data(), "\x07version", 8) == 0) {
// Update version:
ssValue.clear();
ssValue << CLIENT_VERSION;
}
Dbt datKey(ssKey.data(), ssKey.size());
Dbt datValue(ssValue.data(), ssValue.size());
int ret2 = pdbCopy->put(nullptr, &datKey, &datValue,
DB_NOOVERWRITE);
if (ret2 > 0) {
fSuccess = false;
}
}
if (fSuccess) {
db.Close();
env->CloseDb(strFile);
if (pdbCopy->close(0)) {
fSuccess = false;
}
} else {
pdbCopy->close(0);
}
}
if (fSuccess) {
Db dbA(env->dbenv.get(), 0);
if (dbA.remove(strFile.c_str(), nullptr, 0)) {
fSuccess = false;
}
Db dbB(env->dbenv.get(), 0);
if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(),
0)) {
fSuccess = false;
}
}
if (!fSuccess) {
LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite "
"database file %s\n",
strFileRes);
}
return fSuccess;
}
}
MilliSleep(100);
}
}
void BerkeleyEnvironment::Flush(bool fShutdown) {
int64_t nStart = GetTimeMillis();
// Flush log data to the actual data file on all files that are not in use
LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: Flush(%s)%s\n",
fShutdown ? "true" : "false",
fDbEnvInit ? "" : " database not started");
if (!fDbEnvInit) {
return;
}
{
LOCK(cs_db);
std::map::iterator mi = mapFileUseCount.begin();
while (mi != mapFileUseCount.end()) {
std::string strFile = (*mi).first;
int nRefCount = (*mi).second;
LogPrint(
BCLog::DB,
"BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n",
strFile, nRefCount);
if (nRefCount == 0) {
// Move log data to the dat file
CloseDb(strFile);
LogPrint(BCLog::DB,
"BerkeleyEnvironment::Flush: %s checkpoint\n",
strFile);
dbenv->txn_checkpoint(0, 0, 0);
LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s detach\n",
strFile);
if (!fMockDb) {
dbenv->lsn_reset(strFile.c_str(), 0);
}
LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s closed\n",
strFile);
mapFileUseCount.erase(mi++);
} else {
mi++;
}
}
LogPrint(BCLog::DB,
"BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n",
fShutdown ? "true" : "false",
fDbEnvInit ? "" : " database not started",
GetTimeMillis() - nStart);
if (fShutdown) {
char **listp;
if (mapFileUseCount.empty()) {
dbenv->log_archive(&listp, DB_ARCH_REMOVE);
Close();
if (!fMockDb) {
fs::remove_all(fs::path(strPath) / "database");
}
g_dbenvs.erase(strPath);
}
}
}
}
bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase &database) {
if (database.IsDummy()) {
return true;
}
bool ret = false;
BerkeleyEnvironment *env = database.env;
const std::string &strFile = database.strFile;
TRY_LOCK(cs_db, lockDb);
if (lockDb) {
// Don't do this if any databases are in use
int nRefCount = 0;
std::map::iterator mit = env->mapFileUseCount.begin();
while (mit != env->mapFileUseCount.end()) {
nRefCount += (*mit).second;
mit++;
}
if (nRefCount == 0) {
boost::this_thread::interruption_point();
std::map::iterator mi =
env->mapFileUseCount.find(strFile);
if (mi != env->mapFileUseCount.end()) {
LogPrint(BCLog::DB, "Flushing %s\n", strFile);
int64_t nStart = GetTimeMillis();
// Flush wallet file so it's self contained
env->CloseDb(strFile);
env->CheckpointLSN(strFile);
env->mapFileUseCount.erase(mi++);
LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile,
GetTimeMillis() - nStart);
ret = true;
}
}
}
return ret;
}
bool BerkeleyDatabase::Rewrite(const char *pszSkip) {
return BerkeleyBatch::Rewrite(*this, pszSkip);
}
bool BerkeleyDatabase::Backup(const std::string &strDest) {
if (IsDummy()) {
return false;
}
while (true) {
{
LOCK(cs_db);
if (!env->mapFileUseCount.count(strFile) ||
env->mapFileUseCount[strFile] == 0) {
// Flush log data to the dat file
env->CloseDb(strFile);
env->CheckpointLSN(strFile);
env->mapFileUseCount.erase(strFile);
// Copy wallet file.
- fs::path pathSrc = GetWalletDir() / strFile;
+ fs::path pathSrc = env->Directory() / strFile;
fs::path pathDest(strDest);
if (fs::is_directory(pathDest)) {
pathDest /= strFile;
}
try {
if (fs::equivalent(pathSrc, pathDest)) {
LogPrintf("cannot backup to wallet source file %s\n",
pathDest.string());
return false;
}
fs::copy_file(pathSrc, pathDest,
fs::copy_option::overwrite_if_exists);
LogPrintf("copied %s to %s\n", strFile, pathDest.string());
return true;
} catch (const fs::filesystem_error &e) {
LogPrintf("error copying %s to %s - %s\n", strFile,
pathDest.string(), e.what());
return false;
}
}
}
MilliSleep(100);
}
}
void BerkeleyDatabase::Flush(bool shutdown) {
if (!IsDummy()) {
env->Flush(shutdown);
if (shutdown) {
env = nullptr;
}
}
}
diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py
index bc96948ea..a0f680ecc 100755
--- a/test/functional/wallet_multiwallet.py
+++ b/test/functional/wallet_multiwallet.py
@@ -1,288 +1,314 @@
#!/usr/bin/env python3
# Copyright (c) 2017-2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test multiwallet.
Verify that a bitcoind node can load multiple wallet files
"""
import os
import shutil
from test_framework.test_framework import BitcoinTestFramework
from test_framework.test_node import ErrorMatch
from test_framework.util import (
assert_equal,
assert_raises_rpc_error,
)
class MultiWalletTest(BitcoinTestFramework):
def set_test_params(self):
self.setup_clean_chain = True
self.num_nodes = 2
self.supports_cli = True
def run_test(self):
node = self.nodes[0]
data_dir = lambda *p: os.path.join(node.datadir, 'regtest', *p)
wallet_dir = lambda *p: data_dir('wallets', *p)
def wallet(name): return node.get_wallet_rpc(name)
+ def wallet_file(name):
+ if os.path.isdir(wallet_dir(name)):
+ return wallet_dir(name, "wallet.dat")
+ return wallet_dir(name)
+
# check wallet.dat is created
self.stop_nodes()
assert_equal(os.path.isfile(wallet_dir('wallet.dat')), True)
# create symlink to verify wallet directory path can be referenced
# through symlink
os.mkdir(wallet_dir('w7'))
os.symlink('w7', wallet_dir('w7_symlink'))
# rename wallet.dat to make sure plain wallet file paths (as opposed to
# directory paths) can be loaded
os.rename(wallet_dir("wallet.dat"), wallet_dir("w8"))
+ # create another dummy wallet for use in testing backups later
+ self.start_node(0, [])
+ self.stop_nodes()
+ empty_wallet = os.path.join(self.options.tmpdir, 'empty.dat')
+ os.rename(wallet_dir("wallet.dat"), empty_wallet)
+
# restart node with a mix of wallet names:
# w1, w2, w3 - to verify new wallets created when non-existing paths specified
# w - to verify wallet name matching works when one wallet path is prefix of another
# sub/w5 - to verify relative wallet path is created correctly
# extern/w6 - to verify absolute wallet path is created correctly
# w7_symlink - to verify symlinked wallet path is initialized correctly
# w8 - to verify existing wallet file is loaded correctly
# '' - to verify default wallet file is created correctly
wallet_names = ['w1', 'w2', 'w3', 'w', 'sub/w5',
os.path.join(self.options.tmpdir, 'extern/w6'), 'w7_symlink', 'w8', '']
extra_args = ['-wallet={}'.format(n) for n in wallet_names]
self.start_node(0, extra_args)
assert_equal(set(node.listwallets()), set(wallet_names))
# check that all requested wallets were created
self.stop_node(0)
for wallet_name in wallet_names:
- if os.path.isdir(wallet_dir(wallet_name)):
- assert_equal(os.path.isfile(
- wallet_dir(wallet_name, "wallet.dat")), True)
- else:
- assert_equal(os.path.isfile(wallet_dir(wallet_name)), True)
+ assert_equal(os.path.isfile(wallet_file(wallet_name)), True)
# should not initialize if wallet path can't be created
exp_stderr = "boost::filesystem::create_directory: (The system cannot find the path specified|Not a directory):"
self.nodes[0].assert_start_raises_init_error(
['-wallet=wallet.dat/bad'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
self.nodes[0].assert_start_raises_init_error(
['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" does not exist')
self.nodes[0].assert_start_raises_init_error(
['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" is a relative path', cwd=data_dir())
self.nodes[0].assert_start_raises_init_error(
['-walletdir=debug.log'], 'Error: Specified -walletdir "debug.log" is not a directory', cwd=data_dir())
# should not initialize if there are duplicate wallets
self.nodes[0].assert_start_raises_init_error(
['-wallet=w1', '-wallet=w1'], 'Error: Error loading wallet w1. Duplicate -wallet filename specified.')
# should not initialize if one wallet is a copy of another
shutil.copyfile(wallet_dir('w8'), wallet_dir('w8_copy'))
exp_stderr = r"BerkeleyBatch: Can't open database w8_copy \(duplicates fileid \w+ from w8\)"
self.nodes[0].assert_start_raises_init_error(
['-wallet=w8', '-wallet=w8_copy'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
# should not initialize if wallet file is a symlink
os.symlink('w8', wallet_dir('w8_symlink'))
self.nodes[0].assert_start_raises_init_error(
['-wallet=w8_symlink'], r'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX)
# should not initialize if the specified walletdir does not exist
self.nodes[0].assert_start_raises_init_error(
['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist')
# should not initialize if the specified walletdir is not a directory
not_a_dir = wallet_dir('notadir')
open(not_a_dir, 'a', encoding="utf8").close()
self.nodes[0].assert_start_raises_init_error(
['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory')
# if wallets/ doesn't exist, datadir should be the default wallet dir
wallet_dir2 = data_dir('walletdir')
os.rename(wallet_dir(), wallet_dir2)
self.start_node(0, ['-wallet=w4', '-wallet=w5'])
assert_equal(set(node.listwallets()), {"w4", "w5"})
w5 = wallet("w5")
w5.generate(1)
# now if wallets/ exists again, but the rootdir is specified as the walletdir, w4 and w5 should still be loaded
os.rename(wallet_dir2, wallet_dir())
self.restart_node(0, ['-wallet=w4', '-wallet=w5',
'-walletdir=' + data_dir()])
assert_equal(set(node.listwallets()), {"w4", "w5"})
w5 = wallet("w5")
w5_info = w5.getwalletinfo()
assert_equal(w5_info['immature_balance'], 50)
competing_wallet_dir = os.path.join(
self.options.tmpdir, 'competing_walletdir')
os.mkdir(competing_wallet_dir)
self.restart_node(0, ['-walletdir=' + competing_wallet_dir])
exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\"!"
self.nodes[1].assert_start_raises_init_error(
['-walletdir=' + competing_wallet_dir], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
self.restart_node(0, extra_args)
wallets = [wallet(w) for w in wallet_names]
wallet_bad = wallet("bad")
# check wallet names and balances
wallets[0].generate(1)
for wallet_name, wallet in zip(wallet_names, wallets):
info = wallet.getwalletinfo()
assert_equal(info['immature_balance'],
50 if wallet is wallets[0] else 0)
assert_equal(info['walletname'], wallet_name)
# accessing invalid wallet fails
assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded",
wallet_bad.getwalletinfo)
# accessing wallet RPC without using wallet endpoint fails
assert_raises_rpc_error(-19, "Wallet file not specified (must request wallet RPC through /wallet/ uri-path).",
node.getwalletinfo)
w1, w2, w3, w4, *_ = wallets
w1.generate(101)
assert_equal(w1.getbalance(), 100)
assert_equal(w2.getbalance(), 0)
assert_equal(w3.getbalance(), 0)
assert_equal(w4.getbalance(), 0)
w1.sendtoaddress(w2.getnewaddress(), 1)
w1.sendtoaddress(w3.getnewaddress(), 2)
w1.sendtoaddress(w4.getnewaddress(), 3)
w1.generate(1)
assert_equal(w2.getbalance(), 1)
assert_equal(w3.getbalance(), 2)
assert_equal(w4.getbalance(), 3)
batch = w1.batch([w1.getblockchaininfo.get_request(),
w1.getwalletinfo.get_request()])
assert_equal(batch[0]["result"]["chain"], "regtest")
assert_equal(batch[1]["result"]["walletname"], "w1")
self.log.info('Check for per-wallet settxfee call')
assert_equal(w1.getwalletinfo()['paytxfee'], 0)
assert_equal(w2.getwalletinfo()['paytxfee'], 0)
w2.settxfee(4.0)
assert_equal(w1.getwalletinfo()['paytxfee'], 0)
assert_equal(w2.getwalletinfo()['paytxfee'], 4.0)
self.log.info("Test dynamic wallet loading")
self.restart_node(0, ['-nowallet'])
assert_equal(node.listwallets(), [])
assert_raises_rpc_error(-32601, "Method not found", node.getwalletinfo)
self.log.info("Load first wallet")
loadwallet_name = node.loadwallet(wallet_names[0])
assert_equal(loadwallet_name['name'], wallet_names[0])
assert_equal(node.listwallets(), wallet_names[0:1])
node.getwalletinfo()
w1 = node.get_wallet_rpc(wallet_names[0])
w1.getwalletinfo()
self.log.info("Load second wallet")
loadwallet_name = node.loadwallet(wallet_names[1])
assert_equal(loadwallet_name['name'], wallet_names[1])
assert_equal(node.listwallets(), wallet_names[0:2])
assert_raises_rpc_error(-19,
"Wallet file not specified", node.getwalletinfo)
w2 = node.get_wallet_rpc(wallet_names[1])
w2.getwalletinfo()
self.log.info("Load remaining wallets")
for wallet_name in wallet_names[2:]:
loadwallet_name = self.nodes[0].loadwallet(wallet_name)
assert_equal(loadwallet_name['name'], wallet_name)
assert_equal(set(self.nodes[0].listwallets()), set(wallet_names))
# Fail to load if wallet doesn't exist
assert_raises_rpc_error(-18, 'Wallet wallets not found.',
self.nodes[0].loadwallet, 'wallets')
# Fail to load duplicate wallets
assert_raises_rpc_error(-4, 'Wallet file verification failed: Error loading wallet w1. Duplicate -wallet filename specified.',
self.nodes[0].loadwallet, wallet_names[0])
# Fail to load if one wallet is a copy of another
assert_raises_rpc_error(-1, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid",
self.nodes[0].loadwallet, 'w8_copy')
# Fail to load if wallet file is a symlink
assert_raises_rpc_error(-4, "Wallet file verification failed: Invalid -wallet path 'w8_symlink'",
self.nodes[0].loadwallet, 'w8_symlink')
# Fail to load if a directory is specified that doesn't contain a wallet
os.mkdir(wallet_dir('empty_wallet_dir'))
assert_raises_rpc_error(-18, "Directory empty_wallet_dir does not contain a wallet.dat file",
self.nodes[0].loadwallet, 'empty_wallet_dir')
self.log.info("Test dynamic wallet creation.")
# Fail to create a wallet if it already exists.
assert_raises_rpc_error(-4, "Wallet w2 already exists.",
self.nodes[0].createwallet, 'w2')
# Successfully create a wallet with a new name
loadwallet_name = self.nodes[0].createwallet('w9')
assert_equal(loadwallet_name['name'], 'w9')
w9 = node.get_wallet_rpc('w9')
assert_equal(w9.getwalletinfo()['walletname'], 'w9')
assert 'w9' in self.nodes[0].listwallets()
# Successfully create a wallet using a full path
new_wallet_dir = os.path.join(self.options.tmpdir, 'new_walletdir')
new_wallet_name = os.path.join(new_wallet_dir, 'w10')
loadwallet_name = self.nodes[0].createwallet(new_wallet_name)
assert_equal(loadwallet_name['name'], new_wallet_name)
w10 = node.get_wallet_rpc(new_wallet_name)
assert_equal(w10.getwalletinfo()['walletname'], new_wallet_name)
assert new_wallet_name in self.nodes[0].listwallets()
self.log.info("Test dynamic wallet unloading")
# Test `unloadwallet` errors
assert_raises_rpc_error(-1, "JSON value is not a string as expected",
self.nodes[0].unloadwallet)
assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded",
self.nodes[0].unloadwallet, "dummy")
assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded",
node.get_wallet_rpc("dummy").unloadwallet)
assert_raises_rpc_error(-8, "Cannot unload the requested wallet",
w1.unloadwallet, "w2"),
# Successfully unload the specified wallet name
self.nodes[0].unloadwallet("w1")
assert 'w1' not in self.nodes[0].listwallets()
# Successfully unload the wallet referenced by the request endpoint
w2.unloadwallet()
assert 'w2' not in self.nodes[0].listwallets()
# Successfully unload all wallets
for wallet_name in self.nodes[0].listwallets():
self.nodes[0].unloadwallet(wallet_name)
assert_equal(self.nodes[0].listwallets(), [])
assert_raises_rpc_error(-32601, "Method not found (wallet method is disabled because no wallet is loaded)",
self.nodes[0].getwalletinfo)
# Successfully load a previously unloaded wallet
self.nodes[0].loadwallet('w1')
assert_equal(self.nodes[0].listwallets(), ['w1'])
assert_equal(w1.getwalletinfo()['walletname'], 'w1')
+ # Test backing up and restoring wallets
+ self.log.info("Test wallet backup")
+ self.restart_node(0, ['-nowallet'])
+ for wallet_name in wallet_names:
+ self.nodes[0].loadwallet(wallet_name)
+ for wallet_name in wallet_names:
+ rpc = self.nodes[0].get_wallet_rpc(wallet_name)
+ addr = rpc.getnewaddress()
+ backup = os.path.join(self.options.tmpdir, 'backup.dat')
+ rpc.backupwallet(backup)
+ self.nodes[0].unloadwallet(wallet_name)
+ shutil.copyfile(empty_wallet, wallet_file(wallet_name))
+ self.nodes[0].loadwallet(wallet_name)
+ assert_equal(rpc.getaddressinfo(addr)['ismine'], False)
+ self.nodes[0].unloadwallet(wallet_name)
+ shutil.copyfile(backup, wallet_file(wallet_name))
+ self.nodes[0].loadwallet(wallet_name)
+ assert_equal(rpc.getaddressinfo(addr)['ismine'], True)
+
if __name__ == '__main__':
MultiWalletTest().main()