diff --git a/src/util/system.cpp b/src/util/system.cpp index 4784b92217..7b9ac9f198 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -1,1383 +1,1389 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) #include #endif #include #include #include #include #include #include #include #include #include #include #include #if (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) #include #include #endif #ifndef WIN32 // for posix_fallocate #ifdef __linux__ #ifdef _POSIX_C_SOURCE #undef _POSIX_C_SOURCE #endif #define _POSIX_C_SOURCE 200112L #endif // __linux__ #include #include #include #include #include #else #ifdef _MSC_VER #pragma warning(disable : 4786) #pragma warning(disable : 4804) #pragma warning(disable : 4805) #pragma warning(disable : 4717) #endif #ifdef _WIN32_WINNT #undef _WIN32_WINNT #endif #define _WIN32_WINNT 0x0501 #ifdef _WIN32_IE #undef _WIN32_IE #endif #define _WIN32_IE 0x0501 #define WIN32_LEAN_AND_MEAN 1 #ifndef NOMINMAX #define NOMINMAX #endif #include #include /* for _commit */ #include #include #endif #ifdef HAVE_SYS_PRCTL_H #include #endif #ifdef HAVE_MALLOPT_ARENA_MAX #include #endif // Application startup time (used for uptime calculation) const int64_t nStartupTime = GetTime(); const char *const BITCOIN_CONF_FILENAME = "bitcoin.conf"; const char *const BITCOIN_PID_FILENAME = "bitcoind.pid"; ArgsManager gArgs; /** * A map that contains all the currently held directory locks. After successful * locking, these will be held here until the global destructor cleans them up * and thus automatically unlocks them, or ReleaseDirectoryLocks is called. */ static std::map> dir_locks; /** Mutex to protect dir_locks. */ static std::mutex cs_dir_locks; bool LockDirectory(const fs::path &directory, const std::string lockfile_name, bool probe_only) { std::lock_guard ulock(cs_dir_locks); fs::path pathLockFile = directory / lockfile_name; // If a lock for this directory already exists in the map, don't try to // re-lock it if (dir_locks.count(pathLockFile.string())) { return true; } // Create empty lock file if it doesn't exist. FILE *file = fsbridge::fopen(pathLockFile, "a"); if (file) { fclose(file); } auto lock = std::make_unique(pathLockFile); if (!lock->TryLock()) { return error("Error while attempting to lock directory %s: %s", directory.string(), lock->GetReason()); } if (!probe_only) { // Lock successful and we're not just probing, put it into the map dir_locks.emplace(pathLockFile.string(), std::move(lock)); } return true; } +void UnlockDirectory(const fs::path &directory, + const std::string &lockfile_name) { + std::lock_guard lock(cs_dir_locks); + dir_locks.erase((directory / lockfile_name).string()); +} + void ReleaseDirectoryLocks() { std::lock_guard ulock(cs_dir_locks); dir_locks.clear(); } bool DirIsWritable(const fs::path &directory) { fs::path tmpFile = directory / fs::unique_path(); FILE *file = fsbridge::fopen(tmpFile, "a"); if (!file) { return false; } fclose(file); remove(tmpFile); return true; } bool CheckDiskSpace(const fs::path &dir, uint64_t additional_bytes) { // 50 MiB constexpr uint64_t min_disk_space = 52428800; uint64_t free_bytes_available = fs::space(dir).available; return free_bytes_available >= min_disk_space + additional_bytes; } /** * Interpret a string argument as a boolean. * * The definition of atoi() requires that non-numeric string values like "foo", * return 0. This means that if a user unintentionally supplies a non-integer * argument here, the return value is always false. This means that -foo=false * does what the user probably expects, but -foo=true is well defined but does * not do what they probably expected. * * The return value of atoi() is undefined when given input not representable as * an int. On most systems this means string value between "-2147483648" and * "2147483647" are well defined (this method will return true). Setting * -txindex=2147483648 on most systems, however, is probably undefined. * * For a more extensive discussion of this topic (and a wide range of opinions * on the Right Way to change this code), see PR12713. */ static bool InterpretBool(const std::string &strValue) { if (strValue.empty()) { return true; } return (atoi(strValue) != 0); } /** Internal helper functions for ArgsManager */ class ArgsManagerHelper { public: typedef std::map> MapArgs; /** Determine whether to use config settings in the default section, * See also comments around ArgsManager::ArgsManager() below. */ static inline bool UseDefaultSection(const ArgsManager &am, const std::string &arg) EXCLUSIVE_LOCKS_REQUIRED(am.cs_args) { return (am.m_network == CBaseChainParams::MAIN || am.m_network_only_args.count(arg) == 0); } /** Convert regular argument into the network-specific setting */ static inline std::string NetworkArg(const ArgsManager &am, const std::string &arg) { assert(arg.length() > 1 && arg[0] == '-'); return "-" + am.m_network + "." + arg.substr(1); } /** Find arguments in a map and add them to a vector */ static inline void AddArgs(std::vector &res, const MapArgs &map_args, const std::string &arg) { auto it = map_args.find(arg); if (it != map_args.end()) { res.insert(res.end(), it->second.begin(), it->second.end()); } } /** * Return true/false if an argument is set in a map, and also * return the first (or last) of the possibly multiple values it has */ static inline std::pair GetArgHelper(const MapArgs &map_args, const std::string &arg, bool getLast = false) { auto it = map_args.find(arg); if (it == map_args.end() || it->second.empty()) { return std::make_pair(false, std::string()); } if (getLast) { return std::make_pair(true, it->second.back()); } else { return std::make_pair(true, it->second.front()); } } /** * Get the string value of an argument, returning a pair of a boolean * indicating the argument was found, and the value for the argument * if it was found (or the empty string if not found). */ static inline std::pair GetArg(const ArgsManager &am, const std::string &arg) { LOCK(am.cs_args); std::pair found_result(false, std::string()); // We pass "true" to GetArgHelper in order to return the last // argument value seen from the command line (so "bitcoind -foo=bar // -foo=baz" gives GetArg(am,"foo")=={true,"baz"} found_result = GetArgHelper(am.m_override_args, arg, true); if (found_result.first) { return found_result; } // But in contrast we return the first argument seen in a config file, // so "foo=bar \n foo=baz" in the config file gives // GetArg(am,"foo")={true,"bar"} if (!am.m_network.empty()) { found_result = GetArgHelper(am.m_config_args, NetworkArg(am, arg)); if (found_result.first) { return found_result; } } if (UseDefaultSection(am, arg)) { found_result = GetArgHelper(am.m_config_args, arg); if (found_result.first) { return found_result; } } return found_result; } /* Special test for -testnet and -regtest args, because we don't want to be * confused by craziness like "[regtest] testnet=1" */ static inline bool GetNetBoolArg(const ArgsManager &am, const std::string &net_arg) EXCLUSIVE_LOCKS_REQUIRED(am.cs_args) { std::pair found_result(false, std::string()); found_result = GetArgHelper(am.m_override_args, net_arg, true); if (!found_result.first) { found_result = GetArgHelper(am.m_config_args, net_arg, true); if (!found_result.first) { // not set return false; } } // is set, so evaluate return InterpretBool(found_result.second); } }; /** * Interpret -nofoo as if the user supplied -foo=0. * * This method also tracks when the -no form was supplied, and if so, checks * whether there was a double-negative (-nofoo=0 -> -foo=1). * * If there was not a double negative, it removes the "no" from the key, and * returns true, indicating the caller should clear the args vector to indicate * a negated option. * * If there was a double negative, it removes "no" from the key, sets the value * to "1" and returns false. * * If there was no "no", it leaves key and value untouched and returns false. * * Where an option was negated can be later checked using the IsArgNegated() * method. One use case for this is to have a way to disable options that are * not normally boolean (e.g. using -nodebuglogfile to request that debug log * output is not sent to any file at all). */ static bool InterpretNegatedOption(std::string &key, std::string &val) { assert(key[0] == '-'); size_t option_index = key.find('.'); if (option_index == std::string::npos) { option_index = 1; } else { ++option_index; } if (key.substr(option_index, 2) == "no") { bool bool_val = InterpretBool(val); key.erase(option_index, 2); if (!bool_val) { // Double negatives like -nofoo=0 are supported (but discouraged) LogPrintf( "Warning: parsed potentially confusing double-negative %s=%s\n", key, val); val = "1"; } else { return true; } } return false; } ArgsManager::ArgsManager() : /* These options would cause cross-contamination if values for mainnet * were used while running on regtest/testnet (or vice-versa). * Setting them as section_only_args ensures that sharing a config file * between mainnet and regtest/testnet won't cause problems due to these * parameters by accident. */ m_network_only_args{ "-addnode", "-connect", "-port", "-bind", "-rpcport", "-rpcbind", "-wallet", } { // nothing to do } void ArgsManager::WarnForSectionOnlyArgs() { LOCK(cs_args); // if there's no section selected, don't worry if (m_network.empty()) { return; } // if it's okay to use the default section for this network, don't worry if (m_network == CBaseChainParams::MAIN) { return; } for (const auto &arg : m_network_only_args) { std::pair found_result; // if this option is overridden it's fine found_result = ArgsManagerHelper::GetArgHelper(m_override_args, arg); if (found_result.first) { continue; } // if there's a network-specific value for this option, it's fine found_result = ArgsManagerHelper::GetArgHelper( m_config_args, ArgsManagerHelper::NetworkArg(*this, arg)); if (found_result.first) { continue; } // if there isn't a default value for this option, it's fine found_result = ArgsManagerHelper::GetArgHelper(m_config_args, arg); if (!found_result.first) { continue; } // otherwise, issue a warning LogPrintf("Warning: Config setting for %s only applied on %s network " "when in [%s] section.\n", arg, m_network, m_network); } } void ArgsManager::SelectConfigNetwork(const std::string &network) { LOCK(cs_args); m_network = network; } bool ParseKeyValue(std::string &key, std::string &val) { size_t is_index = key.find('='); if (is_index != std::string::npos) { val = key.substr(is_index + 1); key.erase(is_index); } #ifdef WIN32 std::transform(key.begin(), key.end(), key.begin(), ::tolower); if (key[0] == '/') { key[0] = '-'; } #endif if (key[0] != '-') { return false; } // Transform --foo to -foo if (key.length() > 1 && key[1] == '-') { key.erase(0, 1); } return true; } bool ArgsManager::ParseParameters(int argc, const char *const argv[], std::string &error) { LOCK(cs_args); m_override_args.clear(); for (int i = 1; i < argc; i++) { std::string key(argv[i]); std::string val; if (!ParseKeyValue(key, val)) { break; } // Check for -nofoo if (InterpretNegatedOption(key, val)) { m_override_args[key].clear(); } else { m_override_args[key].push_back(val); } // Check that the arg is known if (!(IsSwitchChar(key[0]) && key.size() == 1)) { if (!IsArgKnown(key)) { error = strprintf("Invalid parameter %s", key.c_str()); return false; } } } // we do not allow -includeconf from command line, so we clear it here auto it = m_override_args.find("-includeconf"); if (it != m_override_args.end()) { if (it->second.size() > 0) { for (const auto &ic : it->second) { error += "-includeconf cannot be used from commandline; " "-includeconf=" + ic + "\n"; } return false; } } return true; } bool ArgsManager::IsArgKnown(const std::string &key) const { size_t option_index = key.find('.'); std::string arg_no_net; if (option_index == std::string::npos) { arg_no_net = key; } else { arg_no_net = std::string("-") + key.substr(option_index + 1, std::string::npos); } LOCK(cs_args); for (const auto &arg_map : m_available_args) { if (arg_map.second.count(arg_no_net)) { return true; } } return false; } std::vector ArgsManager::GetArgs(const std::string &strArg) const { std::vector result = {}; // special case if (IsArgNegated(strArg)) { return result; } LOCK(cs_args); ArgsManagerHelper::AddArgs(result, m_override_args, strArg); if (!m_network.empty()) { ArgsManagerHelper::AddArgs( result, m_config_args, ArgsManagerHelper::NetworkArg(*this, strArg)); } if (ArgsManagerHelper::UseDefaultSection(*this, strArg)) { ArgsManagerHelper::AddArgs(result, m_config_args, strArg); } return result; } bool ArgsManager::IsArgSet(const std::string &strArg) const { // special case if (IsArgNegated(strArg)) { return true; } return ArgsManagerHelper::GetArg(*this, strArg).first; } bool ArgsManager::IsArgNegated(const std::string &strArg) const { LOCK(cs_args); const auto &ov = m_override_args.find(strArg); if (ov != m_override_args.end()) { return ov->second.empty(); } if (!m_network.empty()) { const auto &cfs = m_config_args.find(ArgsManagerHelper::NetworkArg(*this, strArg)); if (cfs != m_config_args.end()) { return cfs->second.empty(); } } const auto &cf = m_config_args.find(strArg); if (cf != m_config_args.end()) { return cf->second.empty(); } return false; } std::string ArgsManager::GetArg(const std::string &strArg, const std::string &strDefault) const { if (IsArgNegated(strArg)) { return "0"; } std::pair found_res = ArgsManagerHelper::GetArg(*this, strArg); if (found_res.first) { return found_res.second; } return strDefault; } int64_t ArgsManager::GetArg(const std::string &strArg, int64_t nDefault) const { if (IsArgNegated(strArg)) { return 0; } std::pair found_res = ArgsManagerHelper::GetArg(*this, strArg); if (found_res.first) { return atoi64(found_res.second); } return nDefault; } bool ArgsManager::GetBoolArg(const std::string &strArg, bool fDefault) const { if (IsArgNegated(strArg)) { return false; } std::pair found_res = ArgsManagerHelper::GetArg(*this, strArg); if (found_res.first) { return InterpretBool(found_res.second); } return fDefault; } bool ArgsManager::SoftSetArg(const std::string &strArg, const std::string &strValue) { LOCK(cs_args); if (IsArgSet(strArg)) { return false; } ForceSetArg(strArg, strValue); return true; } bool ArgsManager::SoftSetBoolArg(const std::string &strArg, bool fValue) { if (fValue) { return SoftSetArg(strArg, std::string("1")); } else { return SoftSetArg(strArg, std::string("0")); } } void ArgsManager::ForceSetArg(const std::string &strArg, const std::string &strValue) { LOCK(cs_args); m_override_args[strArg] = {strValue}; } /** * This function is only used for testing purpose so * so we should not worry about element uniqueness and * integrity of mapMultiArgs data structure */ void ArgsManager::ForceSetMultiArg(const std::string &strArg, const std::string &strValue) { LOCK(cs_args); m_override_args[strArg].push_back(strValue); } void ArgsManager::AddArg(const std::string &name, const std::string &help, const bool debug_only, const OptionsCategory &cat) { // Split arg name from its help param size_t eq_index = name.find('='); if (eq_index == std::string::npos) { eq_index = name.size(); } LOCK(cs_args); std::map &arg_map = m_available_args[cat]; auto ret = arg_map.emplace( name.substr(0, eq_index), Arg(name.substr(eq_index, name.size() - eq_index), help, debug_only)); // Make sure an insertion actually happened. assert(ret.second); } void ArgsManager::AddHiddenArgs(const std::vector &names) { for (const std::string &name : names) { AddArg(name, "", false, OptionsCategory::HIDDEN); } } void ArgsManager::ClearArg(const std::string &strArg) { LOCK(cs_args); m_override_args.erase(strArg); m_config_args.erase(strArg); } std::string ArgsManager::GetHelpMessage() const { const bool show_debug = gArgs.GetBoolArg("-help-debug", false); std::string usage = ""; LOCK(cs_args); for (const auto &arg_map : m_available_args) { switch (arg_map.first) { case OptionsCategory::OPTIONS: usage += HelpMessageGroup("Options:"); break; case OptionsCategory::CONNECTION: usage += HelpMessageGroup("Connection options:"); break; case OptionsCategory::ZMQ: usage += HelpMessageGroup("ZeroMQ notification options:"); break; case OptionsCategory::DEBUG_TEST: usage += HelpMessageGroup("Debugging/Testing options:"); break; case OptionsCategory::NODE_RELAY: usage += HelpMessageGroup("Node relay options:"); break; case OptionsCategory::BLOCK_CREATION: usage += HelpMessageGroup("Block creation options:"); break; case OptionsCategory::RPC: usage += HelpMessageGroup("RPC server options:"); break; case OptionsCategory::WALLET: usage += HelpMessageGroup("Wallet options:"); break; case OptionsCategory::WALLET_DEBUG_TEST: if (show_debug) { usage += HelpMessageGroup("Wallet debugging/testing options:"); } break; case OptionsCategory::CHAINPARAMS: usage += HelpMessageGroup("Chain selection options:"); break; case OptionsCategory::GUI: usage += HelpMessageGroup("UI Options:"); break; case OptionsCategory::COMMANDS: usage += HelpMessageGroup("Commands:"); break; case OptionsCategory::REGISTER_COMMANDS: usage += HelpMessageGroup("Register Commands:"); break; default: break; } // When we get to the hidden options, stop if (arg_map.first == OptionsCategory::HIDDEN) { break; } for (const auto &arg : arg_map.second) { if (show_debug || !arg.second.m_debug_only) { std::string name; if (arg.second.m_help_param.empty()) { name = arg.first; } else { name = arg.first + arg.second.m_help_param; } usage += HelpMessageOpt(name, arg.second.m_help_text); } } } return usage; } bool HelpRequested(const ArgsManager &args) { return args.IsArgSet("-?") || args.IsArgSet("-h") || args.IsArgSet("-help"); } static const int screenWidth = 79; static const int optIndent = 2; static const int msgIndent = 7; std::string HelpMessageGroup(const std::string &message) { return std::string(message) + std::string("\n\n"); } std::string HelpMessageOpt(const std::string &option, const std::string &message) { return std::string(optIndent, ' ') + std::string(option) + std::string("\n") + std::string(msgIndent, ' ') + FormatParagraph(message, screenWidth - msgIndent, msgIndent) + std::string("\n\n"); } static std::string FormatException(const std::exception *pex, const char *pszThread) { #ifdef WIN32 char pszModule[MAX_PATH] = ""; GetModuleFileNameA(nullptr, pszModule, sizeof(pszModule)); #else const char *pszModule = "bitcoin"; #endif if (pex) { return strprintf("EXCEPTION: %s \n%s \n%s in %s \n", typeid(*pex).name(), pex->what(), pszModule, pszThread); } else { return strprintf("UNKNOWN EXCEPTION \n%s in %s \n", pszModule, pszThread); } } void PrintExceptionContinue(const std::exception *pex, const char *pszThread) { std::string message = FormatException(pex, pszThread); LogPrintf("\n\n************************\n%s\n", message); fprintf(stderr, "\n\n************************\n%s\n", message.c_str()); } fs::path GetDefaultDataDir() { // Windows < Vista: C:\Documents and Settings\Username\Application Data\Bitcoin // Windows >= Vista: C:\Users\Username\AppData\Roaming\Bitcoin // Mac: ~/Library/Application Support/Bitcoin // Unix: ~/.bitcoin #ifdef WIN32 // Windows return GetSpecialFolderPath(CSIDL_APPDATA) / "Bitcoin"; #else fs::path pathRet; char *pszHome = getenv("HOME"); if (pszHome == nullptr || strlen(pszHome) == 0) { pathRet = fs::path("/"); } else { pathRet = fs::path(pszHome); } #ifdef MAC_OSX // Mac return pathRet / "Library/Application Support/Bitcoin"; #else // Unix return pathRet / ".bitcoin"; #endif #endif } static fs::path g_blocks_path_cache_net_specific; static fs::path pathCached; static fs::path pathCachedNetSpecific; static CCriticalSection csPathCached; const fs::path &GetBlocksDir() { LOCK(csPathCached); fs::path &path = g_blocks_path_cache_net_specific; // This can be called during exceptions by LogPrintf(), so we cache the // value so we don't have to do memory allocations after that. if (!path.empty()) { return path; } if (gArgs.IsArgSet("-blocksdir")) { path = fs::system_complete(gArgs.GetArg("-blocksdir", "")); if (!fs::is_directory(path)) { path = ""; return path; } } else { path = GetDataDir(false); } path /= BaseParams().DataDir(); path /= "blocks"; fs::create_directories(path); return path; } const fs::path &GetDataDir(bool fNetSpecific) { LOCK(csPathCached); fs::path &path = fNetSpecific ? pathCachedNetSpecific : pathCached; // This can be called during exceptions by LogPrintf(), so we cache the // value so we don't have to do memory allocations after that. if (!path.empty()) { return path; } if (gArgs.IsArgSet("-datadir")) { path = fs::system_complete(gArgs.GetArg("-datadir", "")); if (!fs::is_directory(path)) { path = ""; return path; } } else { path = GetDefaultDataDir(); } if (fNetSpecific) { path /= BaseParams().DataDir(); } if (fs::create_directories(path)) { // This is the first run, create wallets subdirectory too // // TODO: this is an ugly way to create the wallets/ directory and // really shouldn't be done here. Once this is fixed, please // also remove the corresponding line in bitcoind.cpp AppInit. // See more info at: // https://reviews.bitcoinabc.org/D3312 fs::create_directories(path / "wallets"); } return path; } void ClearDatadirCache() { LOCK(csPathCached); pathCached = fs::path(); pathCachedNetSpecific = fs::path(); g_blocks_path_cache_net_specific = fs::path(); } fs::path GetConfigFile(const std::string &confPath) { return AbsPathForConfigVal(fs::path(confPath), false); } static std::string TrimString(const std::string &str, const std::string &pattern) { std::string::size_type front = str.find_first_not_of(pattern); if (front == std::string::npos) { return std::string(); } std::string::size_type end = str.find_last_not_of(pattern); return str.substr(front, end - front + 1); } static bool GetConfigOptions(std::istream &stream, std::string &error, std::vector> &options) { std::string str, prefix; std::string::size_type pos; int linenr = 1; while (std::getline(stream, str)) { bool used_hash = false; if ((pos = str.find('#')) != std::string::npos) { str = str.substr(0, pos); used_hash = true; } const static std::string pattern = " \t\r\n"; str = TrimString(str, pattern); if (!str.empty()) { if (*str.begin() == '[' && *str.rbegin() == ']') { prefix = str.substr(1, str.size() - 2) + '.'; } else if (*str.begin() == '-') { error = strprintf( "parse error on line %i: %s, options in configuration file " "must be specified without leading -", linenr, str); return false; } else if ((pos = str.find('=')) != std::string::npos) { std::string name = prefix + TrimString(str.substr(0, pos), pattern); std::string value = TrimString(str.substr(pos + 1), pattern); if (used_hash && name == "rpcpassword") { error = strprintf( "parse error on line %i, using # in rpcpassword can be " "ambiguous and should be avoided", linenr); return false; } options.emplace_back(name, value); } else { error = strprintf("parse error on line %i: %s", linenr, str); if (str.size() >= 2 && str.substr(0, 2) == "no") { error += strprintf(", if you intended to specify a negated " "option, use %s=1 instead", str); } return false; } } ++linenr; } return true; } bool ArgsManager::ReadConfigStream(std::istream &stream, std::string &error, bool ignore_invalid_keys) { LOCK(cs_args); std::vector> options; if (!GetConfigOptions(stream, error, options)) { return false; } for (const std::pair &option : options) { std::string strKey = std::string("-") + option.first; std::string strValue = option.second; if (InterpretNegatedOption(strKey, strValue)) { m_config_args[strKey].clear(); } else { m_config_args[strKey].push_back(strValue); } // Check that the arg is known if (!IsArgKnown(strKey) && !ignore_invalid_keys) { error = strprintf("Invalid configuration value %s", option.first.c_str()); return false; } } return true; } bool ArgsManager::ReadConfigFiles(std::string &error, bool ignore_invalid_keys) { { LOCK(cs_args); m_config_args.clear(); } const std::string confPath = GetArg("-conf", BITCOIN_CONF_FILENAME); fs::ifstream stream(GetConfigFile(confPath)); // ok to not have a config file if (stream.good()) { if (!ReadConfigStream(stream, error, ignore_invalid_keys)) { return false; } // if there is an -includeconf in the override args, but it is empty, // that means the user passed '-noincludeconf' on the command line, in // which case we should not include anything bool emptyIncludeConf; { LOCK(cs_args); emptyIncludeConf = m_override_args.count("-includeconf") == 0; } if (emptyIncludeConf) { std::string chain_id = GetChainName(); std::vector includeconf(GetArgs("-includeconf")); { // We haven't set m_network yet (that happens in // SelectParams()), so manually check for network.includeconf // args. std::vector includeconf_net( GetArgs(std::string("-") + chain_id + ".includeconf")); includeconf.insert(includeconf.end(), includeconf_net.begin(), includeconf_net.end()); } // Remove -includeconf from configuration, so we can warn about // recursion later { LOCK(cs_args); m_config_args.erase("-includeconf"); m_config_args.erase(std::string("-") + chain_id + ".includeconf"); } for (const std::string &to_include : includeconf) { fs::ifstream include_config(GetConfigFile(to_include)); if (include_config.good()) { if (!ReadConfigStream(include_config, error, ignore_invalid_keys)) { return false; } LogPrintf("Included configuration file %s\n", to_include.c_str()); } else { error = "Failed to include configuration file " + to_include; return false; } } // Warn about recursive -includeconf includeconf = GetArgs("-includeconf"); { std::vector includeconf_net( GetArgs(std::string("-") + chain_id + ".includeconf")); includeconf.insert(includeconf.end(), includeconf_net.begin(), includeconf_net.end()); std::string chain_id_final = GetChainName(); if (chain_id_final != chain_id) { // Also warn about recursive includeconf for the chain that // was specified in one of the includeconfs includeconf_net = GetArgs(std::string("-") + chain_id_final + ".includeconf"); includeconf.insert(includeconf.end(), includeconf_net.begin(), includeconf_net.end()); } } for (const std::string &to_include : includeconf) { fprintf(stderr, "warning: -includeconf cannot be used from included " "files; ignoring -includeconf=%s\n", to_include.c_str()); } } } // If datadir is changed in .conf file: ClearDatadirCache(); if (!fs::is_directory(GetDataDir(false))) { error = strprintf("specified data directory \"%s\" does not exist.", gArgs.GetArg("-datadir", "").c_str()); return false; } return true; } std::string ArgsManager::GetChainName() const { LOCK(cs_args); bool fRegTest = ArgsManagerHelper::GetNetBoolArg(*this, "-regtest"); bool fTestNet = ArgsManagerHelper::GetNetBoolArg(*this, "-testnet"); if (fTestNet && fRegTest) { throw std::runtime_error( "Invalid combination of -regtest and -testnet."); } if (fRegTest) { return CBaseChainParams::REGTEST; } if (fTestNet) { return CBaseChainParams::TESTNET; } return CBaseChainParams::MAIN; } #ifndef WIN32 fs::path GetPidFile() { return AbsPathForConfigVal( fs::path(gArgs.GetArg("-pid", BITCOIN_PID_FILENAME))); } void CreatePidFile(const fs::path &path, pid_t pid) { FILE *file = fsbridge::fopen(path, "w"); if (file) { fprintf(file, "%d\n", pid); fclose(file); } } #endif bool RenameOver(fs::path src, fs::path dest) { #ifdef WIN32 return MoveFileExA(src.string().c_str(), dest.string().c_str(), MOVEFILE_REPLACE_EXISTING) != 0; #else int rc = std::rename(src.string().c_str(), dest.string().c_str()); return (rc == 0); #endif /* WIN32 */ } /** * Ignores exceptions thrown by Boost's create_directories if the requested * directory exists. Specifically handles case where path p exists, but it * wasn't possible for the user to write to the parent directory. */ bool TryCreateDirectories(const fs::path &p) { try { return fs::create_directories(p); } catch (const fs::filesystem_error &) { if (!fs::exists(p) || !fs::is_directory(p)) { throw; } } // create_directory didn't create the directory, it had to have existed // already. return false; } bool FileCommit(FILE *file) { // harmless if redundantly called if (fflush(file) != 0) { LogPrintf("%s: fflush failed: %d\n", __func__, errno); return false; } #ifdef WIN32 HANDLE hFile = (HANDLE)_get_osfhandle(_fileno(file)); if (FlushFileBuffers(hFile) == 0) { LogPrintf("%s: FlushFileBuffers failed: %d\n", __func__, GetLastError()); return false; } #else #if defined(__linux__) || defined(__NetBSD__) // Ignore EINVAL for filesystems that don't support sync if (fdatasync(fileno(file)) != 0 && errno != EINVAL) { LogPrintf("%s: fdatasync failed: %d\n", __func__, errno); return false; } #elif defined(MAC_OSX) && defined(F_FULLFSYNC) // Manpage says "value other than -1" is returned on success if (fcntl(fileno(file), F_FULLFSYNC, 0) == -1) { LogPrintf("%s: fcntl F_FULLFSYNC failed: %d\n", __func__, errno); return false; } #else if (fsync(fileno(file)) != 0 && errno != EINVAL) { LogPrintf("%s: fsync failed: %d\n", __func__, errno); return false; } #endif #endif return true; } bool TruncateFile(FILE *file, unsigned int length) { #if defined(WIN32) return _chsize(_fileno(file), length) == 0; #else return ftruncate(fileno(file), length) == 0; #endif } /** * This function tries to raise the file descriptor limit to the requested * number. It returns the actual file descriptor limit (which may be more or * less than nMinFD) */ int RaiseFileDescriptorLimit(int nMinFD) { #if defined(WIN32) return 2048; #else struct rlimit limitFD; if (getrlimit(RLIMIT_NOFILE, &limitFD) != -1) { if (limitFD.rlim_cur < (rlim_t)nMinFD) { limitFD.rlim_cur = nMinFD; if (limitFD.rlim_cur > limitFD.rlim_max) { limitFD.rlim_cur = limitFD.rlim_max; } setrlimit(RLIMIT_NOFILE, &limitFD); getrlimit(RLIMIT_NOFILE, &limitFD); } return limitFD.rlim_cur; } // getrlimit failed, assume it's fine. return nMinFD; #endif } /** * This function tries to make a particular range of a file allocated * (corresponding to disk space) it is advisory, and the range specified in the * arguments will never contain live data. */ void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length) { #if defined(WIN32) // Windows-specific version. HANDLE hFile = (HANDLE)_get_osfhandle(_fileno(file)); LARGE_INTEGER nFileSize; int64_t nEndPos = (int64_t)offset + length; nFileSize.u.LowPart = nEndPos & 0xFFFFFFFF; nFileSize.u.HighPart = nEndPos >> 32; SetFilePointerEx(hFile, nFileSize, 0, FILE_BEGIN); SetEndOfFile(hFile); #elif defined(MAC_OSX) // OSX specific version. fstore_t fst; fst.fst_flags = F_ALLOCATECONTIG; fst.fst_posmode = F_PEOFPOSMODE; fst.fst_offset = 0; fst.fst_length = (off_t)offset + length; fst.fst_bytesalloc = 0; if (fcntl(fileno(file), F_PREALLOCATE, &fst) == -1) { fst.fst_flags = F_ALLOCATEALL; fcntl(fileno(file), F_PREALLOCATE, &fst); } ftruncate(fileno(file), fst.fst_length); #elif defined(__linux__) // Version using posix_fallocate. off_t nEndPos = (off_t)offset + length; posix_fallocate(fileno(file), 0, nEndPos); #else // Fallback version // TODO: just write one byte per block static const char buf[65536] = {}; if (fseek(file, offset, SEEK_SET)) { return; } while (length > 0) { unsigned int now = 65536; if (length < now) { now = length; } // Allowed to fail; this function is advisory anyway. fwrite(buf, 1, now, file); length -= now; } #endif } #ifdef WIN32 fs::path GetSpecialFolderPath(int nFolder, bool fCreate) { char pszPath[MAX_PATH] = ""; if (SHGetSpecialFolderPathA(nullptr, pszPath, nFolder, fCreate)) { return fs::path(pszPath); } LogPrintf( "SHGetSpecialFolderPathA() failed, could not obtain requested path.\n"); return fs::path(""); } #endif void runCommand(const std::string &strCommand) { if (strCommand.empty()) { return; } #ifndef WIN32 int nErr = ::system(strCommand.c_str()); #else int nErr = ::_wsystem( std::wstring_convert, wchar_t>() .from_bytes(strCommand) .c_str()); #endif if (nErr) { LogPrintf("runCommand error: system(%s) returned %d\n", strCommand, nErr); } } void RenameThread(const char *name) { #if defined(PR_SET_NAME) // Only the first 15 characters are used (16 - NUL terminator) ::prctl(PR_SET_NAME, name, 0, 0, 0); #elif (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) pthread_set_name_np(pthread_self(), name); #elif defined(MAC_OSX) pthread_setname_np(name); #else // Prevent warnings for unused parameters... (void)name; #endif } void SetupEnvironment() { #ifdef HAVE_MALLOPT_ARENA_MAX // glibc-specific: On 32-bit systems set the number of arenas to 1. By // default, since glibc 2.10, the C library will create up to two heap // arenas per core. This is known to cause excessive virtual address space // usage in our usage. Work around it by setting the maximum number of // arenas to 1. if (sizeof(void *) == 4) { mallopt(M_ARENA_MAX, 1); } #endif // On most POSIX systems (e.g. Linux, but not BSD) the environment's locale may // be invalid, in which case the "C" locale is used as fallback. #if !defined(WIN32) && !defined(MAC_OSX) && !defined(__FreeBSD__) && \ !defined(__OpenBSD__) try { // Raises a runtime error if current locale is invalid. std::locale(""); } catch (const std::runtime_error &) { setenv("LC_ALL", "C", 1); } #elif defined(WIN32) // Set the default input/output charset is utf-8 SetConsoleCP(CP_UTF8); SetConsoleOutputCP(CP_UTF8); #endif // The path locale is lazy initialized and to avoid deinitialization errors // in multithreading environments, it is set explicitly by the main thread. // A dummy locale is used to extract the internal default locale, used by // fs::path, which is then used to explicitly imbue the path. std::locale loc = fs::path::imbue(std::locale::classic()); #ifndef WIN32 fs::path::imbue(loc); #else fs::path::imbue(std::locale(loc, new std::codecvt_utf8_utf16())); #endif } bool SetupNetworking() { #ifdef WIN32 // Initialize Windows Sockets. WSADATA wsadata; int ret = WSAStartup(MAKEWORD(2, 2), &wsadata); if (ret != NO_ERROR || LOBYTE(wsadata.wVersion) != 2 || HIBYTE(wsadata.wVersion) != 2) { return false; } #endif return true; } int GetNumCores() { return boost::thread::physical_concurrency(); } std::string CopyrightHolders(const std::string &strPrefix) { return strPrefix + strprintf(_(COPYRIGHT_HOLDERS), _(COPYRIGHT_HOLDERS_SUBSTITUTION)); } // Obtain the application startup time (used for uptime calculation) int64_t GetStartupTime() { return nStartupTime; } fs::path AbsPathForConfigVal(const fs::path &path, bool net_specific) { return fs::absolute(path, GetDataDir(net_specific)); } int ScheduleBatchPriority() { #ifdef SCHED_BATCH const static sched_param param{}; if (int ret = pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m)) { LogPrintf("Failed to pthread_setschedparam: %s\n", strerror(errno)); return ret; } return 0; #else return 1; #endif } namespace util { #ifdef WIN32 WinCmdLineArgs::WinCmdLineArgs() { wchar_t **wargv = CommandLineToArgvW(GetCommandLineW(), &argc); std::wstring_convert, wchar_t> utf8_cvt; argv = new char *[argc]; args.resize(argc); for (int i = 0; i < argc; i++) { args[i] = utf8_cvt.to_bytes(wargv[i]); argv[i] = &*args[i].begin(); } LocalFree(wargv); } WinCmdLineArgs::~WinCmdLineArgs() { delete[] argv; } std::pair WinCmdLineArgs::get() { return std::make_pair(argc, argv); } #endif } // namespace util diff --git a/src/util/system.h b/src/util/system.h index dc7edf291d..627b2d8ddf 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -1,397 +1,399 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. /** * Server/client environment: argument handling, config file parsing, * thread wrappers, startup time */ #ifndef BITCOIN_UTIL_SYSTEM_H #define BITCOIN_UTIL_SYSTEM_H #if defined(HAVE_CONFIG_H) #include #endif #include #include #include #include #include #include #include #include #include // for boost::thread_interrupted #include #include #include #include #include #include #include #include #include // Application startup time (used for uptime calculation) int64_t GetStartupTime(); extern const char *const BITCOIN_CONF_FILENAME; extern const char *const BITCOIN_PID_FILENAME; /** Translate a message to the native language of the user. */ const extern std::function G_TRANSLATION_FUN; /** * Translation function. * If no translation function is set, simply return the input. */ inline std::string _(const char *psz) { return G_TRANSLATION_FUN ? (G_TRANSLATION_FUN)(psz) : psz; } void SetupEnvironment(); bool SetupNetworking(); template bool error(const char *fmt, const Args &... args) { LogPrintf("ERROR: %s\n", tfm::format(fmt, args...)); return false; } void PrintExceptionContinue(const std::exception *pex, const char *pszThread); bool FileCommit(FILE *file); bool TruncateFile(FILE *file, unsigned int length); int RaiseFileDescriptorLimit(int nMinFD); void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length); bool RenameOver(fs::path src, fs::path dest); bool LockDirectory(const fs::path &directory, const std::string lockfile_name, bool probe_only = false); +void UnlockDirectory(const fs::path &directory, + const std::string &lockfile_name); bool DirIsWritable(const fs::path &directory); bool CheckDiskSpace(const fs::path &dir, uint64_t additional_bytes = 0); /** * Release all directory locks. This is used for unit testing only, at runtime * the global destructor will take care of the locks. */ void ReleaseDirectoryLocks(); bool TryCreateDirectories(const fs::path &p); fs::path GetDefaultDataDir(); // The blocks directory is always net specific. const fs::path &GetBlocksDir(); const fs::path &GetDataDir(bool fNetSpecific = true); void ClearDatadirCache(); fs::path GetConfigFile(const std::string &confPath); #ifndef WIN32 fs::path GetPidFile(); void CreatePidFile(const fs::path &path, pid_t pid); #endif #ifdef WIN32 fs::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif void runCommand(const std::string &strCommand); NODISCARD bool ParseKeyValue(std::string &key, std::string &val); /** * Most paths passed as configuration arguments are treated as relative to * the datadir if they are not absolute. * * @param path The path to be conditionally prefixed with datadir. * @param net_specific Forwarded to GetDataDir(). * @return The normalized path. */ fs::path AbsPathForConfigVal(const fs::path &path, bool net_specific = true); inline bool IsSwitchChar(char c) { #ifdef WIN32 return c == '-' || c == '/'; #else return c == '-'; #endif } enum class OptionsCategory { OPTIONS, CONNECTION, WALLET, WALLET_DEBUG_TEST, ZMQ, DEBUG_TEST, CHAINPARAMS, NODE_RELAY, BLOCK_CREATION, RPC, GUI, COMMANDS, REGISTER_COMMANDS, // Always the last option to avoid printing these in the help HIDDEN, }; class ArgsManager { protected: friend class ArgsManagerHelper; struct Arg { std::string m_help_param; std::string m_help_text; bool m_debug_only; Arg(const std::string &help_param, const std::string &help_text, bool debug_only) : m_help_param(help_param), m_help_text(help_text), m_debug_only(debug_only){}; }; mutable CCriticalSection cs_args; std::map> m_override_args GUARDED_BY(cs_args); std::map> m_config_args GUARDED_BY(cs_args); std::string m_network GUARDED_BY(cs_args); std::set m_network_only_args GUARDED_BY(cs_args); std::map> m_available_args GUARDED_BY(cs_args); bool ReadConfigStream(std::istream &stream, std::string &error, bool ignore_invalid_keys = false); public: ArgsManager(); /** * Select the network in use */ void SelectConfigNetwork(const std::string &network); NODISCARD bool ParseParameters(int argc, const char *const argv[], std::string &error); bool ReadConfigFiles(std::string &error, bool ignore_invalid_keys = false); /** * Log warnings for options in m_section_only_args when they are specified * in the default section but not overridden on the command line or in a * network-specific section in the config file. */ void WarnForSectionOnlyArgs(); /** * Return a vector of strings of the given argument * * @param strArg Argument to get (e.g. "-foo") * @return command-line arguments */ std::vector GetArgs(const std::string &strArg) const; /** * Return true if the given argument has been manually set. * * @param strArg Argument to get (e.g. "-foo") * @return true if the argument has been set */ bool IsArgSet(const std::string &strArg) const; /** * Return true if the argument was originally passed as a negated option, * i.e. -nofoo. * * @param strArg Argument to get (e.g. "-foo") * @return true if the argument was passed negated */ bool IsArgNegated(const std::string &strArg) const; /** * Return string argument or default value. * * @param strArg Argument to get (e.g. "-foo") * @param default (e.g. "1") * @return command-line argument or default value */ std::string GetArg(const std::string &strArg, const std::string &strDefault) const; /** * Return integer argument or default value. * * @param strArg Argument to get (e.g. "-foo") * @param default (e.g. 1) * @return command-line argument (0 if invalid number) or default value */ int64_t GetArg(const std::string &strArg, int64_t nDefault) const; /** * Return boolean argument or default value. * * @param strArg Argument to get (e.g. "-foo") * @param default (true or false) * @return command-line argument or default value */ bool GetBoolArg(const std::string &strArg, bool fDefault) const; /** * Set an argument if it doesn't already have a value. * * @param strArg Argument to set (e.g. "-foo") * @param strValue Value (e.g. "1") * @return true if argument gets set, false if it already had a value */ bool SoftSetArg(const std::string &strArg, const std::string &strValue); /** * Set a boolean argument if it doesn't already have a value. * * @param strArg Argument to set (e.g. "-foo") * @param fValue Value (e.g. false) * @return true if argument gets set, false if it already had a value */ bool SoftSetBoolArg(const std::string &strArg, bool fValue); // Forces an arg setting. Called by SoftSetArg() if the arg hasn't already // been set. Also called directly in testing. void ForceSetArg(const std::string &strArg, const std::string &strValue); // Forces a multi arg setting, used only in testing void ForceSetMultiArg(const std::string &strArg, const std::string &strValue); /** * Looks for -regtest, -testnet and returns the appropriate BIP70 chain * name. * @return CBaseChainParams::MAIN by default; raises runtime error if an * invalid combination is given. */ std::string GetChainName() const; /** * Add argument */ void AddArg(const std::string &name, const std::string &help, const bool debug_only, const OptionsCategory &cat); // Remove an arg setting, used only in testing void ClearArg(const std::string &strArg); /** * Add many hidden arguments */ void AddHiddenArgs(const std::vector &args); /** * Clear available arguments */ void ClearArgs() { LOCK(cs_args); m_available_args.clear(); } /** * Get the help string */ std::string GetHelpMessage() const; /** * Check whether we know of this arg */ bool IsArgKnown(const std::string &key) const; }; extern ArgsManager gArgs; /** * @return true if help has been requested via a command-line arg */ bool HelpRequested(const ArgsManager &args); /** * Format a string to be used as group of options in help messages. * * @param message Group name (e.g. "RPC server options:") * @return the formatted string */ std::string HelpMessageGroup(const std::string &message); /** * Format a string to be used as option description in help messages. * * @param option Option message (e.g. "-rpcuser=") * @param message Option description (e.g. "Username for JSON-RPC connections") * @return the formatted string */ std::string HelpMessageOpt(const std::string &option, const std::string &message); /** * Return the number of physical cores available on the current system. * @note This does not count virtual cores, such as those provided by * HyperThreading when boost is newer than 1.56. */ int GetNumCores(); void RenameThread(const char *name); /** * .. and a wrapper that just calls func once */ template void TraceThread(const char *name, Callable func) { std::string s = strprintf("bitcoin-%s", name); RenameThread(s.c_str()); try { LogPrintf("%s thread start\n", name); func(); LogPrintf("%s thread exit\n", name); } catch (const boost::thread_interrupted &) { LogPrintf("%s thread interrupt\n", name); throw; } catch (const std::exception &e) { PrintExceptionContinue(&e, name); throw; } catch (...) { PrintExceptionContinue(nullptr, name); throw; } } std::string CopyrightHolders(const std::string &strPrefix); namespace util { //! Simplification of std insertion template inline void insert(Tdst &dst, const Tsrc &src) { dst.insert(dst.begin(), src.begin(), src.end()); } template inline void insert(std::set &dst, const Tsrc &src) { dst.insert(src.begin(), src.end()); } #ifdef WIN32 class WinCmdLineArgs { public: WinCmdLineArgs(); ~WinCmdLineArgs(); std::pair get(); private: int argc; char **argv; std::vector args; }; #endif } // namespace util /** * On platforms that support it, tell the kernel the calling thread is * CPU-intensive and non-interactive. See SCHED_BATCH in sched(7) for details. * * @return The return value of sched_setschedule(), or 1 on systems without * sched_setchedule(). */ int ScheduleBatchPriority(); #endif // BITCOIN_UTIL_SYSTEM_H diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 29dc2bc6e6..17686f526f 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -1,968 +1,977 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include #include // boost::this_thread::interruption_point() (mingw) #include #ifndef WIN32 #include #endif namespace { //! Make sure database has a unique fileid within the environment. If it //! doesn't, throw an error. BDB caches do not work properly when more than one //! open database has the same fileid (values written to one database may show //! up in reads to other databases). //! //! BerkeleyDB generates unique fileids by default //! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html), //! so bitcoin should never create different databases with the same fileid, but //! this error can be triggered if users manually copy database files. void CheckUniqueFileid(const BerkeleyEnvironment &env, const std::string &filename, Db &db, WalletDatabaseFileId &fileid) { if (env.IsMock()) { return; } int ret = db.get_mpf()->get_fileid(fileid.value); if (ret != 0) { throw std::runtime_error(strprintf( "BerkeleyBatch: Can't open database %s (get_fileid failed with %d)", filename, ret)); } for (const auto &item : env.m_fileids) { if (fileid == item.second && &fileid != &item.second) { throw std::runtime_error(strprintf( "BerkeleyBatch: Can't open database %s (duplicates fileid %s " "from %s)", filename, HexStr(std::begin(item.second.value), std::end(item.second.value)), item.first)); } } } CCriticalSection cs_db; //!< Map from directory name to db environment. std::map> g_dbenvs GUARDED_BY(cs_db); } // namespace bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId &rhs) const { return memcmp(value, &rhs.value, sizeof(value)) == 0; } static void SplitWalletPath(const fs::path &wallet_path, fs::path &env_directory, std::string &database_filename) { if (fs::is_regular_file(wallet_path)) { // Special case for backwards compatibility: if wallet path points to an // existing file, treat it as the path to a BDB data file in a parent // directory that also contains BDB log files. env_directory = wallet_path.parent_path(); database_filename = wallet_path.filename().string(); } else { // Normal case: Interpret wallet path as a directory path containing // data and log files. env_directory = wallet_path; database_filename = "wallet.dat"; } } bool IsWalletLoaded(const fs::path &wallet_path) { fs::path env_directory; std::string database_filename; SplitWalletPath(wallet_path, env_directory, database_filename); LOCK(cs_db); auto env = g_dbenvs.find(env_directory.string()); if (env == g_dbenvs.end()) { return false; } auto database = env->second.lock(); return database && database->IsDatabaseLoaded(database_filename); } /** * @param[in] wallet_path Path to wallet directory. Or (for backwards * compatibility only) a path to a berkeley btree data file inside a wallet * directory. * @param[out] database_filename Filename of berkeley btree data file inside the * wallet directory. * @return A shared pointer to the BerkeleyEnvironment object for the wallet * directory, never empty because ~BerkeleyEnvironment erases the weak pointer * from the g_dbenvs map. * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the * directory path key was not already in the map. */ std::shared_ptr GetWalletEnv(const fs::path &wallet_path, std::string &database_filename) { fs::path env_directory; SplitWalletPath(wallet_path, env_directory, database_filename); LOCK(cs_db); auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr()); if (inserted.second) { auto env = std::make_shared(env_directory.string()); inserted.first->second = env; return env; } return inserted.first->second.lock(); } // // BerkeleyBatch // void BerkeleyEnvironment::Close() { if (!fDbEnvInit) { return; } fDbEnvInit = false; for (auto &db : m_databases) { auto count = mapFileUseCount.find(db.first); assert(count == mapFileUseCount.end() || count->second == 0); BerkeleyDatabase &database = db.second.get(); if (database.m_db) { database.m_db->close(0); database.m_db.reset(); } } + FILE *error_file = nullptr; + dbenv->get_errfile(&error_file); + int ret = dbenv->close(0); if (ret != 0) { LogPrintf("BerkeleyEnvironment::Close: Error %d closing database " "environment: %s\n", ret, DbEnv::strerror(ret)); } if (!fMockDb) { DbEnv(u_int32_t(0)).remove(strPath.c_str(), 0); } + + if (error_file) { + fclose(error_file); + } + + UnlockDirectory(strPath, ".walletlock"); } void BerkeleyEnvironment::Reset() { dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS)); fDbEnvInit = false; fMockDb = false; } BerkeleyEnvironment::BerkeleyEnvironment(const fs::path &dir_path) : strPath(dir_path.string()) { Reset(); } BerkeleyEnvironment::~BerkeleyEnvironment() { LOCK(cs_db); g_dbenvs.erase(strPath); Close(); } bool BerkeleyEnvironment::Open(bool retry) { if (fDbEnvInit) { return true; } boost::this_thread::interruption_point(); fs::path pathIn = strPath; TryCreateDirectories(pathIn); if (!LockDirectory(pathIn, ".walletlock")) { LogPrintf("Cannot obtain a lock on wallet directory %s. Another " "instance of bitcoin may be using it.\n", strPath); return false; } fs::path pathLogDir = pathIn / "database"; TryCreateDirectories(pathLogDir); fs::path pathErrorFile = pathIn / "db.log"; LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string()); unsigned int nEnvFlags = 0; if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) { nEnvFlags |= DB_PRIVATE; } dbenv->set_lg_dir(pathLogDir.string().c_str()); // 1 MiB should be enough for just the wallet dbenv->set_cachesize(0, 0x100000, 1); dbenv->set_lg_bsize(0x10000); dbenv->set_lg_max(1048576); dbenv->set_lk_max_locks(40000); dbenv->set_lk_max_objects(40000); /// debug dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); dbenv->set_flags(DB_AUTO_COMMIT, 1); dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1); dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1); int ret = dbenv->open(strPath.c_str(), DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD | DB_RECOVER | nEnvFlags, S_IRUSR | S_IWUSR); if (ret != 0) { LogPrintf("BerkeleyEnvironment::Open: Error %d opening database " "environment: %s\n", ret, DbEnv::strerror(ret)); int ret2 = dbenv->close(0); if (ret2 != 0) { LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed " "database environment: %s\n", ret2, DbEnv::strerror(ret2)); } Reset(); if (retry) { // try moving the database env out of the way fs::path pathDatabaseBak = pathIn / strprintf("database.%d.bak", GetTime()); try { fs::rename(pathLogDir, pathDatabaseBak); LogPrintf("Moved old %s to %s. Retrying.\n", pathLogDir.string(), pathDatabaseBak.string()); } catch (const fs::filesystem_error &) { // failure is ok (well, not really, but it's not worse than what // we started with) } // try opening it again one more time if (!Open(false /* retry */)) { // if it still fails, it probably means we can't even create the // database env return false; } } else { return false; } } fDbEnvInit = true; fMockDb = false; return true; } //! Construct an in-memory mock Berkeley environment for testing and as a //! place-holder for g_dbenvs emplace BerkeleyEnvironment::BerkeleyEnvironment() { Reset(); boost::this_thread::interruption_point(); LogPrint(BCLog::DB, "BerkeleyEnvironment::MakeMock\n"); dbenv->set_cachesize(1, 0, 1); dbenv->set_lg_bsize(10485760 * 4); dbenv->set_lg_max(10485760); dbenv->set_lk_max_locks(10000); dbenv->set_lk_max_objects(10000); dbenv->set_flags(DB_AUTO_COMMIT, 1); dbenv->log_set_config(DB_LOG_IN_MEMORY, 1); int ret = dbenv->open(nullptr, DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD | DB_PRIVATE, S_IRUSR | S_IWUSR); if (ret > 0) { throw std::runtime_error( strprintf("BerkeleyEnvironment::MakeMock: Error %d opening " "database environment.", ret)); } fDbEnvInit = true; fMockDb = true; } BerkeleyEnvironment::VerifyResult BerkeleyEnvironment::Verify(const std::string &strFile, recoverFunc_type recoverFunc, std::string &out_backup_filename) { LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); Db db(dbenv.get(), 0); int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); if (result == 0) { return VerifyResult::VERIFY_OK; } else if (recoverFunc == nullptr) { return VerifyResult::RECOVER_FAIL; } // Try to recover: bool fRecovered = (*recoverFunc)(fs::path(strPath) / strFile, out_backup_filename); return (fRecovered ? VerifyResult::RECOVER_OK : VerifyResult::RECOVER_FAIL); } bool BerkeleyBatch::Recover(const fs::path &file_path, void *callbackDataIn, bool (*recoverKVcallback)(void *callbackData, CDataStream ssKey, CDataStream ssValue), std::string &newFilename) { std::string filename; std::shared_ptr env = GetWalletEnv(file_path, filename); // Recovery procedure: // Move wallet file to walletfilename.timestamp.bak // Call Salvage with fAggressive=true to get as much data as possible. // Rewrite salvaged data to fresh wallet file. // Set -rescan so any missing transactions will be found. int64_t now = GetTime(); newFilename = strprintf("%s.%d.bak", filename, now); int result = env->dbenv->dbrename(nullptr, filename.c_str(), nullptr, newFilename.c_str(), DB_AUTO_COMMIT); if (result == 0) { LogPrintf("Renamed %s to %s\n", filename, newFilename); } else { LogPrintf("Failed to rename %s to %s\n", filename, newFilename); return false; } std::vector salvagedData; bool fSuccess = env->Salvage(newFilename, true, salvagedData); if (salvagedData.empty()) { LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); return false; } LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); std::unique_ptr pdbCopy = std::make_unique(env->dbenv.get(), 0); int ret = pdbCopy->open(nullptr, // Txn pointer filename.c_str(), // Filename "main", // Logical db name DB_BTREE, // Database type DB_CREATE, // Flags 0); if (ret > 0) { LogPrintf("Cannot create database file %s\n", filename); pdbCopy->close(0); return false; } DbTxn *ptxn = env->TxnBegin(); for (BerkeleyEnvironment::KeyValPair &row : salvagedData) { if (recoverKVcallback) { CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue)) { continue; } } Dbt datKey(&row.first[0], row.first.size()); Dbt datValue(&row.second[0], row.second.size()); int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); if (ret2 > 0) { fSuccess = false; } } ptxn->commit(0); pdbCopy->close(0); return fSuccess; } bool BerkeleyBatch::VerifyEnvironment(const fs::path &file_path, std::string &errorStr) { std::string walletFile; std::shared_ptr env = GetWalletEnv(file_path, walletFile); fs::path walletDir = env->Directory(); LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); LogPrintf("Using wallet %s\n", walletFile); // Wallet file must be a plain filename without a directory if (walletFile != fs::basename(walletFile) + fs::extension(walletFile)) { errorStr = strprintf(_("Wallet %s resides outside wallet directory %s"), walletFile, walletDir.string()); return false; } if (!env->Open(true /* retry */)) { errorStr = strprintf( _("Error initializing wallet database environment %s!"), walletDir); return false; } return true; } bool BerkeleyBatch::VerifyDatabaseFile( const fs::path &file_path, std::string &warningStr, std::string &errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc) { std::string walletFile; std::shared_ptr env = GetWalletEnv(file_path, walletFile); fs::path walletDir = env->Directory(); if (fs::exists(walletDir / walletFile)) { std::string backup_filename; BerkeleyEnvironment::VerifyResult r = env->Verify(walletFile, recoverFunc, backup_filename); if (r == BerkeleyEnvironment::VerifyResult::RECOVER_OK) { warningStr = strprintf( _("Warning: Wallet file corrupt, data salvaged! Original %s " "saved as %s in %s; if your balance or transactions are " "incorrect you should restore from a backup."), walletFile, backup_filename, walletDir); } if (r == BerkeleyEnvironment::VerifyResult::RECOVER_FAIL) { errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile); return false; } } // also return true if files does not exists return true; } /* End of headers, beginning of key/value data */ static const char *HEADER_END = "HEADER=END"; /* End of key/value data */ static const char *DATA_END = "DATA=END"; bool BerkeleyEnvironment::Salvage( const std::string &strFile, bool fAggressive, std::vector &vResult) { LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); u_int32_t flags = DB_SALVAGE; if (fAggressive) { flags |= DB_AGGRESSIVE; } std::stringstream strDump; Db db(dbenv.get(), 0); int result = db.verify(strFile.c_str(), nullptr, &strDump, flags); if (result == DB_VERIFY_BAD) { LogPrintf("BerkeleyEnvironment::Salvage: Database salvage found " "errors, all data may not be recoverable.\n"); if (!fAggressive) { LogPrintf("BerkeleyEnvironment::Salvage: Rerun with aggressive " "mode to ignore errors and continue.\n"); return false; } } if (result != 0 && result != DB_VERIFY_BAD) { LogPrintf("BerkeleyEnvironment::Salvage: Database salvage failed with " "result %d.\n", result); return false; } // Format of bdb dump is ascii lines: // header lines... // HEADER=END // hexadecimal key // hexadecimal value // ... repeated // DATA=END std::string strLine; while (!strDump.eof() && strLine != HEADER_END) { // Skip past header getline(strDump, strLine); } std::string keyHex, valueHex; while (!strDump.eof() && keyHex != DATA_END) { getline(strDump, keyHex); if (keyHex != DATA_END) { if (strDump.eof()) { break; } getline(strDump, valueHex); if (valueHex == DATA_END) { LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Number of " "keys in data does not match number of values.\n"); break; } vResult.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex))); } } if (keyHex != DATA_END) { LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Unexpected end of " "file while reading salvage output.\n"); return false; } return (result == 0); } void BerkeleyEnvironment::CheckpointLSN(const std::string &strFile) { dbenv->txn_checkpoint(0, 0, 0); if (fMockDb) { return; } dbenv->lsn_reset(strFile.c_str(), 0); } BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase &database, const char *pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr) { fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); fFlushOnClose = fFlushOnCloseIn; env = database.env.get(); if (database.IsDummy()) { return; } const std::string &strFilename = database.strFile; bool fCreate = strchr(pszMode, 'c') != nullptr; unsigned int nFlags = DB_THREAD; if (fCreate) { nFlags |= DB_CREATE; } { LOCK(cs_db); if (!env->Open(false /* retry */)) { throw std::runtime_error( "BerkeleyBatch: Failed to open database environment."); } pdb = database.m_db.get(); if (pdb == nullptr) { int ret; std::unique_ptr pdb_temp = std::make_unique(env->dbenv.get(), 0); bool fMockDb = env->IsMock(); if (fMockDb) { DbMpoolFile *mpf = pdb_temp->get_mpf(); ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); if (ret != 0) { throw std::runtime_error( strprintf("BerkeleyBatch: Failed to configure for no " "temp file backing for database %s", strFilename)); } } ret = pdb_temp->open( nullptr, // Txn pointer fMockDb ? nullptr : strFilename.c_str(), // Filename fMockDb ? strFilename.c_str() : "main", // Logical db name DB_BTREE, // Database type nFlags, // Flags 0); if (ret != 0) { throw std::runtime_error( strprintf("BerkeleyBatch: Error %d, can't open database %s", ret, strFilename)); } // Call CheckUniqueFileid on the containing BDB environment to // avoid BDB data consistency bugs that happen when different data // files in the same environment have the same fileid. // // Also call CheckUniqueFileid on all the other g_dbenvs to prevent // bitcoin from opening the same data file through another // environment when the file is referenced through equivalent but // not obviously identical symlinked or hard linked or bind mounted // paths. In the future a more relaxed check for equal inode and // device ids could be done instead, which would allow opening // different backup copies of a wallet at the same time. Maybe even // more ideally, an exclusive lock for accessing the database could // be implemented, so no equality checks are needed at all. (Newer // versions of BDB have an set_lk_exclusive method for this // purpose, but the older version we use does not.) for (const auto &dbenv : g_dbenvs) { CheckUniqueFileid(*dbenv.second.lock().get(), strFilename, *pdb_temp, this->env->m_fileids[strFilename]); } pdb = pdb_temp.release(); database.m_db.reset(pdb); if (fCreate && !Exists(std::string("version"))) { bool fTmp = fReadOnly; fReadOnly = false; WriteVersion(CLIENT_VERSION); fReadOnly = fTmp; } } ++env->mapFileUseCount[strFilename]; strFile = strFilename; } } void BerkeleyBatch::Flush() { if (activeTxn) { return; } // Flush database activity from memory pool to disk log unsigned int nMinutes = 0; if (fReadOnly) { nMinutes = 1; } env->dbenv->txn_checkpoint( nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); } void BerkeleyDatabase::IncrementUpdateCounter() { ++nUpdateCounter; } void BerkeleyBatch::Close() { if (!pdb) { return; } if (activeTxn) { activeTxn->abort(); } activeTxn = nullptr; pdb = nullptr; if (fFlushOnClose) { Flush(); } { LOCK(cs_db); --env->mapFileUseCount[strFile]; } env->m_db_in_use.notify_all(); } void BerkeleyEnvironment::CloseDb(const std::string &strFile) { LOCK(cs_db); auto it = m_databases.find(strFile); assert(it != m_databases.end()); BerkeleyDatabase &database = it->second.get(); if (database.m_db) { // Close the database handle database.m_db->close(0); database.m_db.reset(); } } void BerkeleyEnvironment::ReloadDbEnv() { // Make sure that no Db's are in use AssertLockNotHeld(cs_db); std::unique_lock lock(cs_db); m_db_in_use.wait(lock, [this]() { for (auto &count : mapFileUseCount) { if (count.second > 0) { return false; } } return true; }); std::vector filenames; for (auto it : m_databases) { filenames.push_back(it.first); } // Close the individual Db's for (const std::string &filename : filenames) { CloseDb(filename); } // Reset the environment // This will flush and close the environment Flush(true); Reset(); Open(true); } bool BerkeleyBatch::Rewrite(BerkeleyDatabase &database, const char *pszSkip) { if (database.IsDummy()) { return true; } BerkeleyEnvironment *env = database.env.get(); const std::string &strFile = database.strFile; while (true) { { LOCK(cs_db); if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) { // Flush log data to the dat file env->CloseDb(strFile); env->CheckpointLSN(strFile); env->mapFileUseCount.erase(strFile); bool fSuccess = true; LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile); std::string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} BerkeleyBatch db(database, "r"); std::unique_ptr pdbCopy = std::make_unique(env->dbenv.get(), 0); int ret = pdbCopy->open(nullptr, // Txn pointer strFileRes.c_str(), // Filename "main", // Logical db name DB_BTREE, // Database type DB_CREATE, // Flags 0); if (ret > 0) { LogPrintf("BerkeleyBatch::Rewrite: Can't create " "database file %s\n", strFileRes); fSuccess = false; } Dbc *pcursor = db.GetCursor(); if (pcursor) while (fSuccess) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue); if (ret1 == DB_NOTFOUND) { pcursor->close(); break; } if (ret1 != 0) { pcursor->close(); fSuccess = false; break; } if (pszSkip && strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) { continue; } if (strncmp(ssKey.data(), "\x07version", 8) == 0) { // Update version: ssValue.clear(); ssValue << CLIENT_VERSION; } Dbt datKey(ssKey.data(), ssKey.size()); Dbt datValue(ssValue.data(), ssValue.size()); int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE); if (ret2 > 0) { fSuccess = false; } } if (fSuccess) { db.Close(); env->CloseDb(strFile); if (pdbCopy->close(0)) { fSuccess = false; } } else { pdbCopy->close(0); } } if (fSuccess) { Db dbA(env->dbenv.get(), 0); if (dbA.remove(strFile.c_str(), nullptr, 0)) { fSuccess = false; } Db dbB(env->dbenv.get(), 0); if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0)) { fSuccess = false; } } if (!fSuccess) { LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite " "database file %s\n", strFileRes); } return fSuccess; } } MilliSleep(100); } } void BerkeleyEnvironment::Flush(bool fShutdown) { int64_t nStart = GetTimeMillis(); // Flush log data to the actual data file on all files that are not in use LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); if (!fDbEnvInit) { return; } { LOCK(cs_db); std::map::iterator mi = mapFileUseCount.begin(); while (mi != mapFileUseCount.end()) { std::string strFile = (*mi).first; int nRefCount = (*mi).second; LogPrint( BCLog::DB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); if (nRefCount == 0) { // Move log data to the dat file CloseDb(strFile); LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); dbenv->txn_checkpoint(0, 0, 0); LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s detach\n", strFile); if (!fMockDb) { dbenv->lsn_reset(strFile.c_str(), 0); } LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s closed\n", strFile); mapFileUseCount.erase(mi++); } else { mi++; } } LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); if (fShutdown) { char **listp; if (mapFileUseCount.empty()) { dbenv->log_archive(&listp, DB_ARCH_REMOVE); Close(); if (!fMockDb) { fs::remove_all(fs::path(strPath) / "database"); } } } } } bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase &database) { if (database.IsDummy()) { return true; } bool ret = false; BerkeleyEnvironment *env = database.env.get(); const std::string &strFile = database.strFile; TRY_LOCK(cs_db, lockDb); if (lockDb) { // Don't do this if any databases are in use int nRefCount = 0; std::map::iterator mit = env->mapFileUseCount.begin(); while (mit != env->mapFileUseCount.end()) { nRefCount += (*mit).second; mit++; } if (nRefCount == 0) { boost::this_thread::interruption_point(); std::map::iterator mi = env->mapFileUseCount.find(strFile); if (mi != env->mapFileUseCount.end()) { LogPrint(BCLog::DB, "Flushing %s\n", strFile); int64_t nStart = GetTimeMillis(); // Flush wallet file so it's self contained env->CloseDb(strFile); env->CheckpointLSN(strFile); env->mapFileUseCount.erase(mi++); LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); ret = true; } } } return ret; } bool BerkeleyDatabase::Rewrite(const char *pszSkip) { return BerkeleyBatch::Rewrite(*this, pszSkip); } bool BerkeleyDatabase::Backup(const std::string &strDest) { if (IsDummy()) { return false; } while (true) { { LOCK(cs_db); if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) { // Flush log data to the dat file env->CloseDb(strFile); env->CheckpointLSN(strFile); env->mapFileUseCount.erase(strFile); // Copy wallet file. fs::path pathSrc = env->Directory() / strFile; fs::path pathDest(strDest); if (fs::is_directory(pathDest)) { pathDest /= strFile; } try { if (fs::equivalent(pathSrc, pathDest)) { LogPrintf("cannot backup to wallet source file %s\n", pathDest.string()); return false; } fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); LogPrintf("copied %s to %s\n", strFile, pathDest.string()); return true; } catch (const fs::filesystem_error &e) { LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), e.what()); return false; } } } MilliSleep(100); } } void BerkeleyDatabase::Flush(bool shutdown) { if (!IsDummy()) { env->Flush(shutdown); if (shutdown) { LOCK(cs_db); g_dbenvs.erase(env->Directory().string()); env = nullptr; } else { // TODO: To avoid g_dbenvs.erase erasing the environment prematurely // after the first database shutdown when multiple databases are // open in the same environment, should replace raw database `env` // pointers with shared or weak pointers, or else separate the // database and environment shutdowns so environments can be shut // down after databases. env->m_fileids.erase(strFile); } } } void BerkeleyDatabase::ReloadDbEnv() { if (!IsDummy()) { env->ReloadDbEnv(); } } diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 239ca28c0a..a920045971 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -1,323 +1,332 @@ #!/usr/bin/env python3 # Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multiwallet. Verify that a bitcoind node can load multiple wallet files """ import os import shutil from test_framework.test_framework import BitcoinTestFramework from test_framework.test_node import ErrorMatch from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) class MultiWalletTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 self.supports_cli = True def run_test(self): node = self.nodes[0] data_dir = lambda *p: os.path.join(node.datadir, 'regtest', *p) wallet_dir = lambda *p: data_dir('wallets', *p) def wallet(name): return node.get_wallet_rpc(name) def wallet_file(name): if os.path.isdir(wallet_dir(name)): return wallet_dir(name, "wallet.dat") return wallet_dir(name) # check wallet.dat is created self.stop_nodes() assert_equal(os.path.isfile(wallet_dir('wallet.dat')), True) # create symlink to verify wallet directory path can be referenced # through symlink os.mkdir(wallet_dir('w7')) os.symlink('w7', wallet_dir('w7_symlink')) # rename wallet.dat to make sure plain wallet file paths (as opposed to # directory paths) can be loaded os.rename(wallet_dir("wallet.dat"), wallet_dir("w8")) # create another dummy wallet for use in testing backups later self.start_node(0, []) self.stop_nodes() empty_wallet = os.path.join(self.options.tmpdir, 'empty.dat') os.rename(wallet_dir("wallet.dat"), empty_wallet) # restart node with a mix of wallet names: # w1, w2, w3 - to verify new wallets created when non-existing paths specified # w - to verify wallet name matching works when one wallet path is prefix of another # sub/w5 - to verify relative wallet path is created correctly # extern/w6 - to verify absolute wallet path is created correctly # w7_symlink - to verify symlinked wallet path is initialized correctly # w8 - to verify existing wallet file is loaded correctly # '' - to verify default wallet file is created correctly wallet_names = ['w1', 'w2', 'w3', 'w', 'sub/w5', os.path.join(self.options.tmpdir, 'extern/w6'), 'w7_symlink', 'w8', ''] extra_args = ['-wallet={}'.format(n) for n in wallet_names] self.start_node(0, extra_args) assert_equal(set(node.listwallets()), set(wallet_names)) # check that all requested wallets were created self.stop_node(0) for wallet_name in wallet_names: assert_equal(os.path.isfile(wallet_file(wallet_name)), True) # should not initialize if wallet path can't be created exp_stderr = "boost::filesystem::create_directory: (The system cannot find the path specified|Not a directory):" self.nodes[0].assert_start_raises_init_error( ['-wallet=wallet.dat/bad'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) self.nodes[0].assert_start_raises_init_error( ['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" does not exist') self.nodes[0].assert_start_raises_init_error( ['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" is a relative path', cwd=data_dir()) self.nodes[0].assert_start_raises_init_error( ['-walletdir=debug.log'], 'Error: Specified -walletdir "debug.log" is not a directory', cwd=data_dir()) # should not initialize if there are duplicate wallets self.nodes[0].assert_start_raises_init_error( ['-wallet=w1', '-wallet=w1'], 'Error: Error loading wallet w1. Duplicate -wallet filename specified.') # should not initialize if one wallet is a copy of another shutil.copyfile(wallet_dir('w8'), wallet_dir('w8_copy')) exp_stderr = r"BerkeleyBatch: Can't open database w8_copy \(duplicates fileid \w+ from w8\)" self.nodes[0].assert_start_raises_init_error( ['-wallet=w8', '-wallet=w8_copy'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) # should not initialize if wallet file is a symlink os.symlink('w8', wallet_dir('w8_symlink')) self.nodes[0].assert_start_raises_init_error( ['-wallet=w8_symlink'], r'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX) # should not initialize if the specified walletdir does not exist self.nodes[0].assert_start_raises_init_error( ['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist') # should not initialize if the specified walletdir is not a directory not_a_dir = wallet_dir('notadir') open(not_a_dir, 'a', encoding="utf8").close() self.nodes[0].assert_start_raises_init_error( ['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory') # if wallets/ doesn't exist, datadir should be the default wallet dir wallet_dir2 = data_dir('walletdir') os.rename(wallet_dir(), wallet_dir2) self.start_node(0, ['-wallet=w4', '-wallet=w5']) assert_equal(set(node.listwallets()), {"w4", "w5"}) w5 = wallet("w5") w5.generate(1) # now if wallets/ exists again, but the rootdir is specified as the walletdir, w4 and w5 should still be loaded os.rename(wallet_dir2, wallet_dir()) self.restart_node(0, ['-wallet=w4', '-wallet=w5', '-walletdir=' + data_dir()]) assert_equal(set(node.listwallets()), {"w4", "w5"}) w5 = wallet("w5") w5_info = w5.getwalletinfo() assert_equal(w5_info['immature_balance'], 50) competing_wallet_dir = os.path.join( self.options.tmpdir, 'competing_walletdir') os.mkdir(competing_wallet_dir) self.restart_node(0, ['-walletdir=' + competing_wallet_dir]) exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\"!" self.nodes[1].assert_start_raises_init_error( ['-walletdir=' + competing_wallet_dir], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) self.restart_node(0, extra_args) wallets = [wallet(w) for w in wallet_names] wallet_bad = wallet("bad") # check wallet names and balances wallets[0].generate(1) for wallet_name, wallet in zip(wallet_names, wallets): info = wallet.getwalletinfo() assert_equal(info['immature_balance'], 50 if wallet is wallets[0] else 0) assert_equal(info['walletname'], wallet_name) # accessing invalid wallet fails assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", wallet_bad.getwalletinfo) # accessing wallet RPC without using wallet endpoint fails assert_raises_rpc_error(-19, "Wallet file not specified (must request wallet RPC through /wallet/ uri-path).", node.getwalletinfo) w1, w2, w3, w4, *_ = wallets w1.generate(101) assert_equal(w1.getbalance(), 100) assert_equal(w2.getbalance(), 0) assert_equal(w3.getbalance(), 0) assert_equal(w4.getbalance(), 0) w1.sendtoaddress(w2.getnewaddress(), 1) w1.sendtoaddress(w3.getnewaddress(), 2) w1.sendtoaddress(w4.getnewaddress(), 3) w1.generate(1) assert_equal(w2.getbalance(), 1) assert_equal(w3.getbalance(), 2) assert_equal(w4.getbalance(), 3) batch = w1.batch([w1.getblockchaininfo.get_request(), w1.getwalletinfo.get_request()]) assert_equal(batch[0]["result"]["chain"], "regtest") assert_equal(batch[1]["result"]["walletname"], "w1") self.log.info('Check for per-wallet settxfee call') assert_equal(w1.getwalletinfo()['paytxfee'], 0) assert_equal(w2.getwalletinfo()['paytxfee'], 0) w2.settxfee(4.0) assert_equal(w1.getwalletinfo()['paytxfee'], 0) assert_equal(w2.getwalletinfo()['paytxfee'], 4.0) self.log.info("Test dynamic wallet loading") self.restart_node(0, ['-nowallet']) assert_equal(node.listwallets(), []) assert_raises_rpc_error(-32601, "Method not found", node.getwalletinfo) self.log.info("Load first wallet") loadwallet_name = node.loadwallet(wallet_names[0]) assert_equal(loadwallet_name['name'], wallet_names[0]) assert_equal(node.listwallets(), wallet_names[0:1]) node.getwalletinfo() w1 = node.get_wallet_rpc(wallet_names[0]) w1.getwalletinfo() self.log.info("Load second wallet") loadwallet_name = node.loadwallet(wallet_names[1]) assert_equal(loadwallet_name['name'], wallet_names[1]) assert_equal(node.listwallets(), wallet_names[0:2]) assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo) w2 = node.get_wallet_rpc(wallet_names[1]) w2.getwalletinfo() self.log.info("Load remaining wallets") for wallet_name in wallet_names[2:]: loadwallet_name = self.nodes[0].loadwallet(wallet_name) assert_equal(loadwallet_name['name'], wallet_name) assert_equal(set(self.nodes[0].listwallets()), set(wallet_names)) # Fail to load if wallet doesn't exist assert_raises_rpc_error(-18, 'Wallet wallets not found.', self.nodes[0].loadwallet, 'wallets') # Fail to load duplicate wallets assert_raises_rpc_error(-4, 'Wallet file verification failed: Error loading wallet w1. Duplicate -wallet filename specified.', self.nodes[0].loadwallet, wallet_names[0]) # Fail to load duplicate wallets by different ways (directory and filepath) assert_raises_rpc_error(-4, "Wallet file verification failed: Error loading wallet wallet.dat. Duplicate -wallet filename specified.", self.nodes[0].loadwallet, 'wallet.dat') # Fail to load if one wallet is a copy of another assert_raises_rpc_error(-1, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') # Fail to load if one wallet is a copy of another. # Test this twice to make sure that we don't re-introduce https://github.com/bitcoin/bitcoin/issues/14304 assert_raises_rpc_error(-1, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') # Fail to load if wallet file is a symlink assert_raises_rpc_error(-4, "Wallet file verification failed: Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink') # Fail to load if a directory is specified that doesn't contain a wallet os.mkdir(wallet_dir('empty_wallet_dir')) assert_raises_rpc_error(-18, "Directory empty_wallet_dir does not contain a wallet.dat file", self.nodes[0].loadwallet, 'empty_wallet_dir') self.log.info("Test dynamic wallet creation.") # Fail to create a wallet if it already exists. assert_raises_rpc_error(-4, "Wallet w2 already exists.", self.nodes[0].createwallet, 'w2') # Successfully create a wallet with a new name loadwallet_name = self.nodes[0].createwallet('w9') assert_equal(loadwallet_name['name'], 'w9') w9 = node.get_wallet_rpc('w9') assert_equal(w9.getwalletinfo()['walletname'], 'w9') assert 'w9' in self.nodes[0].listwallets() # Successfully create a wallet using a full path new_wallet_dir = os.path.join(self.options.tmpdir, 'new_walletdir') new_wallet_name = os.path.join(new_wallet_dir, 'w10') loadwallet_name = self.nodes[0].createwallet(new_wallet_name) assert_equal(loadwallet_name['name'], new_wallet_name) w10 = node.get_wallet_rpc(new_wallet_name) assert_equal(w10.getwalletinfo()['walletname'], new_wallet_name) assert new_wallet_name in self.nodes[0].listwallets() self.log.info("Test dynamic wallet unloading") # Test `unloadwallet` errors assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].unloadwallet) assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", self.nodes[0].unloadwallet, "dummy") assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", node.get_wallet_rpc("dummy").unloadwallet) assert_raises_rpc_error(-8, "Cannot unload the requested wallet", w1.unloadwallet, "w2"), # Successfully unload the specified wallet name self.nodes[0].unloadwallet("w1") assert 'w1' not in self.nodes[0].listwallets() # Successfully unload the wallet referenced by the request endpoint w2.unloadwallet() assert 'w2' not in self.nodes[0].listwallets() # Successfully unload all wallets for wallet_name in self.nodes[0].listwallets(): self.nodes[0].unloadwallet(wallet_name) assert_equal(self.nodes[0].listwallets(), []) assert_raises_rpc_error(-32601, "Method not found (wallet method is disabled because no wallet is loaded)", self.nodes[0].getwalletinfo) # Successfully load a previously unloaded wallet self.nodes[0].loadwallet('w1') assert_equal(self.nodes[0].listwallets(), ['w1']) assert_equal(w1.getwalletinfo()['walletname'], 'w1') # Test backing up and restoring wallets self.log.info("Test wallet backup") self.restart_node(0, ['-nowallet']) for wallet_name in wallet_names: self.nodes[0].loadwallet(wallet_name) for wallet_name in wallet_names: rpc = self.nodes[0].get_wallet_rpc(wallet_name) addr = rpc.getnewaddress() backup = os.path.join(self.options.tmpdir, 'backup.dat') rpc.backupwallet(backup) self.nodes[0].unloadwallet(wallet_name) shutil.copyfile(empty_wallet, wallet_file(wallet_name)) self.nodes[0].loadwallet(wallet_name) assert_equal(rpc.getaddressinfo(addr)['ismine'], False) self.nodes[0].unloadwallet(wallet_name) shutil.copyfile(backup, wallet_file(wallet_name)) self.nodes[0].loadwallet(wallet_name) assert_equal(rpc.getaddressinfo(addr)['ismine'], True) + # Test .walletlock file is closed + self.start_node(1) + wallet = os.path.join(self.options.tmpdir, 'my_wallet') + self.nodes[0].createwallet(wallet) + assert_raises_rpc_error(-4, "Error initializing wallet database environment", + self.nodes[1].loadwallet, wallet) + self.nodes[0].unloadwallet(wallet) + self.nodes[1].loadwallet(wallet) + if __name__ == '__main__': MultiWalletTest().main()