diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index dc241927f4..88df8b757e 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -1,302 +1,308 @@ // Copyright (c) 2011-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) #include "config/bitcoin-config.h" #endif #include "fs.h" #include "guiutil.h" #include "intro.h" #include "ui_intro.h" #include "util.h" #include #include #include #include static const uint64_t GB_BYTES = 1000000000LL; /** * Minimum free space (in GB) needed for data directory. */ static const uint64_t BLOCK_CHAIN_SIZE = 200; /** * Minimum free space (in GB) needed for data directory when pruned; Does not * include prune target. */ static const uint64_t CHAIN_STATE_SIZE = 4; /** * Total required space (in GB) depending on user choice (prune, not prune). */ static uint64_t requiredSpace; /* Check free space asynchronously to prevent hanging the UI thread. Up to one request to check a path is in flight to this thread; when the check() function runs, the current path is requested from the associated Intro object. The reply is sent back through a signal. This ensures that no queue of checking requests is built up while the user is still entering the path, and that always the most recently entered path is checked as soon as the thread becomes available. */ class FreespaceChecker : public QObject { Q_OBJECT public: explicit FreespaceChecker(Intro *intro); enum Status { ST_OK, ST_ERROR }; public Q_SLOTS: void check(); Q_SIGNALS: void reply(int status, const QString &message, quint64 available); private: Intro *intro; }; #include "intro.moc" FreespaceChecker::FreespaceChecker(Intro *_intro) { this->intro = _intro; } void FreespaceChecker::check() { QString dataDirStr = intro->getPathToCheck(); fs::path dataDir = GUIUtil::qstringToBoostPath(dataDirStr); uint64_t freeBytesAvailable = 0; int replyStatus = ST_OK; QString replyMessage = tr("A new data directory will be created."); /* Find first parent that exists, so that fs::space does not fail */ fs::path parentDir = dataDir; fs::path parentDirOld = fs::path(); while (parentDir.has_parent_path() && !fs::exists(parentDir)) { parentDir = parentDir.parent_path(); /* Check if we make any progress, break if not to prevent an infinite * loop here */ if (parentDirOld == parentDir) break; parentDirOld = parentDir; } try { freeBytesAvailable = fs::space(parentDir).available; if (fs::exists(dataDir)) { if (fs::is_directory(dataDir)) { QString separator = "" + QDir::toNativeSeparators("/") + tr("name") + ""; replyStatus = ST_OK; replyMessage = tr("Directory already exists. Add %1 if you " "intend to create a new directory here.") .arg(separator); } else { replyStatus = ST_ERROR; replyMessage = tr("Path already exists, and is not a directory."); } } } catch (const fs::filesystem_error &) { /* Parent directory does not exist or is not accessible */ replyStatus = ST_ERROR; replyMessage = tr("Cannot create data directory here."); } Q_EMIT reply(replyStatus, replyMessage, freeBytesAvailable); } Intro::Intro(QWidget *parent) : QDialog(parent), ui(new Ui::Intro), thread(0), signalled(false) { ui->setupUi(this); ui->welcomeLabel->setText(ui->welcomeLabel->text().arg(tr(PACKAGE_NAME))); ui->storageLabel->setText(ui->storageLabel->text().arg(tr(PACKAGE_NAME))); uint64_t pruneTarget = std::max(0, gArgs.GetArg("-prune", 0)); requiredSpace = BLOCK_CHAIN_SIZE; if (pruneTarget) { uint64_t prunedGBs = std::ceil(pruneTarget * 1024 * 1024.0 / GB_BYTES); if (prunedGBs <= requiredSpace) { requiredSpace = prunedGBs; } } requiredSpace += CHAIN_STATE_SIZE; ui->sizeWarningLabel->setText( ui->sizeWarningLabel->text().arg(tr(PACKAGE_NAME)).arg(requiredSpace)); startThread(); } Intro::~Intro() { delete ui; /* Ensure thread is finished before it is deleted */ Q_EMIT stopThread(); thread->wait(); } QString Intro::getDataDirectory() { return ui->dataDirectory->text(); } void Intro::setDataDirectory(const QString &dataDir) { ui->dataDirectory->setText(dataDir); if (dataDir == getDefaultDataDirectory()) { ui->dataDirDefault->setChecked(true); ui->dataDirectory->setEnabled(false); ui->ellipsisButton->setEnabled(false); } else { ui->dataDirCustom->setChecked(true); ui->dataDirectory->setEnabled(true); ui->ellipsisButton->setEnabled(true); } } QString Intro::getDefaultDataDirectory() { return GUIUtil::boostPathToQString(GetDefaultDataDir()); } bool Intro::pickDataDirectory() { QSettings settings; /* If data directory provided on command line, no need to look at settings or show a picking dialog */ if (!gArgs.GetArg("-datadir", "").empty()) return true; /* 1) Default data directory for operating system */ QString dataDir = getDefaultDataDirectory(); /* 2) Allow QSettings to override default dir */ dataDir = settings.value("strDataDir", dataDir).toString(); if (!fs::exists(GUIUtil::qstringToBoostPath(dataDir)) || gArgs.GetBoolArg("-choosedatadir", DEFAULT_CHOOSE_DATADIR) || settings.value("fReset", false).toBool() || gArgs.GetBoolArg("-resetguisettings", false)) { /* If current default data directory does not exist, let the user choose * one */ Intro intro; intro.setDataDirectory(dataDir); intro.setWindowIcon(QIcon(":icons/bitcoin")); while (true) { if (!intro.exec()) { /* Cancel clicked */ return false; } dataDir = intro.getDataDirectory(); try { - TryCreateDirectories(GUIUtil::qstringToBoostPath(dataDir)); + if (TryCreateDirectories( + GUIUtil::qstringToBoostPath(dataDir))) { + // If a new data directory has been created, make wallets + // subdirectory too + TryCreateDirectories(GUIUtil::qstringToBoostPath(dataDir) / + "wallets"); + } break; } catch (const fs::filesystem_error &) { QMessageBox::critical(0, tr(PACKAGE_NAME), tr("Error: Specified data directory " "\"%1\" cannot be created.") .arg(dataDir)); /* fall through, back to choosing screen */ } } settings.setValue("strDataDir", dataDir); settings.setValue("fReset", false); } /* Only override -datadir if different from the default, to make it possible * to * override -datadir in the bitcoin.conf file in the default data directory * (to be consistent with bitcoind behavior) */ if (dataDir != getDefaultDataDirectory()) { // use OS locale for path setting gArgs.SoftSetArg("-datadir", GUIUtil::qstringToBoostPath(dataDir).string()); } return true; } void Intro::setStatus(int status, const QString &message, quint64 bytesAvailable) { switch (status) { case FreespaceChecker::ST_OK: ui->errorMessage->setText(message); ui->errorMessage->setStyleSheet(""); break; case FreespaceChecker::ST_ERROR: ui->errorMessage->setText(tr("Error") + ": " + message); ui->errorMessage->setStyleSheet("QLabel { color: #800000 }"); break; } /* Indicate number of bytes available */ if (status == FreespaceChecker::ST_ERROR) { ui->freeSpace->setText(""); } else { QString freeString = tr("%n GB of free space available", "", bytesAvailable / GB_BYTES); if (bytesAvailable < requiredSpace * GB_BYTES) { freeString += " " + tr("(of %n GB needed)", "", requiredSpace); ui->freeSpace->setStyleSheet("QLabel { color: #800000 }"); } else { ui->freeSpace->setStyleSheet(""); } ui->freeSpace->setText(freeString + "."); } /* Don't allow confirm in ERROR state */ ui->buttonBox->button(QDialogButtonBox::Ok) ->setEnabled(status != FreespaceChecker::ST_ERROR); } void Intro::on_dataDirectory_textChanged(const QString &dataDirStr) { /* Disable OK button until check result comes in */ ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(false); checkPath(dataDirStr); } void Intro::on_ellipsisButton_clicked() { QString dir = QDir::toNativeSeparators(QFileDialog::getExistingDirectory( 0, "Choose data directory", ui->dataDirectory->text())); if (!dir.isEmpty()) ui->dataDirectory->setText(dir); } void Intro::on_dataDirDefault_clicked() { setDataDirectory(getDefaultDataDirectory()); } void Intro::on_dataDirCustom_clicked() { ui->dataDirectory->setEnabled(true); ui->ellipsisButton->setEnabled(true); } void Intro::startThread() { thread = new QThread(this); FreespaceChecker *executor = new FreespaceChecker(this); executor->moveToThread(thread); connect(executor, SIGNAL(reply(int, QString, quint64)), this, SLOT(setStatus(int, QString, quint64))); connect(this, SIGNAL(requestCheck()), executor, SLOT(check())); /* make sure executor object is deleted in its own thread */ connect(this, SIGNAL(stopThread()), executor, SLOT(deleteLater())); connect(this, SIGNAL(stopThread()), thread, SLOT(quit())); thread->start(); } void Intro::checkPath(const QString &dataDir) { mutex.lock(); pathToCheck = dataDir; if (!signalled) { signalled = true; Q_EMIT requestCheck(); } mutex.unlock(); } QString Intro::getPathToCheck() { QString retval; mutex.lock(); retval = pathToCheck; signalled = false; /* new request can be queued now */ mutex.unlock(); return retval; } diff --git a/src/util.cpp b/src/util.cpp index 0a904e27fb..4dd51bd4b2 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -1,747 +1,750 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) #include "config/bitcoin-config.h" #endif #include "util.h" #include "chainparamsbase.h" #include "fs.h" #include "random.h" #include "serialize.h" #include "utilstrencodings.h" #include "utiltime.h" #include #if (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) #include #include #endif #ifndef WIN32 // for posix_fallocate #ifdef __linux__ #ifdef _POSIX_C_SOURCE #undef _POSIX_C_SOURCE #endif #define _POSIX_C_SOURCE 200112L #endif // __linux__ #include #include #include #include #else #ifdef _MSC_VER #pragma warning(disable : 4786) #pragma warning(disable : 4804) #pragma warning(disable : 4805) #pragma warning(disable : 4717) #endif #ifdef _WIN32_WINNT #undef _WIN32_WINNT #endif #define _WIN32_WINNT 0x0501 #ifdef _WIN32_IE #undef _WIN32_IE #endif #define _WIN32_IE 0x0501 #define WIN32_LEAN_AND_MEAN 1 #ifndef NOMINMAX #define NOMINMAX #endif #include /* for _commit */ #include #endif #ifdef HAVE_SYS_PRCTL_H #include #endif #ifdef HAVE_MALLOPT_ARENA_MAX #include #endif #include #include #include #include // Application startup time (used for uptime calculation) const int64_t nStartupTime = GetTime(); const char *const BITCOIN_CONF_FILENAME = "bitcoin.conf"; const char *const BITCOIN_PID_FILENAME = "bitcoind.pid"; ArgsManager gArgs; CTranslationInterface translationInterface; /** Init OpenSSL library multithreading support */ static CCriticalSection **ppmutexOpenSSL; void locking_callback(int mode, int i, const char *file, int line) NO_THREAD_SAFETY_ANALYSIS { if (mode & CRYPTO_LOCK) { ENTER_CRITICAL_SECTION(*ppmutexOpenSSL[i]); } else { LEAVE_CRITICAL_SECTION(*ppmutexOpenSSL[i]); } } // Init class CInit { public: CInit() { // Init OpenSSL library multithreading support. ppmutexOpenSSL = (CCriticalSection **)OPENSSL_malloc( CRYPTO_num_locks() * sizeof(CCriticalSection *)); for (int i = 0; i < CRYPTO_num_locks(); i++) { ppmutexOpenSSL[i] = new CCriticalSection(); } CRYPTO_set_locking_callback(locking_callback); // OpenSSL can optionally load a config file which lists optional // loadable modules and engines. We don't use them so we don't require // the config. However some of our libs may call functions which attempt // to load the config file, possibly resulting in an exit() or crash if // it is missing or corrupt. Explicitly tell OpenSSL not to try to load // the file. The result for our libs will be that the config appears to // have been loaded and there are no modules/engines available. OPENSSL_no_config(); #ifdef WIN32 // Seed OpenSSL PRNG with current contents of the screen. RAND_screen(); #endif // Seed OpenSSL PRNG with performance counter. RandAddSeed(); } ~CInit() { // Securely erase the memory used by the PRNG. RAND_cleanup(); // Shutdown OpenSSL library multithreading support. CRYPTO_set_locking_callback(nullptr); for (int i = 0; i < CRYPTO_num_locks(); i++) { delete ppmutexOpenSSL[i]; } OPENSSL_free(ppmutexOpenSSL); } } instance_of_cinit; /** * Interpret a string argument as a boolean. * * The definition of atoi() requires that non-numeric string values like "foo", * return 0. This means that if a user unintentionally supplies a non-integer * argument here, the return value is always false. This means that -foo=false * does what the user probably expects, but -foo=true is well defined but does * not do what they probably expected. * * The return value of atoi() is undefined when given input not representable as * an int. On most systems this means string value between "-2147483648" and * "2147483647" are well defined (this method will return true). Setting * -txindex=2147483648 on most systems, however, is probably undefined. * * For a more extensive discussion of this topic (and a wide range of opinions * on the Right Way to change this code), see PR12713. */ static bool InterpretBool(const std::string &strValue) { if (strValue.empty()) { return true; } return (atoi(strValue) != 0); } /** * Interpret -nofoo as if the user supplied -foo=0. * * This method also tracks when the -no form was supplied, and treats "-foo" as * a negated option when this happens. This can be later checked using the * IsArgNegated() method. One use case for this is to have a way to disable * options that are not normally boolean (e.g. using -nodebuglogfile to request * that debug log output is not sent to any file at all). */ void ArgsManager::InterpretNegatedOption(std::string &key, std::string &val) { if (key.substr(0, 3) == "-no") { bool bool_val = InterpretBool(val); if (!bool_val) { // Double negatives like -nofoo=0 are supported (but discouraged) LogPrintf( "Warning: parsed potentially confusing double-negative %s=%s\n", key, val); } key.erase(1, 2); m_negated_args.insert(key); val = bool_val ? "0" : "1"; } else { // In an invocation like "bitcoind -nofoo -foo" we want to unmark -foo // as negated when we see the second option. m_negated_args.erase(key); } } void ArgsManager::ParseParameters(int argc, const char *const argv[]) { LOCK(cs_args); mapArgs.clear(); mapMultiArgs.clear(); m_negated_args.clear(); for (int i = 1; i < argc; i++) { std::string key(argv[i]); std::string val; size_t is_index = key.find('='); if (is_index != std::string::npos) { val = key.substr(is_index + 1); key.erase(is_index); } #ifdef WIN32 std::transform(key.begin(), key.end(), key.begin(), ::tolower); if (key[0] == '/') { key[0] = '-'; } #endif if (key[0] != '-') { break; } // Transform --foo to -foo if (key.length() > 1 && key[1] == '-') { key.erase(0, 1); } // Transform -nofoo to -foo=0 InterpretNegatedOption(key, val); mapArgs[key] = val; mapMultiArgs[key].push_back(val); } } std::vector ArgsManager::GetArgs(const std::string &strArg) const { LOCK(cs_args); auto it = mapMultiArgs.find(strArg); if (it != mapMultiArgs.end()) { return it->second; } return {}; } bool ArgsManager::IsArgSet(const std::string &strArg) const { LOCK(cs_args); return mapArgs.count(strArg); } bool ArgsManager::IsArgNegated(const std::string &strArg) const { LOCK(cs_args); return m_negated_args.find(strArg) != m_negated_args.end(); } std::string ArgsManager::GetArg(const std::string &strArg, const std::string &strDefault) const { LOCK(cs_args); auto it = mapArgs.find(strArg); if (it != mapArgs.end()) { return it->second; } return strDefault; } int64_t ArgsManager::GetArg(const std::string &strArg, int64_t nDefault) const { LOCK(cs_args); auto it = mapArgs.find(strArg); if (it != mapArgs.end()) { return atoi64(it->second); } return nDefault; } bool ArgsManager::GetBoolArg(const std::string &strArg, bool fDefault) const { LOCK(cs_args); auto it = mapArgs.find(strArg); if (it != mapArgs.end()) { return InterpretBool(it->second); } return fDefault; } bool ArgsManager::SoftSetArg(const std::string &strArg, const std::string &strValue) { LOCK(cs_args); if (IsArgSet(strArg)) { return false; } ForceSetArg(strArg, strValue); return true; } bool ArgsManager::SoftSetBoolArg(const std::string &strArg, bool fValue) { if (fValue) { return SoftSetArg(strArg, std::string("1")); } else { return SoftSetArg(strArg, std::string("0")); } } void ArgsManager::ForceSetArg(const std::string &strArg, const std::string &strValue) { LOCK(cs_args); mapArgs[strArg] = strValue; mapMultiArgs[strArg] = {strValue}; } /** * This function is only used for testing purpose so * so we should not worry about element uniqueness and * integrity of mapMultiArgs data structure */ void ArgsManager::ForceSetMultiArg(const std::string &strArg, const std::string &strValue) { LOCK(cs_args); if (mapArgs.count(strArg) == 0) { mapArgs[strArg] = strValue; } mapMultiArgs[strArg].push_back(strValue); } void ArgsManager::ClearArg(const std::string &strArg) { LOCK(cs_args); mapArgs.erase(strArg); mapMultiArgs.erase(strArg); } bool HelpRequested(const ArgsManager &args) { return args.IsArgSet("-?") || args.IsArgSet("-h") || args.IsArgSet("-help"); } static const int screenWidth = 79; static const int optIndent = 2; static const int msgIndent = 7; std::string HelpMessageGroup(const std::string &message) { return std::string(message) + std::string("\n\n"); } std::string HelpMessageOpt(const std::string &option, const std::string &message) { return std::string(optIndent, ' ') + std::string(option) + std::string("\n") + std::string(msgIndent, ' ') + FormatParagraph(message, screenWidth - msgIndent, msgIndent) + std::string("\n\n"); } static std::string FormatException(const std::exception *pex, const char *pszThread) { #ifdef WIN32 char pszModule[MAX_PATH] = ""; GetModuleFileNameA(nullptr, pszModule, sizeof(pszModule)); #else const char *pszModule = "bitcoin"; #endif if (pex) { return strprintf("EXCEPTION: %s \n%s \n%s in %s \n", typeid(*pex).name(), pex->what(), pszModule, pszThread); } else { return strprintf("UNKNOWN EXCEPTION \n%s in %s \n", pszModule, pszThread); } } void PrintExceptionContinue(const std::exception *pex, const char *pszThread) { std::string message = FormatException(pex, pszThread); LogPrintf("\n\n************************\n%s\n", message); fprintf(stderr, "\n\n************************\n%s\n", message.c_str()); } fs::path GetDefaultDataDir() { // Windows < Vista: C:\Documents and Settings\Username\Application Data\Bitcoin // Windows >= Vista: C:\Users\Username\AppData\Roaming\Bitcoin // Mac: ~/Library/Application Support/Bitcoin // Unix: ~/.bitcoin #ifdef WIN32 // Windows return GetSpecialFolderPath(CSIDL_APPDATA) / "Bitcoin"; #else fs::path pathRet; char *pszHome = getenv("HOME"); if (pszHome == nullptr || strlen(pszHome) == 0) { pathRet = fs::path("/"); } else { pathRet = fs::path(pszHome); } #ifdef MAC_OSX // Mac return pathRet / "Library/Application Support/Bitcoin"; #else // Unix return pathRet / ".bitcoin"; #endif #endif } static fs::path pathCached; static fs::path pathCachedNetSpecific; static CCriticalSection csPathCached; const fs::path &GetDataDir(bool fNetSpecific) { LOCK(csPathCached); fs::path &path = fNetSpecific ? pathCachedNetSpecific : pathCached; // This can be called during exceptions by LogPrintf(), so we cache the // value so we don't have to do memory allocations after that. if (!path.empty()) { return path; } if (gArgs.IsArgSet("-datadir")) { path = fs::system_complete(gArgs.GetArg("-datadir", "")); if (!fs::is_directory(path)) { path = ""; return path; } } else { path = GetDefaultDataDir(); } if (fNetSpecific) { path /= BaseParams().DataDir(); } - fs::create_directories(path); + if (fs::create_directories(path)) { + // This is the first run, create wallets subdirectory too + fs::create_directories(path / "wallets"); + } return path; } void ClearDatadirCache() { LOCK(csPathCached); pathCached = fs::path(); pathCachedNetSpecific = fs::path(); } fs::path GetConfigFile(const std::string &confPath) { fs::path pathConfigFile(confPath); if (!pathConfigFile.is_complete()) { pathConfigFile = GetDataDir(false) / pathConfigFile; } return pathConfigFile; } void ArgsManager::ReadConfigStream(std::istream &stream) { LOCK(cs_args); std::set setOptions; setOptions.insert("*"); for (boost::program_options::detail::config_file_iterator it(stream, setOptions), end; it != end; ++it) { // Don't overwrite existing settings so command line settings override // bitcoin.conf std::string strKey = std::string("-") + it->string_key; std::string strValue = it->value[0]; InterpretNegatedOption(strKey, strValue); if (mapArgs.count(strKey) == 0) { mapArgs[strKey] = strValue; } mapMultiArgs[strKey].push_back(strValue); } } void ArgsManager::ReadConfigFile(const std::string &confPath) { fs::ifstream stream(GetConfigFile(confPath)); // ok to not have a config file if (stream.good()) { ReadConfigStream(stream); } // If datadir is changed in .conf file: ClearDatadirCache(); if (!fs::is_directory(GetDataDir(false))) { throw std::runtime_error( strprintf("specified data directory \"%s\" does not exist.", gArgs.GetArg("-datadir", "").c_str())); } } std::string ArgsManager::GetChainName() const { bool fRegTest = GetBoolArg("-regtest", false); bool fTestNet = GetBoolArg("-testnet", false); if (fTestNet && fRegTest) { throw std::runtime_error( "Invalid combination of -regtest and -testnet."); } if (fRegTest) { return CBaseChainParams::REGTEST; } if (fTestNet) { return CBaseChainParams::TESTNET; } return CBaseChainParams::MAIN; } #ifndef WIN32 fs::path GetPidFile() { fs::path pathPidFile(gArgs.GetArg("-pid", BITCOIN_PID_FILENAME)); if (!pathPidFile.is_complete()) { pathPidFile = GetDataDir() / pathPidFile; } return pathPidFile; } void CreatePidFile(const fs::path &path, pid_t pid) { FILE *file = fsbridge::fopen(path, "w"); if (file) { fprintf(file, "%d\n", pid); fclose(file); } } #endif bool RenameOver(fs::path src, fs::path dest) { #ifdef WIN32 return MoveFileExA(src.string().c_str(), dest.string().c_str(), MOVEFILE_REPLACE_EXISTING) != 0; #else int rc = std::rename(src.string().c_str(), dest.string().c_str()); return (rc == 0); #endif /* WIN32 */ } /** * Ignores exceptions thrown by Boost's create_directories if the requested * directory exists. Specifically handles case where path p exists, but it * wasn't possible for the user to write to the parent directory. */ bool TryCreateDirectories(const fs::path &p) { try { return fs::create_directories(p); } catch (const fs::filesystem_error &) { if (!fs::exists(p) || !fs::is_directory(p)) { throw; } } // create_directory didn't create the directory, it had to have existed // already. return false; } void FileCommit(FILE *file) { // Harmless if redundantly called. fflush(file); #ifdef WIN32 HANDLE hFile = (HANDLE)_get_osfhandle(_fileno(file)); FlushFileBuffers(hFile); #else #if defined(__linux__) || defined(__NetBSD__) fdatasync(fileno(file)); #elif defined(__APPLE__) && defined(F_FULLFSYNC) fcntl(fileno(file), F_FULLFSYNC, 0); #else fsync(fileno(file)); #endif #endif } bool TruncateFile(FILE *file, unsigned int length) { #if defined(WIN32) return _chsize(_fileno(file), length) == 0; #else return ftruncate(fileno(file), length) == 0; #endif } /** * This function tries to raise the file descriptor limit to the requested * number. It returns the actual file descriptor limit (which may be more or * less than nMinFD) */ int RaiseFileDescriptorLimit(int nMinFD) { #if defined(WIN32) return 2048; #else struct rlimit limitFD; if (getrlimit(RLIMIT_NOFILE, &limitFD) != -1) { if (limitFD.rlim_cur < (rlim_t)nMinFD) { limitFD.rlim_cur = nMinFD; if (limitFD.rlim_cur > limitFD.rlim_max) { limitFD.rlim_cur = limitFD.rlim_max; } setrlimit(RLIMIT_NOFILE, &limitFD); getrlimit(RLIMIT_NOFILE, &limitFD); } return limitFD.rlim_cur; } // getrlimit failed, assume it's fine. return nMinFD; #endif } /** * This function tries to make a particular range of a file allocated * (corresponding to disk space) it is advisory, and the range specified in the * arguments will never contain live data. */ void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length) { #if defined(WIN32) // Windows-specific version. HANDLE hFile = (HANDLE)_get_osfhandle(_fileno(file)); LARGE_INTEGER nFileSize; int64_t nEndPos = (int64_t)offset + length; nFileSize.u.LowPart = nEndPos & 0xFFFFFFFF; nFileSize.u.HighPart = nEndPos >> 32; SetFilePointerEx(hFile, nFileSize, 0, FILE_BEGIN); SetEndOfFile(hFile); #elif defined(MAC_OSX) // OSX specific version. fstore_t fst; fst.fst_flags = F_ALLOCATECONTIG; fst.fst_posmode = F_PEOFPOSMODE; fst.fst_offset = 0; fst.fst_length = (off_t)offset + length; fst.fst_bytesalloc = 0; if (fcntl(fileno(file), F_PREALLOCATE, &fst) == -1) { fst.fst_flags = F_ALLOCATEALL; fcntl(fileno(file), F_PREALLOCATE, &fst); } ftruncate(fileno(file), fst.fst_length); #elif defined(__linux__) // Version using posix_fallocate. off_t nEndPos = (off_t)offset + length; posix_fallocate(fileno(file), 0, nEndPos); #else // Fallback version // TODO: just write one byte per block static const char buf[65536] = {}; fseek(file, offset, SEEK_SET); while (length > 0) { unsigned int now = 65536; if (length < now) { now = length; } // Allowed to fail; this function is advisory anyway. fwrite(buf, 1, now, file); length -= now; } #endif } #ifdef WIN32 fs::path GetSpecialFolderPath(int nFolder, bool fCreate) { char pszPath[MAX_PATH] = ""; if (SHGetSpecialFolderPathA(nullptr, pszPath, nFolder, fCreate)) { return fs::path(pszPath); } LogPrintf( "SHGetSpecialFolderPathA() failed, could not obtain requested path.\n"); return fs::path(""); } #endif void runCommand(const std::string &strCommand) { if (strCommand.empty()) { return; } int nErr = ::system(strCommand.c_str()); if (nErr) { LogPrintf("runCommand error: system(%s) returned %d\n", strCommand, nErr); } } void RenameThread(const char *name) { #if defined(PR_SET_NAME) // Only the first 15 characters are used (16 - NUL terminator) ::prctl(PR_SET_NAME, name, 0, 0, 0); #elif (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) pthread_set_name_np(pthread_self(), name); #elif defined(MAC_OSX) pthread_setname_np(name); #else // Prevent warnings for unused parameters... (void)name; #endif } void SetupEnvironment() { #ifdef HAVE_MALLOPT_ARENA_MAX // glibc-specific: On 32-bit systems set the number of arenas to 1. By // default, since glibc 2.10, the C library will create up to two heap // arenas per core. This is known to cause excessive virtual address space // usage in our usage. Work around it by setting the maximum number of // arenas to 1. if (sizeof(void *) == 4) { mallopt(M_ARENA_MAX, 1); } #endif // On most POSIX systems (e.g. Linux, but not BSD) the environment's locale may // be invalid, in which case the "C" locale is used as fallback. #if !defined(WIN32) && !defined(MAC_OSX) && !defined(__FreeBSD__) && \ !defined(__OpenBSD__) try { // Raises a runtime error if current locale is invalid. std::locale(""); } catch (const std::runtime_error &) { setenv("LC_ALL", "C", 1); } #endif // The path locale is lazy initialized and to avoid deinitialization errors // in multithreading environments, it is set explicitly by the main thread. // A dummy locale is used to extract the internal default locale, used by // fs::path, which is then used to explicitly imbue the path. std::locale loc = fs::path::imbue(std::locale::classic()); fs::path::imbue(loc); } bool SetupNetworking() { #ifdef WIN32 // Initialize Windows Sockets. WSADATA wsadata; int ret = WSAStartup(MAKEWORD(2, 2), &wsadata); if (ret != NO_ERROR || LOBYTE(wsadata.wVersion) != 2 || HIBYTE(wsadata.wVersion) != 2) { return false; } #endif return true; } int GetNumCores() { return boost::thread::physical_concurrency(); } std::string CopyrightHolders(const std::string &strPrefix) { return strPrefix + strprintf(_(COPYRIGHT_HOLDERS), _(COPYRIGHT_HOLDERS_SUBSTITUTION)); } // Obtain the application startup time (used for uptime calculation) int64_t GetStartupTime() { return nStartupTime; } diff --git a/test/functional/keypool-topup.py b/test/functional/keypool-topup.py index 4f1ed65346..d261dc4c18 100755 --- a/test/functional/keypool-topup.py +++ b/test/functional/keypool-topup.py @@ -1,81 +1,81 @@ #!/usr/bin/env python3 # Copyright (c) 2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test HD Wallet keypool restore function. Two nodes. Node1 is under test. Node0 is providing transactions and generating blocks. - Start node1, shutdown and backup wallet. - Generate 110 keys (enough to drain the keypool). Store key 90 (in the initial keypool) and key 110 (beyond the initial keypool). Send funds to key 90 and key 110. - Stop node1, clear the datadir, move wallet file back into the datadir and restart node1. - connect node1 to node0. Verify that they sync and node1 receives its funds.""" import shutil from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes_bi, sync_blocks, ) class KeypoolRestoreTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 self.extra_args = [['-usehd=0'], ['-usehd=1', '-keypool=100', '-keypoolmin=20']] def run_test(self): self.tmpdir = self.options.tmpdir self.nodes[0].generate(101) self.log.info("Make backup of wallet") self.stop_node(1) - shutil.copyfile(self.tmpdir + "/node1/regtest/wallet.dat", + shutil.copyfile(self.tmpdir + "/node1/regtest/wallets/wallet.dat", self.tmpdir + "/wallet.bak") self.start_node(1, self.extra_args[1]) connect_nodes_bi(self.nodes[0], self.nodes[1]) self.log.info("Generate keys for wallet") for _ in range(90): addr_oldpool = self.nodes[1].getnewaddress() for _ in range(20): addr_extpool = self.nodes[1].getnewaddress() self.log.info("Send funds to wallet") self.nodes[0].sendtoaddress(addr_oldpool, 10) self.nodes[0].generate(1) self.nodes[0].sendtoaddress(addr_extpool, 5) self.nodes[0].generate(1) sync_blocks(self.nodes) self.log.info("Restart node with wallet backup") self.stop_node(1) shutil.copyfile(self.tmpdir + "/wallet.bak", - self.tmpdir + "/node1/regtest/wallet.dat") + self.tmpdir + "/node1/regtest/wallets/wallet.dat") self.log.info("Verify keypool is restored and balance is correct") self.start_node(1, self.extra_args[1]) connect_nodes_bi(self.nodes[0], self.nodes[1]) self.sync_all() assert_equal(self.nodes[1].getbalance(), 15) assert_equal(self.nodes[1].listtransactions() [0]['category'], "receive") # Check that we have marked all keys up to the used keypool key as used assert_equal(self.nodes[1].validateaddress( self.nodes[1].getnewaddress())['hdkeypath'], "m/0'/0'/110'") if __name__ == '__main__': KeypoolRestoreTest().main() diff --git a/test/functional/multiwallet.py b/test/functional/multiwallet.py index 59dc1ee498..28378d1af6 100755 --- a/test/functional/multiwallet.py +++ b/test/functional/multiwallet.py @@ -1,123 +1,132 @@ #!/usr/bin/env python3 # Copyright (c) 2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multiwallet. Verify that a bitcoind node can load multiple wallet files """ import os import shutil from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error class MultiWalletTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 self.extra_args = [ ['-wallet=w1', '-wallet=w2', '-wallet=w3', '-wallet=w']] def run_test(self): assert_equal(set(self.nodes[0].listwallets()), {"w1", "w2", "w3", "w"}) self.stop_node(0) # should not initialize if there are duplicate wallets self.assert_start_raises_init_error( 0, ['-wallet=w1', '-wallet=w1'], 'Error loading wallet w1. Duplicate -wallet filename specified.') # should not initialize if wallet file is a directory - os.mkdir(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w11')) + wallet_dir = os.path.join( + self.options.tmpdir, 'node0', 'regtest', 'wallets') + os.mkdir(os.path.join(wallet_dir, 'w11')) self.assert_start_raises_init_error( 0, ['-wallet=w11'], 'Error loading wallet w11. -wallet filename must be a regular file.') # should not initialize if one wallet is a copy of another - shutil.copyfile(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w2'), - os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w22')) + shutil.copyfile(os.path.join(wallet_dir, 'w2'), + os.path.join(wallet_dir, 'w22')) self.assert_start_raises_init_error( 0, ['-wallet=w2', '-wallet=w22'], 'duplicates fileid') # should not initialize if wallet file is a symlink - wallet_dir = os.path.abspath(os.path.join( - self.options.tmpdir, 'node0', 'regtest')) os.symlink(os.path.join(wallet_dir, 'w1'), os.path.join(wallet_dir, 'w12')) self.assert_start_raises_init_error( 0, ['-wallet=w12'], 'Error loading wallet w12. -wallet filename must be a regular file.') # should not initialize if the specified walletdir does not exist self.assert_start_raises_init_error( 0, ['-walletdir=bad'], 'Error: Specified wallet directory "bad" does not exist.') - # running the node with specified walletdir should only have the default wallet in it - os.mkdir(os.path.join(self.options.tmpdir, - 'node0', 'regtest', 'walletdir')) + # if wallets/ doesn't exist, datadir should be the default wallet dir + wallet_dir2 = os.path.join( + self.options.tmpdir, 'node0', 'regtest', 'walletdir') + os.rename(wallet_dir, wallet_dir2) + self.start_node(0, ['-wallet=w4', '-wallet=w5']) + assert_equal(set(self.nodes[0].listwallets()), {"w4", "w5"}) + w5 = self.nodes[0].get_wallet_rpc("w5") + w5.generate(1) + self.stop_node(0) + + # now if wallets/ exists again, but the rootdir is specified as the walletdir, w4 and w5 should still be loaded + os.rename(wallet_dir2, wallet_dir) self.start_node(0, ['-wallet=w4', '-wallet=w5', '-walletdir=' + - os.path.join(self.options.tmpdir, 'node0', 'regtest', 'walletdir')]) + os.path.join(self.options.tmpdir, 'node0', 'regtest')]) assert_equal(set(self.nodes[0].listwallets()), {"w4", "w5"}) w5 = self.nodes[0].get_wallet_rpc("w5") w5_info = w5.getwalletinfo() - assert_equal(w5_info['immature_balance'], 0) + assert_equal(w5_info['immature_balance'], 50) self.stop_node(0) self.start_node(0, self.extra_args[0]) w1 = self.nodes[0].get_wallet_rpc("w1") w2 = self.nodes[0].get_wallet_rpc("w2") w3 = self.nodes[0].get_wallet_rpc("w3") w4 = self.nodes[0].get_wallet_rpc("w") wallet_bad = self.nodes[0].get_wallet_rpc("bad") w1.generate(1) # accessing invalid wallet fails assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", wallet_bad.getwalletinfo) # accessing wallet RPC without using wallet endpoint fails assert_raises_rpc_error(-19, "Wallet file not specified (must request wallet RPC through /wallet/ uri-path).", self.nodes[0].getwalletinfo) # check w1 wallet balance w1_info = w1.getwalletinfo() assert_equal(w1_info['immature_balance'], 50) w1_name = w1_info['walletname'] assert_equal(w1_name, "w1") # check w2 wallet balance w2_info = w2.getwalletinfo() assert_equal(w2_info['immature_balance'], 0) w2_name = w2_info['walletname'] assert_equal(w2_name, "w2") w3_name = w3.getwalletinfo()['walletname'] assert_equal(w3_name, "w3") w4_name = w4.getwalletinfo()['walletname'] assert_equal(w4_name, "w") w1.generate(101) assert_equal(w1.getbalance(), 100) assert_equal(w2.getbalance(), 0) assert_equal(w3.getbalance(), 0) assert_equal(w4.getbalance(), 0) w1.sendtoaddress(w2.getnewaddress(), 1) w1.sendtoaddress(w3.getnewaddress(), 2) w1.sendtoaddress(w4.getnewaddress(), 3) w1.generate(1) assert_equal(w2.getbalance(), 1) assert_equal(w3.getbalance(), 2) assert_equal(w4.getbalance(), 3) batch = w1.batch([w1.getblockchaininfo.get_request(), w1.getwalletinfo.get_request()]) assert_equal(batch[0]["result"]["chain"], "regtest") assert_equal(batch[1]["result"]["walletname"], "w1") if __name__ == '__main__': MultiWalletTest().main() diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 9b6fe44a02..9b48eda404 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,509 +1,510 @@ #!/usr/bin/env python3 # Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" from collections import deque from enum import Enum import logging import argparse import os import pdb import shutil import sys import tempfile import time import traceback from .authproxy import JSONRPCException from . import coverage from .test_node import TestNode from .util import ( MAX_NODES, PortSeed, assert_equal, check_json_precision, connect_nodes_bi, disconnect_nodes, initialize_datadir, log_filename, p2p_port, rpc_port, set_node_times, sync_blocks, sync_mempools, ) class TestStatus(Enum): PASSED = 1 FAILED = 2 SKIPPED = 3 TEST_EXIT_PASSED = 0 TEST_EXIT_FAILED = 1 TEST_EXIT_SKIPPED = 77 class BitcoinTestFramework(): """Base class for a bitcoin test script. Individual bitcoin test scripts should subclass this class and override the set_test_params() and run_test() methods. Individual tests can also override the following methods to customize the test setup: - add_options() - setup_chain() - setup_network() - setup_nodes() The __init__() and main() methods should not be overridden. This class also contains various public and private helper methods.""" def __init__(self): """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method""" self.setup_clean_chain = False self.nodes = [] self.mocktime = 0 def main(self): """Main function. This should not be overridden by the subclass test scripts.""" parser = argparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") parser.add_argument("--noshutdown", dest="noshutdown", default=False, action="store_true", help="Don't stop bitcoinds after the test execution") parser.add_argument("--srcdir", dest="srcdir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../../../src"), help="Source directory containing bitcoind/bitcoin-cli (default: %(default)s)") parser.add_argument("--cachedir", dest="cachedir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../../cache"), help="Directory for caching pregenerated datadirs") parser.add_argument("--tmpdir", dest="tmpdir", help="Root directory for datadirs") parser.add_argument("-l", "--loglevel", dest="loglevel", default="INFO", help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory.") parser.add_argument("--tracerpc", dest="trace_rpc", default=False, action="store_true", help="Print out all RPC calls as they are made") parser.add_argument("--portseed", dest="port_seed", default=os.getpid(), type=int, help="The seed to use for assigning port numbers (default: current process id)") parser.add_argument("--coveragedir", dest="coveragedir", help="Write tested RPC commands into this directory") parser.add_argument("--configfile", dest="configfile", help="Location of the test framework config file") parser.add_argument("--pdbonfailure", dest="pdbonfailure", default=False, action="store_true", help="Attach a python debugger if test fails") self.add_options(parser) self.options = parser.parse_args() self.set_test_params() assert hasattr( self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" PortSeed.n = self.options.port_seed os.environ['PATH'] = self.options.srcdir + ":" + \ self.options.srcdir + "/qt:" + os.environ['PATH'] check_json_precision() self.options.cachedir = os.path.abspath(self.options.cachedir) # Set up temp directory and start logging if self.options.tmpdir: self.options.tmpdir = os.path.abspath(self.options.tmpdir) os.makedirs(self.options.tmpdir, exist_ok=False) else: self.options.tmpdir = tempfile.mkdtemp(prefix="test") self._start_logging() success = TestStatus.FAILED try: self.setup_chain() self.setup_network() self.run_test() success = TestStatus.PASSED except JSONRPCException as e: self.log.exception("JSONRPC error") except SkipTest as e: self.log.warning("Test Skipped: %s" % e.message) success = TestStatus.SKIPPED except AssertionError as e: self.log.exception("Assertion failed") except KeyError as e: self.log.exception("Key error") except Exception as e: self.log.exception("Unexpected exception caught during testing") except KeyboardInterrupt as e: self.log.warning("Exiting after keyboard interrupt") if success == TestStatus.FAILED and self.options.pdbonfailure: print("Testcase failed. Attaching python debugger. Enter ? for help") pdb.set_trace() if not self.options.noshutdown: self.log.info("Stopping nodes") if self.nodes: self.stop_nodes() else: self.log.info( "Note: bitcoinds were not stopped and may still be running") if not self.options.nocleanup and not self.options.noshutdown and success != TestStatus.FAILED: self.log.info("Cleaning up") shutil.rmtree(self.options.tmpdir) else: self.log.warning("Not cleaning up dir %s" % self.options.tmpdir) if os.getenv("PYTHON_DEBUG", ""): # Dump the end of the debug logs, to aid in debugging rare # travis failures. import glob filenames = [self.options.tmpdir + "/test_framework.log"] filenames += glob.glob(self.options.tmpdir + "/node*/regtest/debug.log") MAX_LINES_TO_PRINT = 1000 for fn in filenames: try: with open(fn, 'r') as f: print("From", fn, ":") print("".join(deque(f, MAX_LINES_TO_PRINT))) except OSError: print("Opening file %s failed." % fn) traceback.print_exc() if success == TestStatus.PASSED: self.log.info("Tests successful") sys.exit(TEST_EXIT_PASSED) elif success == TestStatus.SKIPPED: self.log.info("Test skipped") sys.exit(TEST_EXIT_SKIPPED) else: self.log.error( "Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) logging.shutdown() sys.exit(TEST_EXIT_FAILED) # Methods to override in subclass test scripts. def set_test_params(self): """Tests must this method to change default values for number of nodes, topology, etc""" raise NotImplementedError def add_options(self, parser): """Override this method to add command-line options to the test""" pass def setup_chain(self): """Override this method to customize blockchain setup""" self.log.info("Initializing test directory " + self.options.tmpdir) if self.setup_clean_chain: self._initialize_chain_clean() else: self._initialize_chain() def setup_network(self): """Override this method to customize test network topology""" self.setup_nodes() # Connect the nodes as a "chain". This allows us # to split the network between nodes 1 and 2 to get # two halves that can work on competing chains. for i in range(self.num_nodes - 1): connect_nodes_bi(self.nodes[i], self.nodes[i + 1]) self.sync_all() def setup_nodes(self): """Override this method to customize test node setup""" extra_args = None if hasattr(self, "extra_args"): extra_args = self.extra_args self.add_nodes(self.num_nodes, extra_args) self.start_nodes() def run_test(self): """Tests must override this method to define test logic""" raise NotImplementedError # Public helper methods. These can be accessed by the subclass test scripts. def add_nodes(self, num_nodes, extra_args=None, rpchost=None, timewait=None, binary=None): """Instantiate TestNode objects""" if extra_args is None: extra_args = [[]] * num_nodes if binary is None: binary = [None] * num_nodes assert_equal(len(extra_args), num_nodes) assert_equal(len(binary), num_nodes) for i in range(num_nodes): self.nodes.append(TestNode(i, self.options.tmpdir, extra_args[i], rpchost, rpc_port=rpc_port(i), p2p_port=p2p_port(i), timewait=timewait, binary=binary[i], stderr=None, mocktime=self.mocktime, coverage_dir=self.options.coveragedir)) def start_node(self, i, extra_args=None, stderr=None): """Start a bitcoind""" node = self.nodes[i] node.start(extra_args, stderr) node.wait_for_rpc_connection() if self.options.coveragedir is not None: coverage.write_all_rpc_commands(self.options.coveragedir, node.rpc) def start_nodes(self, extra_args=None): """Start multiple bitcoinds""" if extra_args is None: extra_args = [None] * self.num_nodes assert_equal(len(extra_args), self.num_nodes) try: for i, node in enumerate(self.nodes): node.start(extra_args[i]) for node in self.nodes: node.wait_for_rpc_connection() except: # If one node failed to start, stop the others self.stop_nodes() raise if self.options.coveragedir is not None: for node in self.nodes: coverage.write_all_rpc_commands( self.options.coveragedir, node.rpc) def stop_node(self, i): """Stop a bitcoind test node""" self.nodes[i].stop_node() self.nodes[i].wait_until_stopped() def stop_nodes(self): """Stop multiple bitcoind test nodes""" for node in self.nodes: # Issue RPC to stop nodes node.stop_node() for node in self.nodes: # Wait for nodes to stop node.wait_until_stopped() def restart_node(self, i, extra_args=None): """Stop and start a test node""" self.stop_node(i) self.start_node(i, extra_args) def assert_start_raises_init_error(self, i, extra_args=None, expected_msg=None): with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr: try: self.start_node(i, extra_args, stderr=log_stderr) self.stop_node(i) except Exception as e: assert 'bitcoind exited' in str(e) # node must have shutdown self.nodes[i].running = False self.nodes[i].process = None if expected_msg is not None: log_stderr.seek(0) stderr = log_stderr.read().decode('utf-8') if expected_msg not in stderr: raise AssertionError( "Expected error \"" + expected_msg + "\" not found in:\n" + stderr) else: if expected_msg is None: assert_msg = "bitcoind should have exited with an error" else: assert_msg = "bitcoind should have exited with expected error " + expected_msg raise AssertionError(assert_msg) def wait_for_node_exit(self, i, timeout): self.nodes[i].process.wait(timeout) def split_network(self): """ Split the network of four nodes into nodes 0/1 and 2/3. """ disconnect_nodes(self.nodes[1], self.nodes[2]) disconnect_nodes(self.nodes[2], self.nodes[1]) self.sync_all([self.nodes[:2], self.nodes[2:]]) def join_network(self): """ Join the (previously split) network halves together. """ connect_nodes_bi(self.nodes[1], self.nodes[2]) self.sync_all() def sync_all(self, node_groups=None): if not node_groups: node_groups = [self.nodes] for group in node_groups: sync_blocks(group) sync_mempools(group) def enable_mocktime(self, mocktime): """Enable mocktime for the script. mocktime may be needed for scripts that use the cached version of the blockchain. If the cached version of the blockchain is used without mocktime then the mempools will not sync due to IBD. """ if self.mocktime == 0: self.mocktime = mocktime else: self.log.warning("mocktime overriden by test.") def disable_mocktime(self): self.mocktime = 0 # Private helper methods. These should not be accessed by the subclass test scripts. def _start_logging(self): # Add logger and logging handlers self.log = logging.getLogger('TestFramework') self.log.setLevel(logging.DEBUG) # Create file handler to log all messages fh = logging.FileHandler(self.options.tmpdir + '/test_framework.log') fh.setLevel(logging.DEBUG) # Create console handler to log messages to stderr. By default this # logs only error messages, but can be configured with --loglevel. ch = logging.StreamHandler(sys.stdout) # User can provide log level as a number or string (eg DEBUG). loglevel # was caught as a string, so try to convert it to an int ll = int(self.options.loglevel) if self.options.loglevel.isdigit( ) else self.options.loglevel.upper() ch.setLevel(ll) # Format logs the same as bitcoind's debug.log with microprecision (so log files can be concatenated and sorted) formatter = logging.Formatter( fmt='%(asctime)s.%(msecs)03d000 %(name)s (%(levelname)s): %(message)s', datefmt='%Y-%m-%d %H:%M:%S') formatter.converter = time.gmtime fh.setFormatter(formatter) ch.setFormatter(formatter) # add the handlers to the logger self.log.addHandler(fh) self.log.addHandler(ch) if self.options.trace_rpc: rpc_logger = logging.getLogger("BitcoinRPC") rpc_logger.setLevel(logging.DEBUG) rpc_handler = logging.StreamHandler(sys.stdout) rpc_handler.setLevel(logging.DEBUG) rpc_logger.addHandler(rpc_handler) def _initialize_chain(self): """Initialize a pre-mined blockchain for use by the test. Create a cache of a 200-block-long chain (with wallet) for MAX_NODES Afterward, create num_nodes copies from the cache.""" assert self.num_nodes <= MAX_NODES create_cache = False for i in range(MAX_NODES): if not os.path.isdir(os.path.join(self.options.cachedir, 'node' + str(i))): create_cache = True break if create_cache: self.log.debug("Creating data directories from cached datadir") # find and delete old cache directories if any exist for i in range(MAX_NODES): if os.path.isdir(os.path.join(self.options.cachedir, "node" + str(i))): shutil.rmtree(os.path.join( self.options.cachedir, "node" + str(i))) # Create cache directories, run bitcoinds: for i in range(MAX_NODES): datadir = initialize_datadir(self.options.cachedir, i) args = [os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir=" + datadir, "-discover=0"] if i > 0: args.append("-connect=127.0.0.1:" + str(p2p_port(0))) self.nodes.append(TestNode(i, self.options.cachedir, extra_args=[], host=None, rpc_port=rpc_port(i), p2p_port=p2p_port(i), timewait=None, binary=None, stderr=None, mocktime=self.mocktime, coverage_dir=None)) self.nodes[i].args = args self.start_node(i) # Wait for RPC connections to be ready for node in self.nodes: node.wait_for_rpc_connection() # Create a 200-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. # Note: To preserve compatibility with older versions of # initialize_chain, only 4 nodes will generate coins. # # blocks are created with timestamps 10 minutes apart # starting from 2010 minutes in the past self.enable_mocktime(int(time.time()) - (201 * 10 * 60)) for i in range(2): for peer in range(4): for j in range(25): set_node_times(self.nodes, self.mocktime) self.nodes[peer].generate(1) self.mocktime += 10 * 60 # Must sync before next peer starts generating blocks sync_blocks(self.nodes) # Shut them down, and clean up cache directories: self.stop_nodes() self.nodes = [] self.disable_mocktime() for i in range(MAX_NODES): os.remove(log_filename(self.options.cachedir, i, "debug.log")) - os.remove(log_filename(self.options.cachedir, i, "db.log")) + os.remove(log_filename( + self.options.cachedir, i, "wallets/db.log")) os.remove(log_filename(self.options.cachedir, i, "peers.dat")) os.remove(log_filename( self.options.cachedir, i, "fee_estimates.dat")) for i in range(self.num_nodes): from_dir = os.path.join(self.options.cachedir, "node" + str(i)) to_dir = os.path.join(self.options.tmpdir, "node" + str(i)) shutil.copytree(from_dir, to_dir) # Overwrite port/rpcport in bitcoin.conf initialize_datadir(self.options.tmpdir, i) def _initialize_chain_clean(self): """Initialize empty blockchain for use by the test. Create an empty blockchain and num_nodes wallets. Useful if a test case wants complete control over initialization.""" for i in range(self.num_nodes): initialize_datadir(self.options.tmpdir, i) class ComparisonTestFramework(BitcoinTestFramework): """Test framework for doing p2p comparison testing Sets up some bitcoind binaries: - 1 binary: test binary - 2 binaries: 1 test binary, 1 ref binary - n>2 binaries: 1 test binary, n-1 ref binaries""" def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True def add_options(self, parser): parser.add_argument("--testbinary", dest="testbinary", default=os.getenv("BITCOIND", "bitcoind"), help="bitcoind binary to test") parser.add_argument("--refbinary", dest="refbinary", default=os.getenv("BITCOIND", "bitcoind"), help="bitcoind binary to use for reference nodes (if any)") def setup_network(self): extra_args = [['-whitelist=127.0.0.1']] * self.num_nodes if hasattr(self, "extra_args"): extra_args = self.extra_args self.add_nodes(self.num_nodes, extra_args, binary=[self.options.testbinary] + [self.options.refbinary] * (self.num_nodes - 1)) self.start_nodes() class SkipTest(Exception): """This exception is raised to skip a test""" def __init__(self, message): self.message = message diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 8f7aae24cc..7fa4bd5305 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -1,137 +1,137 @@ #!/usr/bin/env python3 # Copyright (c) 2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Hierarchical Deterministic wallet function.""" from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes_bi, ) import shutil import os class WalletHDTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 self.extra_args = [['-usehd=0'], ['-usehd=1', '-keypool=0']] def run_test(self): tmpdir = self.options.tmpdir # Make sure can't switch off usehd after wallet creation self.stop_node(1) self.assert_start_raises_init_error( 1, ['-usehd=0'], 'already existing HD wallet') self.start_node(1) connect_nodes_bi(self.nodes[0], self.nodes[1]) # Make sure we use hd, keep masterkeyid masterkeyid = self.nodes[1].getwalletinfo()['hdmasterkeyid'] assert_equal(len(masterkeyid), 40) # create an internal key change_addr = self.nodes[1].getrawchangeaddress() change_addrV = self.nodes[1].validateaddress(change_addr) # first internal child key assert_equal(change_addrV["hdkeypath"], "m/0'/1'/0'") # Import a non-HD private key in the HD wallet non_hd_add = self.nodes[0].getnewaddress() self.nodes[1].importprivkey(self.nodes[0].dumpprivkey(non_hd_add)) # This should be enough to keep the master key and the non-HD key self.nodes[1].backupwallet(tmpdir + "/hd.bak") #self.nodes[1].dumpwallet(tmpdir + "/hd.dump") # Derive some HD addresses and remember the last # Also send funds to each add self.nodes[0].generate(101) hd_add = None num_hd_adds = 300 for i in range(num_hd_adds): hd_add = self.nodes[1].getnewaddress() hd_info = self.nodes[1].validateaddress(hd_add) assert_equal(hd_info["hdkeypath"], "m/0'/0'/" + str(i) + "'") assert_equal(hd_info["hdmasterkeyid"], masterkeyid) self.nodes[0].sendtoaddress(hd_add, 1) self.nodes[0].generate(1) self.nodes[0].sendtoaddress(non_hd_add, 1) self.nodes[0].generate(1) # create an internal key (again) change_addr = self.nodes[1].getrawchangeaddress() change_addrV = self.nodes[1].validateaddress(change_addr) # second internal child key assert_equal(change_addrV["hdkeypath"], "m/0'/1'/1'") self.sync_all() assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1) self.log.info("Restore backup ...") self.stop_node(1) # we need to delete the complete regtest directory # otherwise node1 would auto-recover all funds in flag the keypool keys as used shutil.rmtree(os.path.join(tmpdir, "node1/regtest/blocks")) shutil.rmtree(os.path.join(tmpdir, "node1/regtest/chainstate")) shutil.copyfile(os.path.join(tmpdir, "hd.bak"), - os.path.join(tmpdir, "node1/regtest/wallet.dat")) + os.path.join(tmpdir, "node1/regtest/wallets/wallet.dat")) self.start_node(1) # Assert that derivation is deterministic hd_add_2 = None for _ in range(num_hd_adds): hd_add_2 = self.nodes[1].getnewaddress() hd_info_2 = self.nodes[1].validateaddress(hd_add_2) assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/" + str(_) + "'") assert_equal(hd_info_2["hdmasterkeyid"], masterkeyid) assert_equal(hd_add, hd_add_2) connect_nodes_bi(self.nodes[0], self.nodes[1]) self.sync_all() # Needs rescan self.stop_node(1) self.start_node(1, extra_args=self.extra_args[1] + ['-rescan']) assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1) # Try a RPC based rescan self.stop_node(1) shutil.rmtree(os.path.join(tmpdir, "node1/regtest/blocks")) shutil.rmtree(os.path.join(tmpdir, "node1/regtest/chainstate")) shutil.copyfile(os.path.join(tmpdir, "hd.bak"), os.path.join(tmpdir, "node1/regtest/wallet.dat")) self.start_node(1, extra_args=self.extra_args[1]) connect_nodes_bi(self.nodes[0], self.nodes[1]) self.sync_all() out = self.nodes[1].rescanblockchain(0, 1) assert_equal(out['start_height'], 0) assert_equal(out['stop_height'], 1) out = self.nodes[1].rescanblockchain(2, 4) assert_equal(out['start_height'], 2) assert_equal(out['stop_height'], 4) out = self.nodes[1].rescanblockchain(3) assert_equal(out['start_height'], 3) assert_equal(out['stop_height'], self.nodes[1].getblockcount()) out = self.nodes[1].rescanblockchain() assert_equal(out['start_height'], 0) assert_equal(out['stop_height'], self.nodes[1].getblockcount()) assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1) # send a tx and make sure its using the internal chain for the changeoutput txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1) outs = self.nodes[1].decoderawtransaction( self.nodes[1].gettransaction(txid)['hex'])['vout'] keypath = "" for out in outs: if out['value'] != 1: keypath = self.nodes[1].validateaddress( out['scriptPubKey']['addresses'][0])['hdkeypath'] assert_equal(keypath[0:7], "m/0'/1'") if __name__ == '__main__': WalletHDTest().main() diff --git a/test/functional/walletbackup.py b/test/functional/walletbackup.py index 407a1fab49..6d45100b93 100755 --- a/test/functional/walletbackup.py +++ b/test/functional/walletbackup.py @@ -1,211 +1,211 @@ #!/usr/bin/env python3 # Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet backup features. Test case is: 4 nodes. 1 2 and 3 send transactions between each other, fourth node is a miner. 1 2 3 each mine a block to start, then Miner creates 100 blocks so 1 2 3 each have 50 mature coins to spend. Then 5 iterations of 1/2/3 sending coins amongst themselves to get transactions in the wallets, and the miner mining one block. Wallets are backed up using dumpwallet/backupwallet. Then 5 more iterations of transactions and mining a block. Miner then generates 101 more blocks, so any transaction fees paid mature. Sanity check: Sum(1,2,3,4 balances) == 114*50 1/2/3 are shutdown, and their wallets erased. Then restore using wallet.dat backup. And confirm 1/2/3/4 balances are same as before. Shutdown again, restore using importwallet, and confirm again balances are correct. """ from random import randint import shutil from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * class WalletBackupTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.setup_clean_chain = True # nodes 1, 2,3 are spenders, let's give them a keypool=100 self.extra_args = [["-keypool=100"], ["-keypool=100"], ["-keypool=100"], []] def setup_network(self, split=False): self.setup_nodes() connect_nodes(self.nodes[0], self.nodes[3]) connect_nodes(self.nodes[1], self.nodes[3]) connect_nodes(self.nodes[2], self.nodes[3]) connect_nodes(self.nodes[2], self.nodes[0]) self.sync_all() def one_send(self, from_node, to_address): if (randint(1, 2) == 1): amount = Decimal(randint(1, 10)) / Decimal(10) self.nodes[from_node].sendtoaddress(to_address, amount) def do_one_round(self): a0 = self.nodes[0].getnewaddress() a1 = self.nodes[1].getnewaddress() a2 = self.nodes[2].getnewaddress() self.one_send(0, a1) self.one_send(0, a2) self.one_send(1, a0) self.one_send(1, a2) self.one_send(2, a0) self.one_send(2, a1) # Have the miner (node3) mine a block. # Must sync mempools before mining. sync_mempools(self.nodes) self.nodes[3].generate(1) sync_blocks(self.nodes) # As above, this mirrors the original bash test. def start_three(self): self.start_node(0) self.start_node(1) self.start_node(2) connect_nodes(self.nodes[0], self.nodes[3]) connect_nodes(self.nodes[1], self.nodes[3]) connect_nodes(self.nodes[2], self.nodes[3]) connect_nodes(self.nodes[2], self.nodes[0]) def stop_three(self): self.stop_node(0) self.stop_node(1) self.stop_node(2) def erase_three(self): - os.remove(self.options.tmpdir + "/node0/regtest/wallet.dat") - os.remove(self.options.tmpdir + "/node1/regtest/wallet.dat") - os.remove(self.options.tmpdir + "/node2/regtest/wallet.dat") + os.remove(self.options.tmpdir + "/node0/regtest/wallets/wallet.dat") + os.remove(self.options.tmpdir + "/node1/regtest/wallets/wallet.dat") + os.remove(self.options.tmpdir + "/node2/regtest/wallets/wallet.dat") def run_test(self): self.log.info("Generating initial blockchain") self.nodes[0].generate(1) sync_blocks(self.nodes) self.nodes[1].generate(1) sync_blocks(self.nodes) self.nodes[2].generate(1) sync_blocks(self.nodes) self.nodes[3].generate(100) sync_blocks(self.nodes) assert_equal(self.nodes[0].getbalance(), 50) assert_equal(self.nodes[1].getbalance(), 50) assert_equal(self.nodes[2].getbalance(), 50) assert_equal(self.nodes[3].getbalance(), 0) self.log.info("Creating transactions") # Five rounds of sending each other transactions. for i in range(5): self.do_one_round() self.log.info("Backing up") tmpdir = self.options.tmpdir self.nodes[0].backupwallet(tmpdir + "/node0/wallet.bak") self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.dump") self.nodes[1].backupwallet(tmpdir + "/node1/wallet.bak") self.nodes[1].dumpwallet(tmpdir + "/node1/wallet.dump") self.nodes[2].backupwallet(tmpdir + "/node2/wallet.bak") self.nodes[2].dumpwallet(tmpdir + "/node2/wallet.dump") self.log.info("More transactions") for i in range(5): self.do_one_round() # Generate 101 more blocks, so any fees paid mature self.nodes[3].generate(101) self.sync_all() balance0 = self.nodes[0].getbalance() balance1 = self.nodes[1].getbalance() balance2 = self.nodes[2].getbalance() balance3 = self.nodes[3].getbalance() total = balance0 + balance1 + balance2 + balance3 # At this point, there are 214 blocks (103 for setup, then 10 rounds, then 101.) # 114 are mature, so the sum of all wallets should be 114 * 50 = 5700. assert_equal(total, 5700) ## # Test restoring spender wallets from backups ## self.log.info("Restoring using wallet.dat") self.stop_three() self.erase_three() # Start node2 with no chain shutil.rmtree(self.options.tmpdir + "/node2/regtest/blocks") shutil.rmtree(self.options.tmpdir + "/node2/regtest/chainstate") # Restore wallets from backup shutil.copyfile(tmpdir + "/node0/wallet.bak", - tmpdir + "/node0/regtest/wallet.dat") + tmpdir + "/node0/regtest/wallets/wallet.dat") shutil.copyfile(tmpdir + "/node1/wallet.bak", - tmpdir + "/node1/regtest/wallet.dat") + tmpdir + "/node1/regtest/wallets/wallet.dat") shutil.copyfile(tmpdir + "/node2/wallet.bak", - tmpdir + "/node2/regtest/wallet.dat") + tmpdir + "/node2/regtest/wallets/wallet.dat") self.log.info("Re-starting nodes") self.start_three() sync_blocks(self.nodes) assert_equal(self.nodes[0].getbalance(), balance0) assert_equal(self.nodes[1].getbalance(), balance1) assert_equal(self.nodes[2].getbalance(), balance2) self.log.info("Restoring using dumped wallet") self.stop_three() self.erase_three() # start node2 with no chain shutil.rmtree(self.options.tmpdir + "/node2/regtest/blocks") shutil.rmtree(self.options.tmpdir + "/node2/regtest/chainstate") self.start_three() assert_equal(self.nodes[0].getbalance(), 0) assert_equal(self.nodes[1].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 0) self.nodes[0].importwallet(tmpdir + "/node0/wallet.dump") self.nodes[1].importwallet(tmpdir + "/node1/wallet.dump") self.nodes[2].importwallet(tmpdir + "/node2/wallet.dump") sync_blocks(self.nodes) assert_equal(self.nodes[0].getbalance(), balance0) assert_equal(self.nodes[1].getbalance(), balance1) assert_equal(self.nodes[2].getbalance(), balance2) # Backup to source wallet file must fail sourcePaths = [ - tmpdir + "/node0/regtest/wallet.dat", - tmpdir + "/node0/./regtest/wallet.dat", - tmpdir + "/node0/regtest/", - tmpdir + "/node0/regtest"] + tmpdir + "/node0/regtest/wallets/wallet.dat", + tmpdir + "/node0/./regtest/wallets/wallet.dat", + tmpdir + "/node0/regtest/wallets/", + tmpdir + "/node0/regtest/wallets"] for sourcePath in sourcePaths: assert_raises_rpc_error(-4, "backup failed", self.nodes[0].backupwallet, sourcePath) if __name__ == '__main__': WalletBackupTest().main()