diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index f49fe016e..56a429567 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -1,1485 +1,1486 @@ // Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef ENABLE_WALLET +#include #include #include #endif #include #include #include #include #include #include #include #include #include #include // TODO: add a scrollback limit, as there is currently none // TODO: make it possible to filter out categories (esp debug messages when // implemented) // TODO: receive errors and debug messages through ClientModel const int CONSOLE_HISTORY = 50; const int INITIAL_TRAFFIC_GRAPH_MINS = 30; const QSize FONT_RANGE(4, 40); const char fontSizeSettingsKey[] = "consoleFontSize"; const struct { const char *url; const char *source; } ICON_MAPPING[] = {{"cmd-request", ":/icons/tx_input"}, {"cmd-reply", ":/icons/tx_output"}, {"cmd-error", ":/icons/tx_output"}, {"misc", ":/icons/tx_inout"}, {nullptr, nullptr}}; namespace { // don't add private key handling cmd's to the history const QStringList historyFilter = QStringList() << "importprivkey" << "importmulti" << "sethdseed" << "signmessagewithprivkey" << "signrawtransactionwithkey" << "walletpassphrase" << "walletpassphrasechange" << "encryptwallet"; } // namespace /* Object for executing console RPC commands in a separate thread. */ class RPCExecutor : public QObject { Q_OBJECT public: explicit RPCExecutor(interfaces::Node &node) : m_node(node) {} public Q_SLOTS: void request(const QString &command, const WalletModel *wallet_model); Q_SIGNALS: void reply(int category, const QString &command); private: interfaces::Node &m_node; }; /** Class for handling RPC timers * (used for e.g. re-locking the wallet after a timeout) */ class QtRPCTimerBase : public QObject, public RPCTimerBase { Q_OBJECT public: QtRPCTimerBase(std::function &_func, int64_t millis) : func(_func) { timer.setSingleShot(true); connect(&timer, &QTimer::timeout, [this] { func(); }); timer.start(millis); } ~QtRPCTimerBase() {} private: QTimer timer; std::function func; }; class QtRPCTimerInterface : public RPCTimerInterface { public: ~QtRPCTimerInterface() {} const char *Name() override { return "Qt"; } RPCTimerBase *NewTimer(std::function &func, int64_t millis) override { return new QtRPCTimerBase(func, millis); } }; #include /** * Split shell command line into a list of arguments and optionally execute the * command(s). * Aims to emulate \c bash and friends. * * - Command nesting is possible with parenthesis; for example: * validateaddress(getnewaddress()) * - Arguments are delimited with whitespace or comma * - Extra whitespace at the beginning and end and between arguments will be * ignored * - Text can be "double" or 'single' quoted * - The backslash \c \ is used as escape character * - Outside quotes, any character can be escaped * - Within double quotes, only escape \c " and backslashes before a \c " or * another backslash * - Within single quotes, no escaping is possible and no special * interpretation takes place * * @param[in] node optional node to execute command on * @param[out] strResult stringified result from the executed command(chain) * @param[in] strCommand Command line to split * @param[in] fExecute set true if you want the command to be executed * @param[out] pstrFilteredOut Command line, filtered to remove any sensitive * data */ bool RPCConsole::RPCParseCommandLine(interfaces::Node *node, std::string &strResult, const std::string &strCommand, const bool fExecute, std::string *const pstrFilteredOut, const WalletModel *wallet_model) { std::vector> stack; stack.push_back(std::vector()); enum CmdParseState { STATE_EATING_SPACES, STATE_EATING_SPACES_IN_ARG, STATE_EATING_SPACES_IN_BRACKETS, STATE_ARGUMENT, STATE_SINGLEQUOTED, STATE_DOUBLEQUOTED, STATE_ESCAPE_OUTER, STATE_ESCAPE_DOUBLEQUOTED, STATE_COMMAND_EXECUTED, STATE_COMMAND_EXECUTED_INNER } state = STATE_EATING_SPACES; std::string curarg; UniValue lastResult; unsigned nDepthInsideSensitive = 0; size_t filter_begin_pos = 0, chpos; std::vector> filter_ranges; auto add_to_current_stack = [&](const std::string &strArg) { if (stack.back().empty() && (!nDepthInsideSensitive) && historyFilter.contains(QString::fromStdString(strArg), Qt::CaseInsensitive)) { nDepthInsideSensitive = 1; filter_begin_pos = chpos; } // Make sure stack is not empty before adding something if (stack.empty()) { stack.push_back(std::vector()); } stack.back().push_back(strArg); }; auto close_out_params = [&]() { if (nDepthInsideSensitive) { if (!--nDepthInsideSensitive) { assert(filter_begin_pos); filter_ranges.push_back( std::make_pair(filter_begin_pos, chpos)); filter_begin_pos = 0; } } stack.pop_back(); }; std::string strCommandTerminated = strCommand; if (strCommandTerminated.back() != '\n') { strCommandTerminated += "\n"; } for (chpos = 0; chpos < strCommandTerminated.size(); ++chpos) { char ch = strCommandTerminated[chpos]; switch (state) { case STATE_COMMAND_EXECUTED_INNER: case STATE_COMMAND_EXECUTED: { bool breakParsing = true; switch (ch) { case '[': curarg.clear(); state = STATE_COMMAND_EXECUTED_INNER; break; default: if (state == STATE_COMMAND_EXECUTED_INNER) { if (ch != ']') { // append char to the current argument (which is // also used for the query command) curarg += ch; break; } if (curarg.size() && fExecute) { // if we have a value query, query arrays with // index and objects with a string key UniValue subelement; if (lastResult.isArray()) { for (char argch : curarg) { if (!IsDigit(argch)) { throw std::runtime_error( "Invalid result query"); } } subelement = lastResult[atoi(curarg.c_str())]; } else if (lastResult.isObject()) { subelement = find_value(lastResult, curarg); } else { // no array or object: abort throw std::runtime_error( "Invalid result query"); } lastResult = subelement; } state = STATE_COMMAND_EXECUTED; break; } // don't break parsing when the char is required for the // next argument breakParsing = false; // pop the stack and return the result to the current // command arguments close_out_params(); // don't stringify the json in case of a string to avoid // doublequotes if (lastResult.isStr()) { curarg = lastResult.get_str(); } else { curarg = lastResult.write(2); } // if we have a non empty result, use it as stack // argument otherwise as general result if (curarg.size()) { if (stack.size()) { add_to_current_stack(curarg); } else { strResult = curarg; } } curarg.clear(); // assume eating space state state = STATE_EATING_SPACES; } if (breakParsing) { break; } } // FALLTHROUGH case STATE_ARGUMENT: // In or after argument case STATE_EATING_SPACES_IN_ARG: case STATE_EATING_SPACES_IN_BRACKETS: case STATE_EATING_SPACES: // Handle runs of whitespace switch (ch) { case '"': state = STATE_DOUBLEQUOTED; break; case '\'': state = STATE_SINGLEQUOTED; break; case '\\': state = STATE_ESCAPE_OUTER; break; case '(': case ')': case '\n': if (state == STATE_EATING_SPACES_IN_ARG) { throw std::runtime_error("Invalid Syntax"); } if (state == STATE_ARGUMENT) { if (ch == '(' && stack.size() && stack.back().size() > 0) { if (nDepthInsideSensitive) { ++nDepthInsideSensitive; } stack.push_back(std::vector()); } // don't allow commands after executed commands on // baselevel if (!stack.size()) { throw std::runtime_error("Invalid Syntax"); } add_to_current_stack(curarg); curarg.clear(); state = STATE_EATING_SPACES_IN_BRACKETS; } if ((ch == ')' || ch == '\n') && stack.size() > 0) { if (fExecute) { // Convert argument list to JSON objects in // method-dependent way, and pass it along with // the method name to the dispatcher. UniValue params = RPCConvertValues( stack.back()[0], std::vector( stack.back().begin() + 1, stack.back().end())); std::string method = stack.back()[0]; std::string uri; #ifdef ENABLE_WALLET if (wallet_model) { QByteArray encodedName = QUrl::toPercentEncoding( wallet_model->getWalletName()); uri = "/wallet/" + std::string(encodedName.constData(), encodedName.length()); } #endif GlobalConfig config; assert(node); lastResult = node->executeRpc(config, method, params, uri); } state = STATE_COMMAND_EXECUTED; curarg.clear(); } break; case ' ': case ',': case '\t': if (state == STATE_EATING_SPACES_IN_ARG && curarg.empty() && ch == ',') { throw std::runtime_error("Invalid Syntax"); } else if (state == STATE_ARGUMENT) { // Space ends argument add_to_current_stack(curarg); curarg.clear(); } if ((state == STATE_EATING_SPACES_IN_BRACKETS || state == STATE_ARGUMENT) && ch == ',') { state = STATE_EATING_SPACES_IN_ARG; break; } state = STATE_EATING_SPACES; break; default: curarg += ch; state = STATE_ARGUMENT; } break; case STATE_SINGLEQUOTED: // Single-quoted string switch (ch) { case '\'': state = STATE_ARGUMENT; break; default: curarg += ch; } break; case STATE_DOUBLEQUOTED: // Double-quoted string switch (ch) { case '"': state = STATE_ARGUMENT; break; case '\\': state = STATE_ESCAPE_DOUBLEQUOTED; break; default: curarg += ch; } break; case STATE_ESCAPE_OUTER: // '\' outside quotes curarg += ch; state = STATE_ARGUMENT; break; case STATE_ESCAPE_DOUBLEQUOTED: // '\' in double-quoted text if (ch != '"' && ch != '\\') { // keep '\' for everything but the quote and '\' itself curarg += '\\'; } curarg += ch; state = STATE_DOUBLEQUOTED; break; } } if (pstrFilteredOut) { if (STATE_COMMAND_EXECUTED == state) { assert(!stack.empty()); close_out_params(); } *pstrFilteredOut = strCommand; for (auto i = filter_ranges.rbegin(); i != filter_ranges.rend(); ++i) { pstrFilteredOut->replace(i->first, i->second - i->first, "(…)"); } } // final state switch (state) { case STATE_COMMAND_EXECUTED: if (lastResult.isStr()) { strResult = lastResult.get_str(); } else { strResult = lastResult.write(2); } // FALLTHROUGH case STATE_ARGUMENT: case STATE_EATING_SPACES: return true; default: // ERROR to end in one of the other states return false; } } void RPCExecutor::request(const QString &command, const WalletModel *wallet_model) { try { std::string result; std::string executableCommand = command.toStdString() + "\n"; // Catch the console-only-help command before RPC call is executed and // reply with help text as-if a RPC reply. if (executableCommand == "help-console\n") { Q_EMIT reply( RPCConsole::CMD_REPLY, QString(("\n" "This console accepts RPC commands using the standard " "syntax.\n" " example: getblockhash 0\n\n" "This console can also accept RPC commands using " "parenthesized syntax.\n" " example: getblockhash(0)\n\n" "Commands may be nested when specified with the " "parenthesized syntax.\n" " example: getblock(getblockhash(0) 1)\n\n" "A space or a comma can be used to delimit arguments " "for either syntax.\n" " example: getblockhash 0\n" " getblockhash,0\n\n" "Named results can be queried with a non-quoted key " "string in brackets.\n" " example: getblock(getblockhash(0) true)[tx]\n\n" "Results without keys can be queried using an integer " "in brackets.\n" " example: " "getblock(getblockhash(0),true)[tx][0]\n\n"))); return; } if (!RPCConsole::RPCExecuteCommandLine( m_node, result, executableCommand, nullptr, wallet_model)) { Q_EMIT reply(RPCConsole::CMD_ERROR, QString("Parse error: unbalanced ' or \"")); return; } Q_EMIT reply(RPCConsole::CMD_REPLY, QString::fromStdString(result)); } catch (UniValue &objError) { // Nice formatting for standard-format error try { int code = find_value(objError, "code").get_int(); std::string message = find_value(objError, "message").get_str(); Q_EMIT reply(RPCConsole::CMD_ERROR, QString::fromStdString(message) + " (code " + QString::number(code) + ")"); } catch (const std::runtime_error &) { // raised when converting to invalid type, i.e. missing code or // message. Show raw JSON object. Q_EMIT reply(RPCConsole::CMD_ERROR, QString::fromStdString(objError.write())); } } catch (const std::exception &e) { Q_EMIT reply(RPCConsole::CMD_ERROR, QString("Error: ") + QString::fromStdString(e.what())); } } RPCConsole::RPCConsole(interfaces::Node &node, const PlatformStyle *_platformStyle, QWidget *parent) : QWidget(parent), m_node(node), ui(new Ui::RPCConsole), platformStyle(_platformStyle) { ui->setupUi(this); QSettings settings; if (!restoreGeometry( settings.value("RPCConsoleWindowGeometry").toByteArray())) { // Restore failed (perhaps missing setting), center the window move(QGuiApplication::primaryScreen()->availableGeometry().center() - frameGeometry().center()); } QChar nonbreaking_hyphen(8209); ui->dataDir->setToolTip( ui->dataDir->toolTip().arg(QString(nonbreaking_hyphen) + "datadir")); ui->blocksDir->setToolTip(ui->blocksDir->toolTip().arg( QString(nonbreaking_hyphen) + "blocksdir")); ui->openDebugLogfileButton->setToolTip( ui->openDebugLogfileButton->toolTip().arg(PACKAGE_NAME)); if (platformStyle->getImagesOnButtons()) { ui->openDebugLogfileButton->setIcon( platformStyle->SingleColorIcon(":/icons/export")); } ui->clearButton->setIcon(platformStyle->SingleColorIcon(":/icons/remove")); ui->fontBiggerButton->setIcon( platformStyle->SingleColorIcon(":/icons/fontbigger")); ui->fontSmallerButton->setIcon( platformStyle->SingleColorIcon(":/icons/fontsmaller")); // Install event filter for up and down arrow ui->lineEdit->installEventFilter(this); ui->messagesWidget->installEventFilter(this); connect(ui->clearButton, &QPushButton::clicked, this, &RPCConsole::clear); connect(ui->fontBiggerButton, &QPushButton::clicked, this, &RPCConsole::fontBigger); connect(ui->fontSmallerButton, &QPushButton::clicked, this, &RPCConsole::fontSmaller); connect(ui->btnClearTrafficGraph, &QPushButton::clicked, ui->trafficGraph, &TrafficGraphWidget::clear); // disable the wallet selector by default ui->WalletSelector->setVisible(false); ui->WalletSelectorLabel->setVisible(false); // set library version labels #ifdef ENABLE_WALLET ui->berkeleyDBVersion->setText(DbEnv::version(nullptr, nullptr, nullptr)); #else ui->label_berkeleyDBVersion->hide(); ui->berkeleyDBVersion->hide(); #endif // Register RPC timer interface rpcTimerInterface = new QtRPCTimerInterface(); // avoid accidentally overwriting an existing, non QTThread // based timer interface m_node.rpcSetTimerInterfaceIfUnset(rpcTimerInterface); setTrafficGraphRange(INITIAL_TRAFFIC_GRAPH_MINS); ui->detailWidget->hide(); ui->peerHeading->setText(tr("Select a peer to view detailed information.")); consoleFontSize = settings.value(fontSizeSettingsKey, QFontInfo(QFont()).pointSize()) .toInt(); clear(); } RPCConsole::~RPCConsole() { QSettings settings; settings.setValue("RPCConsoleWindowGeometry", saveGeometry()); m_node.rpcUnsetTimerInterface(rpcTimerInterface); delete rpcTimerInterface; delete ui; } bool RPCConsole::eventFilter(QObject *obj, QEvent *event) { // Special key handling if (event->type() == QEvent::KeyPress) { QKeyEvent *keyevt = static_cast(event); int key = keyevt->key(); Qt::KeyboardModifiers mod = keyevt->modifiers(); switch (key) { case Qt::Key_Up: if (obj == ui->lineEdit) { browseHistory(-1); return true; } break; case Qt::Key_Down: if (obj == ui->lineEdit) { browseHistory(1); return true; } break; case Qt::Key_PageUp: /* pass paging keys to messages widget */ case Qt::Key_PageDown: if (obj == ui->lineEdit) { QApplication::postEvent(ui->messagesWidget, new QKeyEvent(*keyevt)); return true; } break; case Qt::Key_Return: case Qt::Key_Enter: // forward these events to lineEdit if (obj == autoCompleter->popup()) { QApplication::postEvent(ui->lineEdit, new QKeyEvent(*keyevt)); return true; } break; default: // Typing in messages widget brings focus to line edit, and // redirects key there. Exclude most combinations and keys that // emit no text, except paste shortcuts. if (obj == ui->messagesWidget && ((!mod && !keyevt->text().isEmpty() && key != Qt::Key_Tab) || ((mod & Qt::ControlModifier) && key == Qt::Key_V) || ((mod & Qt::ShiftModifier) && key == Qt::Key_Insert))) { ui->lineEdit->setFocus(); QApplication::postEvent(ui->lineEdit, new QKeyEvent(*keyevt)); return true; } } } return QWidget::eventFilter(obj, event); } void RPCConsole::setClientModel(ClientModel *model) { clientModel = model; bool wallet_enabled{false}; #ifdef ENABLE_WALLET wallet_enabled = WalletModel::isWalletEnabled(); #endif // ENABLE_WALLET if (model && !wallet_enabled) { // Show warning, for example if this is a prerelease version connect(model, &ClientModel::alertsChanged, this, &RPCConsole::updateAlerts); updateAlerts(model->getStatusBarWarnings()); } ui->trafficGraph->setClientModel(model); if (model && clientModel->getPeerTableModel() && clientModel->getBanTableModel()) { // Keep up to date with client setNumConnections(model->getNumConnections()); connect(model, &ClientModel::numConnectionsChanged, this, &RPCConsole::setNumConnections); interfaces::Node &node = clientModel->node(); setNumBlocks(node.getNumBlocks(), QDateTime::fromTime_t(node.getLastBlockTime()), node.getVerificationProgress(), false); connect(model, &ClientModel::numBlocksChanged, this, &RPCConsole::setNumBlocks); updateNetworkState(); connect(model, &ClientModel::networkActiveChanged, this, &RPCConsole::setNetworkActive); updateTrafficStats(node.getTotalBytesRecv(), node.getTotalBytesSent()); connect(model, &ClientModel::bytesChanged, this, &RPCConsole::updateTrafficStats); connect(model, &ClientModel::mempoolSizeChanged, this, &RPCConsole::setMempoolSize); // set up peer table ui->peerWidget->setModel(model->getPeerTableModel()); ui->peerWidget->verticalHeader()->hide(); ui->peerWidget->setEditTriggers(QAbstractItemView::NoEditTriggers); ui->peerWidget->setSelectionBehavior(QAbstractItemView::SelectRows); ui->peerWidget->setSelectionMode(QAbstractItemView::ExtendedSelection); ui->peerWidget->setContextMenuPolicy(Qt::CustomContextMenu); ui->peerWidget->setColumnWidth(PeerTableModel::Address, ADDRESS_COLUMN_WIDTH); ui->peerWidget->setColumnWidth(PeerTableModel::Subversion, SUBVERSION_COLUMN_WIDTH); ui->peerWidget->setColumnWidth(PeerTableModel::Ping, PING_COLUMN_WIDTH); ui->peerWidget->horizontalHeader()->setStretchLastSection(true); // create peer table context menu actions QAction *disconnectAction = new QAction(tr("&Disconnect"), this); QAction *banAction1h = new QAction(tr("Ban for") + " " + tr("1 &hour"), this); QAction *banAction24h = new QAction(tr("Ban for") + " " + tr("1 &day"), this); QAction *banAction7d = new QAction(tr("Ban for") + " " + tr("1 &week"), this); QAction *banAction365d = new QAction(tr("Ban for") + " " + tr("1 &year"), this); // create peer table context menu peersTableContextMenu = new QMenu(this); peersTableContextMenu->addAction(disconnectAction); peersTableContextMenu->addAction(banAction1h); peersTableContextMenu->addAction(banAction24h); peersTableContextMenu->addAction(banAction7d); peersTableContextMenu->addAction(banAction365d); connect(banAction1h, &QAction::triggered, [this] { banSelectedNode(60 * 60); }); connect(banAction24h, &QAction::triggered, [this] { banSelectedNode(60 * 60 * 24); }); connect(banAction7d, &QAction::triggered, [this] { banSelectedNode(60 * 60 * 24 * 7); }); connect(banAction365d, &QAction::triggered, [this] { banSelectedNode(60 * 60 * 24 * 365); }); // peer table context menu signals connect(ui->peerWidget, &QTableView::customContextMenuRequested, this, &RPCConsole::showPeersTableContextMenu); connect(disconnectAction, &QAction::triggered, this, &RPCConsole::disconnectSelectedNode); // peer table signal handling - update peer details when selecting new // node connect(ui->peerWidget->selectionModel(), &QItemSelectionModel::selectionChanged, this, &RPCConsole::peerSelected); // peer table signal handling - update peer details when new nodes are // added to the model connect(model->getPeerTableModel(), &PeerTableModel::layoutChanged, this, &RPCConsole::peerLayoutChanged); // peer table signal handling - cache selected node ids connect(model->getPeerTableModel(), &PeerTableModel::layoutAboutToBeChanged, this, &RPCConsole::peerLayoutAboutToChange); // set up ban table ui->banlistWidget->setModel(model->getBanTableModel()); ui->banlistWidget->verticalHeader()->hide(); ui->banlistWidget->setEditTriggers(QAbstractItemView::NoEditTriggers); ui->banlistWidget->setSelectionBehavior(QAbstractItemView::SelectRows); ui->banlistWidget->setSelectionMode(QAbstractItemView::SingleSelection); ui->banlistWidget->setContextMenuPolicy(Qt::CustomContextMenu); ui->banlistWidget->setColumnWidth(BanTableModel::Address, BANSUBNET_COLUMN_WIDTH); ui->banlistWidget->setColumnWidth(BanTableModel::Bantime, BANTIME_COLUMN_WIDTH); ui->banlistWidget->horizontalHeader()->setStretchLastSection(true); // create ban table context menu action QAction *unbanAction = new QAction(tr("&Unban"), this); // create ban table context menu banTableContextMenu = new QMenu(this); banTableContextMenu->addAction(unbanAction); // ban table context menu signals connect(ui->banlistWidget, &QTableView::customContextMenuRequested, this, &RPCConsole::showBanTableContextMenu); connect(unbanAction, &QAction::triggered, this, &RPCConsole::unbanSelectedNode); // ban table signal handling - clear peer details when clicking a peer // in the ban table connect(ui->banlistWidget, &QTableView::clicked, this, &RPCConsole::clearSelectedNode); // ban table signal handling - ensure ban table is shown or hidden (if // empty) connect(model->getBanTableModel(), &BanTableModel::layoutChanged, this, &RPCConsole::showOrHideBanTableIfRequired); showOrHideBanTableIfRequired(); // Provide initial values ui->clientVersion->setText(model->formatFullVersion()); ui->clientUserAgent->setText(model->formatSubVersion()); ui->dataDir->setText(model->dataDir()); ui->blocksDir->setText(model->blocksDir()); ui->startupTime->setText(model->formatClientStartupTime()); ui->networkName->setText( QString::fromStdString(Params().NetworkIDString())); // Setup autocomplete and attach it QStringList wordList; std::vector commandList = m_node.listRpcCommands(); for (size_t i = 0; i < commandList.size(); ++i) { wordList << commandList[i].c_str(); wordList << ("help " + commandList[i]).c_str(); } wordList << "help-console"; wordList.sort(); autoCompleter = new QCompleter(wordList, this); autoCompleter->setModelSorting(QCompleter::CaseSensitivelySortedModel); // ui->lineEdit is initially disabled because running commands is only // possible from now on. ui->lineEdit->setEnabled(true); ui->lineEdit->setCompleter(autoCompleter); autoCompleter->popup()->installEventFilter(this); // Start thread to execute RPC commands. startExecutor(); } if (!model) { // Client model is being set to 0, this means shutdown() is about to be // called. thread.quit(); thread.wait(); } } #ifdef ENABLE_WALLET void RPCConsole::addWallet(WalletModel *const walletModel) { // use name for text and wallet model for internal data object (to allow to // move to a wallet id later) ui->WalletSelector->addItem(walletModel->getDisplayName(), QVariant::fromValue(walletModel)); if (ui->WalletSelector->count() == 2 && !isVisible()) { // First wallet added, set to default so long as the window isn't // presently visible (and potentially in use) ui->WalletSelector->setCurrentIndex(1); } if (ui->WalletSelector->count() > 2) { ui->WalletSelector->setVisible(true); ui->WalletSelectorLabel->setVisible(true); } } void RPCConsole::removeWallet(WalletModel *const walletModel) { ui->WalletSelector->removeItem( ui->WalletSelector->findData(QVariant::fromValue(walletModel))); if (ui->WalletSelector->count() == 2) { ui->WalletSelector->setVisible(false); ui->WalletSelectorLabel->setVisible(false); } } #endif static QString categoryClass(int category) { switch (category) { case RPCConsole::CMD_REQUEST: return "cmd-request"; break; case RPCConsole::CMD_REPLY: return "cmd-reply"; break; case RPCConsole::CMD_ERROR: return "cmd-error"; break; default: return "misc"; } } void RPCConsole::fontBigger() { setFontSize(consoleFontSize + 1); } void RPCConsole::fontSmaller() { setFontSize(consoleFontSize - 1); } void RPCConsole::setFontSize(int newSize) { QSettings settings; // don't allow an insane font size if (newSize < FONT_RANGE.width() || newSize > FONT_RANGE.height()) { return; } // temp. store the console content QString str = ui->messagesWidget->toHtml(); // replace font tags size in current content str.replace(QString("font-size:%1pt").arg(consoleFontSize), QString("font-size:%1pt").arg(newSize)); // store the new font size consoleFontSize = newSize; settings.setValue(fontSizeSettingsKey, consoleFontSize); // clear console (reset icon sizes, default stylesheet) and re-add the // content float oldPosFactor = 1.0 / ui->messagesWidget->verticalScrollBar()->maximum() * ui->messagesWidget->verticalScrollBar()->value(); clear(false); ui->messagesWidget->setHtml(str); ui->messagesWidget->verticalScrollBar()->setValue( oldPosFactor * ui->messagesWidget->verticalScrollBar()->maximum()); } void RPCConsole::clear(bool clearHistory) { ui->messagesWidget->clear(); if (clearHistory) { history.clear(); historyPtr = 0; } ui->lineEdit->clear(); ui->lineEdit->setFocus(); // Add smoothly scaled icon images. // (when using width/height on an img, Qt uses nearest instead of linear // interpolation) for (int i = 0; ICON_MAPPING[i].url; ++i) { ui->messagesWidget->document()->addResource( QTextDocument::ImageResource, QUrl(ICON_MAPPING[i].url), platformStyle->SingleColorImage(ICON_MAPPING[i].source) .scaled(QSize(consoleFontSize * 2, consoleFontSize * 2), Qt::IgnoreAspectRatio, Qt::SmoothTransformation)); } // Set default style sheet QFontInfo fixedFontInfo(GUIUtil::fixedPitchFont()); ui->messagesWidget->document()->setDefaultStyleSheet( QString("table { }" "td.time { color: #808080; font-size: %2; padding-top: 3px; } " "td.message { font-family: %1; font-size: %2; " "white-space:pre-wrap; } " "td.cmd-request { color: #006060; } " "td.cmd-error { color: red; } " ".secwarning { color: red; }" "b { color: #006060; } ") .arg(fixedFontInfo.family(), QString("%1pt").arg(consoleFontSize))); #ifdef Q_OS_MAC QString clsKey = "(⌘)-L"; #else QString clsKey = "Ctrl-L"; #endif message(CMD_REPLY, (tr("Welcome to the %1 RPC console.").arg(PACKAGE_NAME) + "
" + tr("Use up and down arrows to navigate history, and " "%1 to clear screen.") .arg("" + clsKey + "") + "
" + tr("Type %1 for an overview of available commands.") .arg("help") + "
" + tr("For more information on using this console type %1.") .arg("help-console") + "

" + tr("WARNING: Scammers have been active, telling users to type " "commands here, stealing their wallet contents. Do not use " "this console without fully understanding the ramifications " "of a command.") + ""), true); } void RPCConsole::keyPressEvent(QKeyEvent *event) { if (windowType() != Qt::Widget && event->key() == Qt::Key_Escape) { close(); } } void RPCConsole::message(int category, const QString &message, bool html) { QTime time = QTime::currentTime(); QString timeString = time.toString(); QString out; out += ""; out += ""; out += "
" + timeString + ""; if (html) { out += message; } else { out += GUIUtil::HtmlEscape(message, false); } out += "
"; ui->messagesWidget->append(out); } void RPCConsole::updateNetworkState() { QString connections = QString::number(clientModel->getNumConnections()) + " ("; connections += tr("In:") + " " + QString::number(clientModel->getNumConnections( ClientModel::CONNECTIONS_IN)) + " / "; connections += tr("Out:") + " " + QString::number(clientModel->getNumConnections( ClientModel::CONNECTIONS_OUT)) + ")"; if (!clientModel->node().getNetworkActive()) { connections += " (" + tr("Network activity disabled") + ")"; } ui->numberOfConnections->setText(connections); } void RPCConsole::setNumConnections(int count) { if (!clientModel) { return; } updateNetworkState(); } void RPCConsole::setNetworkActive(bool networkActive) { updateNetworkState(); } void RPCConsole::setNumBlocks(int count, const QDateTime &blockDate, double nVerificationProgress, bool headers) { if (!headers) { ui->numberOfBlocks->setText(QString::number(count)); ui->lastBlockTime->setText(blockDate.toString()); } } void RPCConsole::setMempoolSize(long numberOfTxs, size_t dynUsage) { ui->mempoolNumberTxs->setText(QString::number(numberOfTxs)); if (dynUsage < 1000000) { ui->mempoolSize->setText(QString::number(dynUsage / 1000.0, 'f', 2) + " KB"); } else { ui->mempoolSize->setText(QString::number(dynUsage / 1000000.0, 'f', 2) + " MB"); } } void RPCConsole::on_lineEdit_returnPressed() { QString cmd = ui->lineEdit->text(); if (!cmd.isEmpty()) { std::string strFilteredCmd; try { std::string dummy; if (!RPCParseCommandLine(nullptr, dummy, cmd.toStdString(), false, &strFilteredCmd)) { // Failed to parse command, so we cannot even filter it for the // history throw std::runtime_error("Invalid command line"); } } catch (const std::exception &e) { QMessageBox::critical(this, "Error", QString("Error: ") + QString::fromStdString(e.what())); return; } ui->lineEdit->clear(); cmdBeforeBrowsing = QString(); #ifdef ENABLE_WALLET WalletModel *wallet_model{nullptr}; const int wallet_index = ui->WalletSelector->currentIndex(); if (wallet_index > 0) { wallet_model = ui->WalletSelector->itemData(wallet_index) .value(); } if (m_last_wallet_model != wallet_model) { if (wallet_model) { message(CMD_REQUEST, tr("Executing command using \"%1\" wallet") .arg(wallet_model->getWalletName())); } else { message(CMD_REQUEST, tr("Executing command without any wallet")); } m_last_wallet_model = wallet_model; } #endif message(CMD_REQUEST, QString::fromStdString(strFilteredCmd)); Q_EMIT cmdRequest(cmd, m_last_wallet_model); cmd = QString::fromStdString(strFilteredCmd); // Remove command, if already in history history.removeOne(cmd); // Append command to history history.append(cmd); // Enforce maximum history size while (history.size() > CONSOLE_HISTORY) { history.removeFirst(); } // Set pointer to end of history historyPtr = history.size(); // Scroll console view to end scrollToEnd(); } } void RPCConsole::browseHistory(int offset) { // store current text when start browsing through the history if (historyPtr == history.size()) { cmdBeforeBrowsing = ui->lineEdit->text(); } historyPtr += offset; if (historyPtr < 0) { historyPtr = 0; } if (historyPtr > history.size()) { historyPtr = history.size(); } QString cmd; if (historyPtr < history.size()) { cmd = history.at(historyPtr); } else if (!cmdBeforeBrowsing.isNull()) { cmd = cmdBeforeBrowsing; } ui->lineEdit->setText(cmd); } void RPCConsole::startExecutor() { RPCExecutor *executor = new RPCExecutor(m_node); executor->moveToThread(&thread); // Replies from executor object must go to this object connect(executor, &RPCExecutor::reply, this, static_cast( &RPCConsole::message)); // Requests from this object must go to executor connect(this, &RPCConsole::cmdRequest, executor, &RPCExecutor::request); // Make sure executor object is deleted in its own thread connect(&thread, &QThread::finished, executor, &RPCExecutor::deleteLater); // Default implementation of QThread::run() simply spins up an event loop in // the thread, which is what we want. thread.start(); } void RPCConsole::on_tabWidget_currentChanged(int index) { if (ui->tabWidget->widget(index) == ui->tab_console) { ui->lineEdit->setFocus(); } } void RPCConsole::on_openDebugLogfileButton_clicked() { GUIUtil::openDebugLogfile(); } void RPCConsole::scrollToEnd() { QScrollBar *scrollbar = ui->messagesWidget->verticalScrollBar(); scrollbar->setValue(scrollbar->maximum()); } void RPCConsole::on_sldGraphRange_valueChanged(int value) { const int multiplier = 5; // each position on the slider represents 5 min int mins = value * multiplier; setTrafficGraphRange(mins); } void RPCConsole::setTrafficGraphRange(int mins) { ui->trafficGraph->setGraphRangeMins(mins); ui->lblGraphRange->setText(GUIUtil::formatDurationStr(mins * 60)); } void RPCConsole::updateTrafficStats(quint64 totalBytesIn, quint64 totalBytesOut) { ui->lblBytesIn->setText(GUIUtil::formatBytes(totalBytesIn)); ui->lblBytesOut->setText(GUIUtil::formatBytes(totalBytesOut)); } void RPCConsole::peerSelected(const QItemSelection &selected, const QItemSelection &deselected) { Q_UNUSED(deselected); if (!clientModel || !clientModel->getPeerTableModel() || selected.indexes().isEmpty()) { return; } const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats( selected.indexes().first().row()); if (stats) { updateNodeDetail(stats); } } void RPCConsole::peerLayoutAboutToChange() { QModelIndexList selected = ui->peerWidget->selectionModel()->selectedIndexes(); cachedNodeids.clear(); for (int i = 0; i < selected.size(); i++) { const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats( selected.at(i).row()); cachedNodeids.append(stats->nodeStats.nodeid); } } void RPCConsole::peerLayoutChanged() { if (!clientModel || !clientModel->getPeerTableModel()) { return; } const CNodeCombinedStats *stats = nullptr; bool fUnselect = false; bool fReselect = false; // no node selected yet if (cachedNodeids.empty()) { return; } // find the currently selected row int selectedRow = -1; QModelIndexList selectedModelIndex = ui->peerWidget->selectionModel()->selectedIndexes(); if (!selectedModelIndex.isEmpty()) { selectedRow = selectedModelIndex.first().row(); } // check if our detail node has a row in the table (it may not necessarily // be at selectedRow since its position can change after a layout change) int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(cachedNodeids.first()); if (detailNodeRow < 0) { // detail node disappeared from table (node disconnected) fUnselect = true; } else { if (detailNodeRow != selectedRow) { // detail node moved position fUnselect = true; fReselect = true; } // get fresh stats on the detail node. stats = clientModel->getPeerTableModel()->getNodeStats(detailNodeRow); } if (fUnselect && selectedRow >= 0) { clearSelectedNode(); } if (fReselect) { for (int i = 0; i < cachedNodeids.size(); i++) { ui->peerWidget->selectRow( clientModel->getPeerTableModel()->getRowByNodeId( cachedNodeids.at(i))); } } if (stats) { updateNodeDetail(stats); } } void RPCConsole::updateNodeDetail(const CNodeCombinedStats *stats) { // update the detail ui with latest node information QString peerAddrDetails(QString::fromStdString(stats->nodeStats.addrName) + " "); peerAddrDetails += tr("(node id: %1)").arg(QString::number(stats->nodeStats.nodeid)); if (!stats->nodeStats.addrLocal.empty()) { peerAddrDetails += "
" + tr("via %1").arg(QString::fromStdString( stats->nodeStats.addrLocal)); } ui->peerHeading->setText(peerAddrDetails); ui->peerServices->setText( GUIUtil::formatServicesStr(stats->nodeStats.nServices)); ui->peerLastSend->setText( stats->nodeStats.nLastSend ? GUIUtil::formatDurationStr(GetSystemTimeInSeconds() - stats->nodeStats.nLastSend) : tr("never")); ui->peerLastRecv->setText( stats->nodeStats.nLastRecv ? GUIUtil::formatDurationStr(GetSystemTimeInSeconds() - stats->nodeStats.nLastRecv) : tr("never")); ui->peerBytesSent->setText( GUIUtil::formatBytes(stats->nodeStats.nSendBytes)); ui->peerBytesRecv->setText( GUIUtil::formatBytes(stats->nodeStats.nRecvBytes)); ui->peerConnTime->setText(GUIUtil::formatDurationStr( GetSystemTimeInSeconds() - stats->nodeStats.nTimeConnected)); ui->peerPingTime->setText( GUIUtil::formatPingTime(stats->nodeStats.m_ping_usec)); ui->peerPingWait->setText( GUIUtil::formatPingTime(stats->nodeStats.m_ping_wait_usec)); ui->peerMinPing->setText( GUIUtil::formatPingTime(stats->nodeStats.m_min_ping_usec)); ui->timeoffset->setText( GUIUtil::formatTimeOffset(stats->nodeStats.nTimeOffset)); ui->peerVersion->setText( QString("%1").arg(QString::number(stats->nodeStats.nVersion))); ui->peerSubversion->setText( QString::fromStdString(stats->nodeStats.cleanSubVer)); ui->peerDirection->setText(stats->nodeStats.fInbound ? tr("Inbound") : tr("Outbound")); ui->peerHeight->setText( QString("%1").arg(QString::number(stats->nodeStats.nStartingHeight))); ui->peerWhitelisted->setText( stats->nodeStats.m_legacyWhitelisted ? tr("Yes") : tr("No")); // This check fails for example if the lock was busy and // nodeStateStats couldn't be fetched. if (stats->fNodeStateStatsAvailable) { // Ban score is init to 0 ui->peerBanScore->setText( QString("%1").arg(stats->nodeStateStats.nMisbehavior)); // Sync height is init to -1 if (stats->nodeStateStats.nSyncHeight > -1) { ui->peerSyncHeight->setText( QString("%1").arg(stats->nodeStateStats.nSyncHeight)); } else { ui->peerSyncHeight->setText(tr("Unknown")); } // Common height is init to -1 if (stats->nodeStateStats.nCommonHeight > -1) { ui->peerCommonHeight->setText( QString("%1").arg(stats->nodeStateStats.nCommonHeight)); } else { ui->peerCommonHeight->setText(tr("Unknown")); } } ui->detailWidget->show(); } void RPCConsole::resizeEvent(QResizeEvent *event) { QWidget::resizeEvent(event); } void RPCConsole::showEvent(QShowEvent *event) { QWidget::showEvent(event); if (!clientModel || !clientModel->getPeerTableModel()) { return; } // start PeerTableModel auto refresh clientModel->getPeerTableModel()->startAutoRefresh(); } void RPCConsole::hideEvent(QHideEvent *event) { QWidget::hideEvent(event); if (!clientModel || !clientModel->getPeerTableModel()) { return; } // stop PeerTableModel auto refresh clientModel->getPeerTableModel()->stopAutoRefresh(); } void RPCConsole::showPeersTableContextMenu(const QPoint &point) { QModelIndex index = ui->peerWidget->indexAt(point); if (index.isValid()) { peersTableContextMenu->exec(QCursor::pos()); } } void RPCConsole::showBanTableContextMenu(const QPoint &point) { QModelIndex index = ui->banlistWidget->indexAt(point); if (index.isValid()) { banTableContextMenu->exec(QCursor::pos()); } } void RPCConsole::disconnectSelectedNode() { // Get selected peer addresses QList nodes = GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId); for (int i = 0; i < nodes.count(); i++) { // Get currently selected peer address NodeId id = nodes.at(i).data().toLongLong(); // Find the node, disconnect it and clear the selected node if (m_node.disconnectById(id)) { clearSelectedNode(); } } } void RPCConsole::banSelectedNode(int bantime) { if (!clientModel) { return; } // Get selected peer addresses QList nodes = GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId); for (int i = 0; i < nodes.count(); i++) { // Get currently selected peer address NodeId id = nodes.at(i).data().toLongLong(); // Get currently selected peer address int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(id); if (detailNodeRow < 0) { return; } // Find possible nodes, ban it and clear the selected node const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats(detailNodeRow); if (stats) { m_node.ban(stats->nodeStats.addr, bantime); m_node.disconnectByAddress(stats->nodeStats.addr); } } clearSelectedNode(); clientModel->getBanTableModel()->refresh(); } void RPCConsole::unbanSelectedNode() { if (!clientModel) { return; } // Get selected ban addresses QList nodes = GUIUtil::getEntryData(ui->banlistWidget, BanTableModel::Address); for (int i = 0; i < nodes.count(); i++) { // Get currently selected ban address QString strNode = nodes.at(i).data().toString(); CSubNet possibleSubnet; LookupSubNet(strNode.toStdString().c_str(), possibleSubnet); if (possibleSubnet.IsValid() && m_node.unban(possibleSubnet)) { clientModel->getBanTableModel()->refresh(); } } } void RPCConsole::clearSelectedNode() { ui->peerWidget->selectionModel()->clearSelection(); cachedNodeids.clear(); ui->detailWidget->hide(); ui->peerHeading->setText(tr("Select a peer to view detailed information.")); } void RPCConsole::showOrHideBanTableIfRequired() { if (!clientModel) { return; } bool visible = clientModel->getBanTableModel()->shouldShow(); ui->banlistWidget->setVisible(visible); ui->banHeading->setVisible(visible); } void RPCConsole::setTabFocus(enum TabTypes tabType) { ui->tabWidget->setCurrentIndex(int(tabType)); } QString RPCConsole::tabTitle(TabTypes tab_type) const { return ui->tabWidget->tabText(int(tab_type)); } QKeySequence RPCConsole::tabShortcut(TabTypes tab_type) const { switch (tab_type) { case TabTypes::INFO: return QKeySequence(Qt::CTRL + Qt::Key_I); case TabTypes::CONSOLE: return QKeySequence(Qt::CTRL + Qt::Key_T); case TabTypes::GRAPH: return QKeySequence(Qt::CTRL + Qt::Key_N); case TabTypes::PEERS: return QKeySequence(Qt::CTRL + Qt::Key_P); } // no default case, so the compiler can warn about missing cases assert(false); } void RPCConsole::updateAlerts(const QString &warnings) { this->ui->label_alerts->setVisible(!warnings.isEmpty()); this->ui->label_alerts->setText(warnings); } diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index 4b43a6b71..29733adc7 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -1,37 +1,38 @@ # Copyright (c) 2017 The Bitcoin developers project(wallet) # Add Berkeley DB dependency. find_package(BerkeleyDB 5.3 REQUIRED COMPONENTS CXX) # PR15638(https://reviews.bitcoinabc.org/D6000) moved some wallet load # functions to wallet/load.cpp, the others in wallet/init.cpp remain in # the server target_sources(server PRIVATE init.cpp) add_library(wallet ../interfaces/wallet.cpp + bdb.cpp coincontrol.cpp coinselection.cpp context.cpp crypter.cpp db.cpp fees.cpp load.cpp rpcdump.cpp rpcwallet.cpp salvage.cpp scriptpubkeyman.cpp wallet.cpp walletdb.cpp walletutil.cpp ) # There is a circular dependency between wallet and server, see: # https://github.com/bitcoin/bitcoin/pull/14437#discussion_r226237048 target_link_libraries(wallet bitcoinconsensus univalue BerkeleyDB::CXX) # wallet-tool library add_library(wallet-tool wallettool.cpp) target_link_libraries(wallet-tool wallet) diff --git a/src/wallet/db.cpp b/src/wallet/bdb.cpp similarity index 96% copy from src/wallet/db.cpp copy to src/wallet/bdb.cpp index 2b19b4122..c790cbd6d 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/bdb.cpp @@ -1,904 +1,882 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include #include #include #include // boost::this_thread::interruption_point() (mingw) #include #ifndef WIN32 #include #endif namespace { //! Make sure database has a unique fileid within the environment. If it //! doesn't, throw an error. BDB caches do not work properly when more than one //! open database has the same fileid (values written to one database may show //! up in reads to other databases). //! //! BerkeleyDB generates unique fileids by default //! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html), //! so bitcoin should never create different databases with the same fileid, but //! this error can be triggered if users manually copy database files. void CheckUniqueFileid(const BerkeleyEnvironment &env, const std::string &filename, Db &db, WalletDatabaseFileId &fileid) { if (env.IsMock()) { return; } int ret = db.get_mpf()->get_fileid(fileid.value); if (ret != 0) { throw std::runtime_error(strprintf( "BerkeleyBatch: Can't open database %s (get_fileid failed with %d)", filename, ret)); } for (const auto &item : env.m_fileids) { if (fileid == item.second && &fileid != &item.second) { throw std::runtime_error(strprintf( "BerkeleyBatch: Can't open database %s (duplicates fileid %s " "from %s)", filename, HexStr(std::begin(item.second.value), std::end(item.second.value)), item.first)); } } } RecursiveMutex cs_db; //! Map from directory name to db environment. std::map> g_dbenvs GUARDED_BY(cs_db); } // namespace bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId &rhs) const { return memcmp(value, &rhs.value, sizeof(value)) == 0; } -void SplitWalletPath(const fs::path &wallet_path, fs::path &env_directory, - std::string &database_filename) { - if (fs::is_regular_file(wallet_path)) { - // Special case for backwards compatibility: if wallet path points to an - // existing file, treat it as the path to a BDB data file in a parent - // directory that also contains BDB log files. - env_directory = wallet_path.parent_path(); - database_filename = wallet_path.filename().string(); - } else { - // Normal case: Interpret wallet path as a directory path containing - // data and log files. - env_directory = wallet_path; - database_filename = "wallet.dat"; - } -} - bool IsBDBWalletLoaded(const fs::path &wallet_path) { fs::path env_directory; std::string database_filename; SplitWalletPath(wallet_path, env_directory, database_filename); LOCK(cs_db); auto env = g_dbenvs.find(env_directory.string()); if (env == g_dbenvs.end()) { return false; } auto database = env->second.lock(); return database && database->IsDatabaseLoaded(database_filename); } -fs::path WalletDataFilePath(const fs::path &wallet_path) { - fs::path env_directory; - std::string database_filename; - SplitWalletPath(wallet_path, env_directory, database_filename); - return env_directory / database_filename; -} - /** * @param[in] wallet_path Path to wallet directory. Or (for backwards * compatibility only) a path to a berkeley btree data file inside a wallet * directory. * @param[out] database_filename Filename of berkeley btree data file inside the * wallet directory. * @return A shared pointer to the BerkeleyEnvironment object for the wallet * directory, never empty because ~BerkeleyEnvironment erases the weak pointer * from the g_dbenvs map. * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the * directory path key was not already in the map. */ std::shared_ptr GetWalletEnv(const fs::path &wallet_path, std::string &database_filename) { fs::path env_directory; SplitWalletPath(wallet_path, env_directory, database_filename); LOCK(cs_db); auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr()); if (inserted.second) { auto env = std::make_shared(env_directory.string()); inserted.first->second = env; return env; } return inserted.first->second.lock(); } // // BerkeleyBatch // void BerkeleyEnvironment::Close() { if (!fDbEnvInit) { return; } fDbEnvInit = false; for (auto &db : m_databases) { auto count = mapFileUseCount.find(db.first); assert(count == mapFileUseCount.end() || count->second == 0); BerkeleyDatabase &database = db.second.get(); if (database.m_db) { database.m_db->close(0); database.m_db.reset(); } } FILE *error_file = nullptr; dbenv->get_errfile(&error_file); int ret = dbenv->close(0); if (ret != 0) { LogPrintf("BerkeleyEnvironment::Close: Error %d closing database " "environment: %s\n", ret, DbEnv::strerror(ret)); } if (!fMockDb) { DbEnv(u_int32_t(0)).remove(strPath.c_str(), 0); } if (error_file) { fclose(error_file); } UnlockDirectory(strPath, ".walletlock"); } void BerkeleyEnvironment::Reset() { dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS)); fDbEnvInit = false; fMockDb = false; } BerkeleyEnvironment::BerkeleyEnvironment(const fs::path &dir_path) : strPath(dir_path.string()) { Reset(); } BerkeleyEnvironment::~BerkeleyEnvironment() { LOCK(cs_db); g_dbenvs.erase(strPath); Close(); } bool BerkeleyEnvironment::Open(bool retry) { if (fDbEnvInit) { return true; } fs::path pathIn = strPath; TryCreateDirectories(pathIn); if (!LockDirectory(pathIn, ".walletlock")) { LogPrintf("Cannot obtain a lock on wallet directory %s. Another " "instance of bitcoin may be using it.\n", strPath); return false; } fs::path pathLogDir = pathIn / "database"; TryCreateDirectories(pathLogDir); fs::path pathErrorFile = pathIn / "db.log"; LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string()); unsigned int nEnvFlags = 0; if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) { nEnvFlags |= DB_PRIVATE; } dbenv->set_lg_dir(pathLogDir.string().c_str()); // 1 MiB should be enough for just the wallet dbenv->set_cachesize(0, 0x100000, 1); dbenv->set_lg_bsize(0x10000); dbenv->set_lg_max(1048576); dbenv->set_lk_max_locks(40000); dbenv->set_lk_max_objects(40000); /// debug dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); dbenv->set_flags(DB_AUTO_COMMIT, 1); dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1); dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1); int ret = dbenv->open(strPath.c_str(), DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD | DB_RECOVER | nEnvFlags, S_IRUSR | S_IWUSR); if (ret != 0) { LogPrintf("BerkeleyEnvironment::Open: Error %d opening database " "environment: %s\n", ret, DbEnv::strerror(ret)); int ret2 = dbenv->close(0); if (ret2 != 0) { LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed " "database environment: %s\n", ret2, DbEnv::strerror(ret2)); } Reset(); if (retry) { // try moving the database env out of the way fs::path pathDatabaseBak = pathIn / strprintf("database.%d.bak", GetTime()); try { fs::rename(pathLogDir, pathDatabaseBak); LogPrintf("Moved old %s to %s. Retrying.\n", pathLogDir.string(), pathDatabaseBak.string()); } catch (const fs::filesystem_error &) { // failure is ok (well, not really, but it's not worse than what // we started with) } // try opening it again one more time if (!Open(false /* retry */)) { // if it still fails, it probably means we can't even create the // database env return false; } } else { return false; } } fDbEnvInit = true; fMockDb = false; return true; } //! Construct an in-memory mock Berkeley environment for testing BerkeleyEnvironment::BerkeleyEnvironment() { Reset(); LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n"); dbenv->set_cachesize(1, 0, 1); dbenv->set_lg_bsize(10485760 * 4); dbenv->set_lg_max(10485760); dbenv->set_lk_max_locks(10000); dbenv->set_lk_max_objects(10000); dbenv->set_flags(DB_AUTO_COMMIT, 1); dbenv->log_set_config(DB_LOG_IN_MEMORY, 1); int ret = dbenv->open(nullptr, DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD | DB_PRIVATE, S_IRUSR | S_IWUSR); if (ret > 0) { throw std::runtime_error( strprintf("BerkeleyEnvironment::MakeMock: Error %d opening " "database environment.", ret)); } fDbEnvInit = true; fMockDb = true; } bool BerkeleyEnvironment::Verify(const std::string &strFile) { LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); Db db(dbenv.get(), 0); int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); return result == 0; } BerkeleyBatch::SafeDbt::SafeDbt() { m_dbt.set_flags(DB_DBT_MALLOC); } BerkeleyBatch::SafeDbt::SafeDbt(void *data, size_t size) : m_dbt(data, size) {} BerkeleyBatch::SafeDbt::~SafeDbt() { if (m_dbt.get_data() != nullptr) { // Clear memory, e.g. in case it was a private key memory_cleanse(m_dbt.get_data(), m_dbt.get_size()); // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be // freed by the caller. // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html if (m_dbt.get_flags() & DB_DBT_MALLOC) { free(m_dbt.get_data()); } } } const void *BerkeleyBatch::SafeDbt::get_data() const { return m_dbt.get_data(); } u_int32_t BerkeleyBatch::SafeDbt::get_size() const { return m_dbt.get_size(); } BerkeleyBatch::SafeDbt::operator Dbt *() { return &m_dbt; } bool BerkeleyBatch::VerifyEnvironment(const fs::path &file_path, bilingual_str &errorStr) { std::string walletFile; std::shared_ptr env = GetWalletEnv(file_path, walletFile); fs::path walletDir = env->Directory(); LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(nullptr, nullptr, nullptr)); LogPrintf("Using wallet %s\n", file_path.string()); if (!env->Open(true /* retry */)) { errorStr = strprintf( _("Error initializing wallet database environment %s!"), walletDir); return false; } return true; } bool BerkeleyBatch::VerifyDatabaseFile(const fs::path &file_path, bilingual_str &errorStr) { std::string walletFile; std::shared_ptr env = GetWalletEnv(file_path, walletFile); fs::path walletDir = env->Directory(); if (fs::exists(walletDir / walletFile)) { if (!env->Verify(walletFile)) { errorStr = strprintf(_("%s corrupt. Try using the wallet tool " "bitcoin-wallet to salvage or restoring a backup."), walletFile); return false; } } // also return true if files does not exists return true; } void BerkeleyEnvironment::CheckpointLSN(const std::string &strFile) { dbenv->txn_checkpoint(0, 0, 0); if (fMockDb) { return; } dbenv->lsn_reset(strFile.c_str(), 0); } BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase &database, const char *pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr) { fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); fFlushOnClose = fFlushOnCloseIn; env = database.env.get(); if (database.IsDummy()) { return; } const std::string &strFilename = database.strFile; bool fCreate = strchr(pszMode, 'c') != nullptr; unsigned int nFlags = DB_THREAD; if (fCreate) { nFlags |= DB_CREATE; } { LOCK(cs_db); if (!env->Open(false /* retry */)) { throw std::runtime_error( "BerkeleyBatch: Failed to open database environment."); } pdb = database.m_db.get(); if (pdb == nullptr) { int ret; std::unique_ptr pdb_temp = std::make_unique(env->dbenv.get(), 0); bool fMockDb = env->IsMock(); if (fMockDb) { DbMpoolFile *mpf = pdb_temp->get_mpf(); ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); if (ret != 0) { throw std::runtime_error( strprintf("BerkeleyBatch: Failed to configure for no " "temp file backing for database %s", strFilename)); } } ret = pdb_temp->open( nullptr, // Txn pointer fMockDb ? nullptr : strFilename.c_str(), // Filename fMockDb ? strFilename.c_str() : "main", // Logical db name DB_BTREE, // Database type nFlags, // Flags 0); if (ret != 0) { throw std::runtime_error( strprintf("BerkeleyBatch: Error %d, can't open database %s", ret, strFilename)); } // Call CheckUniqueFileid on the containing BDB environment to // avoid BDB data consistency bugs that happen when different data // files in the same environment have the same fileid. // // Also call CheckUniqueFileid on all the other g_dbenvs to prevent // bitcoin from opening the same data file through another // environment when the file is referenced through equivalent but // not obviously identical symlinked or hard linked or bind mounted // paths. In the future a more relaxed check for equal inode and // device ids could be done instead, which would allow opening // different backup copies of a wallet at the same time. Maybe even // more ideally, an exclusive lock for accessing the database could // be implemented, so no equality checks are needed at all. (Newer // versions of BDB have an set_lk_exclusive method for this // purpose, but the older version we use does not.) for (const auto &dbenv : g_dbenvs) { CheckUniqueFileid(*dbenv.second.lock().get(), strFilename, *pdb_temp, this->env->m_fileids[strFilename]); } pdb = pdb_temp.release(); database.m_db.reset(pdb); if (fCreate && !Exists(std::string("version"))) { bool fTmp = fReadOnly; fReadOnly = false; Write(std::string("version"), CLIENT_VERSION); fReadOnly = fTmp; } } ++env->mapFileUseCount[strFilename]; strFile = strFilename; } } void BerkeleyBatch::Flush() { if (activeTxn) { return; } // Flush database activity from memory pool to disk log unsigned int nMinutes = 0; if (fReadOnly) { nMinutes = 1; } // env is nullptr for dummy databases (i.e. in tests). Don't actually flush // if env is nullptr so we don't segfault if (env) { env->dbenv->txn_checkpoint( nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); } } void BerkeleyDatabase::IncrementUpdateCounter() { ++nUpdateCounter; } void BerkeleyBatch::Close() { if (!pdb) { return; } if (activeTxn) { activeTxn->abort(); } activeTxn = nullptr; pdb = nullptr; if (fFlushOnClose) { Flush(); } { LOCK(cs_db); --env->mapFileUseCount[strFile]; } env->m_db_in_use.notify_all(); } void BerkeleyEnvironment::CloseDb(const std::string &strFile) { LOCK(cs_db); auto it = m_databases.find(strFile); assert(it != m_databases.end()); BerkeleyDatabase &database = it->second.get(); if (database.m_db) { // Close the database handle database.m_db->close(0); database.m_db.reset(); } } void BerkeleyEnvironment::ReloadDbEnv() { // Make sure that no Db's are in use AssertLockNotHeld(cs_db); std::unique_lock lock(cs_db); m_db_in_use.wait(lock, [this]() { for (auto &count : mapFileUseCount) { if (count.second > 0) { return false; } } return true; }); std::vector filenames; for (auto it : m_databases) { filenames.push_back(it.first); } // Close the individual Db's for (const std::string &filename : filenames) { CloseDb(filename); } // Reset the environment // This will flush and close the environment Flush(true); Reset(); Open(true); } bool BerkeleyBatch::Rewrite(BerkeleyDatabase &database, const char *pszSkip) { if (database.IsDummy()) { return true; } BerkeleyEnvironment *env = database.env.get(); const std::string &strFile = database.strFile; while (true) { { LOCK(cs_db); if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) { // Flush log data to the dat file env->CloseDb(strFile); env->CheckpointLSN(strFile); env->mapFileUseCount.erase(strFile); bool fSuccess = true; LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile); std::string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} BerkeleyBatch db(database, "r"); std::unique_ptr pdbCopy = std::make_unique(env->dbenv.get(), 0); int ret = pdbCopy->open(nullptr, // Txn pointer strFileRes.c_str(), // Filename "main", // Logical db name DB_BTREE, // Database type DB_CREATE, // Flags 0); if (ret > 0) { LogPrintf("BerkeleyBatch::Rewrite: Can't create " "database file %s\n", strFileRes); fSuccess = false; } Dbc *pcursor = db.GetCursor(); if (pcursor) { while (fSuccess) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue); if (ret1 == DB_NOTFOUND) { pcursor->close(); break; } if (ret1 != 0) { pcursor->close(); fSuccess = false; break; } if (pszSkip && strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) { continue; } if (strncmp(ssKey.data(), "\x07version", 8) == 0) { // Update version: ssValue.clear(); ssValue << CLIENT_VERSION; } Dbt datKey(ssKey.data(), ssKey.size()); Dbt datValue(ssValue.data(), ssValue.size()); int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE); if (ret2 > 0) { fSuccess = false; } } } if (fSuccess) { db.Close(); env->CloseDb(strFile); if (pdbCopy->close(0)) { fSuccess = false; } } else { pdbCopy->close(0); } } if (fSuccess) { Db dbA(env->dbenv.get(), 0); if (dbA.remove(strFile.c_str(), nullptr, 0)) { fSuccess = false; } Db dbB(env->dbenv.get(), 0); if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0)) { fSuccess = false; } } if (!fSuccess) { LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite " "database file %s\n", strFileRes); } return fSuccess; } } UninterruptibleSleep(std::chrono::milliseconds{100}); } } void BerkeleyEnvironment::Flush(bool fShutdown) { int64_t nStart = GetTimeMillis(); // Flush log data to the actual data file on all files that are not in use LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); if (!fDbEnvInit) { return; } { LOCK(cs_db); std::map::iterator mi = mapFileUseCount.begin(); while (mi != mapFileUseCount.end()) { std::string strFile = (*mi).first; int nRefCount = (*mi).second; LogPrint( BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); if (nRefCount == 0) { // Move log data to the dat file CloseDb(strFile); LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); dbenv->txn_checkpoint(0, 0, 0); LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s detach\n", strFile); if (!fMockDb) { dbenv->lsn_reset(strFile.c_str(), 0); } LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s closed\n", strFile); mapFileUseCount.erase(mi++); } else { mi++; } } LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); if (fShutdown) { char **listp; if (mapFileUseCount.empty()) { dbenv->log_archive(&listp, DB_ARCH_REMOVE); Close(); if (!fMockDb) { fs::remove_all(fs::path(strPath) / "database"); } } } } } bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase &database) { if (database.IsDummy()) { return true; } bool ret = false; BerkeleyEnvironment *env = database.env.get(); const std::string &strFile = database.strFile; TRY_LOCK(cs_db, lockDb); if (lockDb) { // Don't do this if any databases are in use int nRefCount = 0; std::map::iterator mit = env->mapFileUseCount.begin(); while (mit != env->mapFileUseCount.end()) { nRefCount += (*mit).second; mit++; } if (nRefCount == 0) { boost::this_thread::interruption_point(); std::map::iterator mi = env->mapFileUseCount.find(strFile); if (mi != env->mapFileUseCount.end()) { LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); int64_t nStart = GetTimeMillis(); // Flush wallet file so it's self contained env->CloseDb(strFile); env->CheckpointLSN(strFile); env->mapFileUseCount.erase(mi++); LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); ret = true; } } } return ret; } bool BerkeleyDatabase::Rewrite(const char *pszSkip) { return BerkeleyBatch::Rewrite(*this, pszSkip); } bool BerkeleyDatabase::Backup(const std::string &strDest) const { if (IsDummy()) { return false; } while (true) { { LOCK(cs_db); if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) { // Flush log data to the dat file env->CloseDb(strFile); env->CheckpointLSN(strFile); env->mapFileUseCount.erase(strFile); // Copy wallet file. fs::path pathSrc = env->Directory() / strFile; fs::path pathDest(strDest); if (fs::is_directory(pathDest)) { pathDest /= strFile; } try { if (fs::equivalent(pathSrc, pathDest)) { LogPrintf("cannot backup to wallet source file %s\n", pathDest.string()); return false; } fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); LogPrintf("copied %s to %s\n", strFile, pathDest.string()); return true; } catch (const fs::filesystem_error &e) { LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), fsbridge::get_filesystem_error_message(e)); return false; } } } UninterruptibleSleep(std::chrono::milliseconds{100}); } } void BerkeleyDatabase::Flush(bool shutdown) { if (!IsDummy()) { env->Flush(shutdown); if (shutdown) { LOCK(cs_db); g_dbenvs.erase(env->Directory().string()); env = nullptr; } else { // TODO: To avoid g_dbenvs.erase erasing the environment prematurely // after the first database shutdown when multiple databases are // open in the same environment, should replace raw database `env` // pointers with shared or weak pointers, or else separate the // database and environment shutdowns so environments can be shut // down after databases. env->m_fileids.erase(strFile); } } } void BerkeleyDatabase::ReloadDbEnv() { if (!IsDummy()) { env->ReloadDbEnv(); } } Dbc *BerkeleyBatch::GetCursor() { if (!pdb) { return nullptr; } Dbc *pcursor = nullptr; int ret = pdb->cursor(nullptr, &pcursor, 0); if (ret != 0) { return nullptr; } return pcursor; } int BerkeleyBatch::ReadAtCursor(Dbc *pcursor, CDataStream &ssKey, CDataStream &ssValue) { // Read at cursor SafeDbt datKey; SafeDbt datValue; int ret = pcursor->get(datKey, datValue, DB_NEXT); if (ret != 0) { return ret; } else if (datKey.get_data() == nullptr || datValue.get_data() == nullptr) { return 99999; } // Convert to streams ssKey.SetType(SER_DISK); ssKey.clear(); ssKey.write((char *)datKey.get_data(), datKey.get_size()); ssValue.SetType(SER_DISK); ssValue.clear(); ssValue.write((char *)datValue.get_data(), datValue.get_size()); return 0; } bool BerkeleyBatch::TxnBegin() { if (!pdb || activeTxn) { return false; } DbTxn *ptxn = env->TxnBegin(); if (!ptxn) { return false; } activeTxn = ptxn; return true; } bool BerkeleyBatch::TxnCommit() { if (!pdb || !activeTxn) { return false; } int ret = activeTxn->commit(0); activeTxn = nullptr; return (ret == 0); } bool BerkeleyBatch::TxnAbort() { if (!pdb || !activeTxn) { return false; } int ret = activeTxn->abort(); activeTxn = nullptr; return (ret == 0); } diff --git a/src/wallet/db.h b/src/wallet/bdb.h similarity index 93% copy from src/wallet/db.h copy to src/wallet/bdb.h index 662a79071..c18863073 100644 --- a/src/wallet/db.h +++ b/src/wallet/bdb.h @@ -1,358 +1,356 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_WALLET_DB_H -#define BITCOIN_WALLET_DB_H +#ifndef BITCOIN_WALLET_BDB_H +#define BITCOIN_WALLET_BDB_H #include #include #include #include #include +#include #include #include #include #include #include #include #include struct bilingual_str; static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100; static const bool DEFAULT_WALLET_PRIVDB = true; struct WalletDatabaseFileId { u_int8_t value[DB_FILE_ID_LEN]; bool operator==(const WalletDatabaseFileId &rhs) const; }; class BerkeleyDatabase; class BerkeleyEnvironment { private: bool fDbEnvInit; bool fMockDb; // Don't change into fs::path, as that can result in // shutdown problems/crashes caused by a static initialized internal // pointer. std::string strPath; public: std::unique_ptr dbenv; std::map mapFileUseCount; std::map> m_databases; std::unordered_map m_fileids; std::condition_variable_any m_db_in_use; BerkeleyEnvironment(const fs::path &env_directory); BerkeleyEnvironment(); ~BerkeleyEnvironment(); void Reset(); void MakeMock(); bool IsMock() const { return fMockDb; } bool IsInitialized() const { return fDbEnvInit; } bool IsDatabaseLoaded(const std::string &db_filename) const { return m_databases.find(db_filename) != m_databases.end(); } fs::path Directory() const { return strPath; } bool Verify(const std::string &strFile); bool Open(bool retry); void Close(); void Flush(bool fShutdown); void CheckpointLSN(const std::string &strFile); void CloseDb(const std::string &strFile); void ReloadDbEnv(); DbTxn *TxnBegin(int flags = DB_TXN_WRITE_NOSYNC) { DbTxn *ptxn = nullptr; int ret = dbenv->txn_begin(nullptr, &ptxn, flags); - if (!ptxn || ret != 0) return nullptr; + if (!ptxn || ret != 0) { + return nullptr; + } return ptxn; } }; -/** Return whether a wallet database is currently loaded. */ -bool IsBDBWalletLoaded(const fs::path &wallet_path); - -/** - * Given a wallet directory path or legacy file path, return path to main data - * file in the wallet database. */ -fs::path WalletDataFilePath(const fs::path &wallet_path); -void SplitWalletPath(const fs::path &wallet_path, fs::path &env_directory, - std::string &database_filename); - /** Get BerkeleyEnvironment and database filename given a wallet path. */ std::shared_ptr GetWalletEnv(const fs::path &wallet_path, std::string &database_filename); +/** Return whether a BDB wallet database is currently loaded. */ +bool IsBDBWalletLoaded(const fs::path &wallet_path); + /** * An instance of this class represents one database. * For BerkeleyDB this is just a (env, strFile) tuple. */ class BerkeleyDatabase { friend class BerkeleyBatch; public: /** Create dummy DB handle */ BerkeleyDatabase() : nUpdateCounter(0), nLastSeen(0), nLastFlushed(0), nLastWalletUpdate(0), env(nullptr) {} /** Create DB handle to real database */ BerkeleyDatabase(std::shared_ptr envIn, std::string filename) : nUpdateCounter(0), nLastSeen(0), nLastFlushed(0), nLastWalletUpdate(0), env(std::move(envIn)), strFile(std::move(filename)) { auto inserted = this->env->m_databases.emplace(strFile, std::ref(*this)); assert(inserted.second); } ~BerkeleyDatabase() { if (env) { size_t erased = env->m_databases.erase(strFile); assert(erased == 1); } } /** Return object for accessing database at specified path. */ static std::unique_ptr Create(const fs::path &path) { std::string filename; return std::make_unique(GetWalletEnv(path, filename), std::move(filename)); } /** * Return object for accessing dummy database with no read/write * capabilities. */ static std::unique_ptr CreateDummy() { return std::make_unique(); } /** * Return object for accessing temporary in-memory database. */ static std::unique_ptr CreateMock() { return std::make_unique( std::make_shared(), ""); } /** * Rewrite the entire database on disk, with the exception of key pszSkip if * non-zero */ bool Rewrite(const char *pszSkip = nullptr); /** * Back up the entire database to a file. */ bool Backup(const std::string &strDest) const; /** * Make sure all changes are flushed to disk. */ void Flush(bool shutdown); void IncrementUpdateCounter(); void ReloadDbEnv(); std::atomic nUpdateCounter; unsigned int nLastSeen; unsigned int nLastFlushed; int64_t nLastWalletUpdate; /** * Pointer to shared database environment. * * Normally there is only one BerkeleyDatabase object per * BerkeleyEnvivonment, but in the special, backwards compatible case where * multiple wallet BDB data files are loaded from the same directory, this * will point to a shared instance that gets freed when the last data file * is closed. */ std::shared_ptr env; /** * Database pointer. This is initialized lazily and reset during flushes, * so it can be null. */ std::unique_ptr m_db; private: std::string strFile; /** * Return whether this database handle is a dummy for testing. * Only to be used at a low level, application should ideally not care * about this. */ bool IsDummy() const { return env == nullptr; } }; /** RAII class that provides access to a Berkeley database */ class BerkeleyBatch { /** RAII class that automatically cleanses its data on destruction */ class SafeDbt final { Dbt m_dbt; public: // construct Dbt with internally-managed data SafeDbt(); // construct Dbt with provided data SafeDbt(void *data, size_t size); ~SafeDbt(); // delegate to Dbt const void *get_data() const; u_int32_t get_size() const; // conversion operator to access the underlying Dbt operator Dbt *(); }; protected: Db *pdb; std::string strFile; DbTxn *activeTxn; bool fReadOnly; bool fFlushOnClose; BerkeleyEnvironment *env; public: explicit BerkeleyBatch(BerkeleyDatabase &database, const char *pszMode = "r+", bool fFlushOnCloseIn = true); ~BerkeleyBatch() { Close(); } BerkeleyBatch(const BerkeleyBatch &) = delete; BerkeleyBatch &operator=(const BerkeleyBatch &) = delete; void Flush(); void Close(); - /* flush the wallet passively (TRY_LOCK) - ideal to be called periodically */ + /* + * flush the wallet passively (TRY_LOCK)i + * ideal to be called periodically + */ static bool PeriodicFlush(BerkeleyDatabase &database); /* verifies the database environment */ static bool VerifyEnvironment(const fs::path &file_path, bilingual_str &errorStr); /* verifies the database file */ static bool VerifyDatabaseFile(const fs::path &file_path, bilingual_str &errorStr); template bool Read(const K &key, T &value) { if (!pdb) { return false; } // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; SafeDbt datKey(ssKey.data(), ssKey.size()); // Read SafeDbt datValue; int ret = pdb->get(activeTxn, datKey, datValue, 0); bool success = false; if (datValue.get_data() != nullptr) { // Unserialize value try { CDataStream ssValue((char *)datValue.get_data(), (char *)datValue.get_data() + datValue.get_size(), SER_DISK, CLIENT_VERSION); ssValue >> value; success = true; } catch (const std::exception &) { // In this case success remains 'false' } } return ret == 0 && success; } template bool Write(const K &key, const T &value, bool fOverwrite = true) { if (!pdb) { return true; } if (fReadOnly) { assert(!"Write called on database in read-only mode"); } // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; SafeDbt datKey(ssKey.data(), ssKey.size()); // Value CDataStream ssValue(SER_DISK, CLIENT_VERSION); ssValue.reserve(10000); ssValue << value; SafeDbt datValue(ssValue.data(), ssValue.size()); // Write int ret = pdb->put(activeTxn, datKey, datValue, (fOverwrite ? 0 : DB_NOOVERWRITE)); return (ret == 0); } template bool Erase(const K &key) { if (!pdb) { return false; } if (fReadOnly) { assert(!"Erase called on database in read-only mode"); } // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; SafeDbt datKey(ssKey.data(), ssKey.size()); // Erase int ret = pdb->del(activeTxn, datKey, 0); return (ret == 0 || ret == DB_NOTFOUND); } template bool Exists(const K &key) { if (!pdb) { return false; } // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; SafeDbt datKey(ssKey.data(), ssKey.size()); // Exists int ret = pdb->exists(activeTxn, datKey, 0); return (ret == 0); } Dbc *GetCursor(); int ReadAtCursor(Dbc *pcursor, CDataStream &ssKey, CDataStream &ssValue); bool TxnBegin(); bool TxnCommit(); bool TxnAbort(); static bool Rewrite(BerkeleyDatabase &database, const char *pszSkip = nullptr); }; -#endif // BITCOIN_WALLET_DB_H +#endif // BITCOIN_WALLET_BDB_H diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 2b19b4122..37fdf43c5 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -1,904 +1,32 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include -#include -#include - -#include // boost::this_thread::interruption_point() (mingw) - -#include -#ifndef WIN32 -#include -#endif - -namespace { -//! Make sure database has a unique fileid within the environment. If it -//! doesn't, throw an error. BDB caches do not work properly when more than one -//! open database has the same fileid (values written to one database may show -//! up in reads to other databases). -//! -//! BerkeleyDB generates unique fileids by default -//! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html), -//! so bitcoin should never create different databases with the same fileid, but -//! this error can be triggered if users manually copy database files. -void CheckUniqueFileid(const BerkeleyEnvironment &env, - const std::string &filename, Db &db, - WalletDatabaseFileId &fileid) { - if (env.IsMock()) { - return; - } - - int ret = db.get_mpf()->get_fileid(fileid.value); - if (ret != 0) { - throw std::runtime_error(strprintf( - "BerkeleyBatch: Can't open database %s (get_fileid failed with %d)", - filename, ret)); - } - - for (const auto &item : env.m_fileids) { - if (fileid == item.second && &fileid != &item.second) { - throw std::runtime_error(strprintf( - "BerkeleyBatch: Can't open database %s (duplicates fileid %s " - "from %s)", - filename, - HexStr(std::begin(item.second.value), - std::end(item.second.value)), - item.first)); - } - } -} - -RecursiveMutex cs_db; - -//! Map from directory name to db environment. -std::map> - g_dbenvs GUARDED_BY(cs_db); -} // namespace - -bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId &rhs) const { - return memcmp(value, &rhs.value, sizeof(value)) == 0; -} +#include void SplitWalletPath(const fs::path &wallet_path, fs::path &env_directory, std::string &database_filename) { if (fs::is_regular_file(wallet_path)) { // Special case for backwards compatibility: if wallet path points to an // existing file, treat it as the path to a BDB data file in a parent // directory that also contains BDB log files. env_directory = wallet_path.parent_path(); database_filename = wallet_path.filename().string(); } else { // Normal case: Interpret wallet path as a directory path containing // data and log files. env_directory = wallet_path; database_filename = "wallet.dat"; } } -bool IsBDBWalletLoaded(const fs::path &wallet_path) { - fs::path env_directory; - std::string database_filename; - SplitWalletPath(wallet_path, env_directory, database_filename); - - LOCK(cs_db); - auto env = g_dbenvs.find(env_directory.string()); - if (env == g_dbenvs.end()) { - return false; - } - - auto database = env->second.lock(); - return database && database->IsDatabaseLoaded(database_filename); -} - fs::path WalletDataFilePath(const fs::path &wallet_path) { fs::path env_directory; std::string database_filename; SplitWalletPath(wallet_path, env_directory, database_filename); return env_directory / database_filename; } - -/** - * @param[in] wallet_path Path to wallet directory. Or (for backwards - * compatibility only) a path to a berkeley btree data file inside a wallet - * directory. - * @param[out] database_filename Filename of berkeley btree data file inside the - * wallet directory. - * @return A shared pointer to the BerkeleyEnvironment object for the wallet - * directory, never empty because ~BerkeleyEnvironment erases the weak pointer - * from the g_dbenvs map. - * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the - * directory path key was not already in the map. - */ -std::shared_ptr -GetWalletEnv(const fs::path &wallet_path, std::string &database_filename) { - fs::path env_directory; - SplitWalletPath(wallet_path, env_directory, database_filename); - LOCK(cs_db); - auto inserted = g_dbenvs.emplace(env_directory.string(), - std::weak_ptr()); - if (inserted.second) { - auto env = - std::make_shared(env_directory.string()); - inserted.first->second = env; - return env; - } - return inserted.first->second.lock(); -} - -// -// BerkeleyBatch -// - -void BerkeleyEnvironment::Close() { - if (!fDbEnvInit) { - return; - } - - fDbEnvInit = false; - - for (auto &db : m_databases) { - auto count = mapFileUseCount.find(db.first); - assert(count == mapFileUseCount.end() || count->second == 0); - BerkeleyDatabase &database = db.second.get(); - if (database.m_db) { - database.m_db->close(0); - database.m_db.reset(); - } - } - - FILE *error_file = nullptr; - dbenv->get_errfile(&error_file); - - int ret = dbenv->close(0); - if (ret != 0) { - LogPrintf("BerkeleyEnvironment::Close: Error %d closing database " - "environment: %s\n", - ret, DbEnv::strerror(ret)); - } - if (!fMockDb) { - DbEnv(u_int32_t(0)).remove(strPath.c_str(), 0); - } - - if (error_file) { - fclose(error_file); - } - - UnlockDirectory(strPath, ".walletlock"); -} - -void BerkeleyEnvironment::Reset() { - dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS)); - fDbEnvInit = false; - fMockDb = false; -} - -BerkeleyEnvironment::BerkeleyEnvironment(const fs::path &dir_path) - : strPath(dir_path.string()) { - Reset(); -} - -BerkeleyEnvironment::~BerkeleyEnvironment() { - LOCK(cs_db); - g_dbenvs.erase(strPath); - Close(); -} - -bool BerkeleyEnvironment::Open(bool retry) { - if (fDbEnvInit) { - return true; - } - - fs::path pathIn = strPath; - TryCreateDirectories(pathIn); - if (!LockDirectory(pathIn, ".walletlock")) { - LogPrintf("Cannot obtain a lock on wallet directory %s. Another " - "instance of bitcoin may be using it.\n", - strPath); - return false; - } - - fs::path pathLogDir = pathIn / "database"; - TryCreateDirectories(pathLogDir); - fs::path pathErrorFile = pathIn / "db.log"; - LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", - pathLogDir.string(), pathErrorFile.string()); - - unsigned int nEnvFlags = 0; - if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) { - nEnvFlags |= DB_PRIVATE; - } - - dbenv->set_lg_dir(pathLogDir.string().c_str()); - // 1 MiB should be enough for just the wallet - dbenv->set_cachesize(0, 0x100000, 1); - dbenv->set_lg_bsize(0x10000); - dbenv->set_lg_max(1048576); - dbenv->set_lk_max_locks(40000); - dbenv->set_lk_max_objects(40000); - /// debug - dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); - dbenv->set_flags(DB_AUTO_COMMIT, 1); - dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1); - dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1); - int ret = - dbenv->open(strPath.c_str(), - DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | - DB_INIT_TXN | DB_THREAD | DB_RECOVER | nEnvFlags, - S_IRUSR | S_IWUSR); - if (ret != 0) { - LogPrintf("BerkeleyEnvironment::Open: Error %d opening database " - "environment: %s\n", - ret, DbEnv::strerror(ret)); - int ret2 = dbenv->close(0); - if (ret2 != 0) { - LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed " - "database environment: %s\n", - ret2, DbEnv::strerror(ret2)); - } - Reset(); - if (retry) { - // try moving the database env out of the way - fs::path pathDatabaseBak = - pathIn / strprintf("database.%d.bak", GetTime()); - try { - fs::rename(pathLogDir, pathDatabaseBak); - LogPrintf("Moved old %s to %s. Retrying.\n", - pathLogDir.string(), pathDatabaseBak.string()); - } catch (const fs::filesystem_error &) { - // failure is ok (well, not really, but it's not worse than what - // we started with) - } - // try opening it again one more time - if (!Open(false /* retry */)) { - // if it still fails, it probably means we can't even create the - // database env - return false; - } - } else { - return false; - } - } - - fDbEnvInit = true; - fMockDb = false; - return true; -} - -//! Construct an in-memory mock Berkeley environment for testing -BerkeleyEnvironment::BerkeleyEnvironment() { - Reset(); - - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n"); - - dbenv->set_cachesize(1, 0, 1); - dbenv->set_lg_bsize(10485760 * 4); - dbenv->set_lg_max(10485760); - dbenv->set_lk_max_locks(10000); - dbenv->set_lk_max_objects(10000); - dbenv->set_flags(DB_AUTO_COMMIT, 1); - dbenv->log_set_config(DB_LOG_IN_MEMORY, 1); - int ret = - dbenv->open(nullptr, - DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | - DB_INIT_TXN | DB_THREAD | DB_PRIVATE, - S_IRUSR | S_IWUSR); - if (ret > 0) { - throw std::runtime_error( - strprintf("BerkeleyEnvironment::MakeMock: Error %d opening " - "database environment.", - ret)); - } - - fDbEnvInit = true; - fMockDb = true; -} - -bool BerkeleyEnvironment::Verify(const std::string &strFile) { - LOCK(cs_db); - assert(mapFileUseCount.count(strFile) == 0); - - Db db(dbenv.get(), 0); - int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); - return result == 0; -} - -BerkeleyBatch::SafeDbt::SafeDbt() { - m_dbt.set_flags(DB_DBT_MALLOC); -} - -BerkeleyBatch::SafeDbt::SafeDbt(void *data, size_t size) : m_dbt(data, size) {} - -BerkeleyBatch::SafeDbt::~SafeDbt() { - if (m_dbt.get_data() != nullptr) { - // Clear memory, e.g. in case it was a private key - memory_cleanse(m_dbt.get_data(), m_dbt.get_size()); - // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be - // freed by the caller. - // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html - if (m_dbt.get_flags() & DB_DBT_MALLOC) { - free(m_dbt.get_data()); - } - } -} - -const void *BerkeleyBatch::SafeDbt::get_data() const { - return m_dbt.get_data(); -} - -u_int32_t BerkeleyBatch::SafeDbt::get_size() const { - return m_dbt.get_size(); -} - -BerkeleyBatch::SafeDbt::operator Dbt *() { - return &m_dbt; -} - -bool BerkeleyBatch::VerifyEnvironment(const fs::path &file_path, - bilingual_str &errorStr) { - std::string walletFile; - std::shared_ptr env = - GetWalletEnv(file_path, walletFile); - fs::path walletDir = env->Directory(); - - LogPrintf("Using BerkeleyDB version %s\n", - DbEnv::version(nullptr, nullptr, nullptr)); - LogPrintf("Using wallet %s\n", file_path.string()); - - if (!env->Open(true /* retry */)) { - errorStr = strprintf( - _("Error initializing wallet database environment %s!"), walletDir); - return false; - } - - return true; -} - -bool BerkeleyBatch::VerifyDatabaseFile(const fs::path &file_path, - bilingual_str &errorStr) { - std::string walletFile; - std::shared_ptr env = - GetWalletEnv(file_path, walletFile); - fs::path walletDir = env->Directory(); - - if (fs::exists(walletDir / walletFile)) { - if (!env->Verify(walletFile)) { - errorStr = - strprintf(_("%s corrupt. Try using the wallet tool " - "bitcoin-wallet to salvage or restoring a backup."), - walletFile); - return false; - } - } - // also return true if files does not exists - return true; -} - -void BerkeleyEnvironment::CheckpointLSN(const std::string &strFile) { - dbenv->txn_checkpoint(0, 0, 0); - if (fMockDb) { - return; - } - dbenv->lsn_reset(strFile.c_str(), 0); -} - -BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase &database, const char *pszMode, - bool fFlushOnCloseIn) - : pdb(nullptr), activeTxn(nullptr) { - fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); - fFlushOnClose = fFlushOnCloseIn; - env = database.env.get(); - if (database.IsDummy()) { - return; - } - - const std::string &strFilename = database.strFile; - - bool fCreate = strchr(pszMode, 'c') != nullptr; - unsigned int nFlags = DB_THREAD; - if (fCreate) { - nFlags |= DB_CREATE; - } - - { - LOCK(cs_db); - if (!env->Open(false /* retry */)) { - throw std::runtime_error( - "BerkeleyBatch: Failed to open database environment."); - } - - pdb = database.m_db.get(); - if (pdb == nullptr) { - int ret; - std::unique_ptr pdb_temp = - std::make_unique(env->dbenv.get(), 0); - - bool fMockDb = env->IsMock(); - if (fMockDb) { - DbMpoolFile *mpf = pdb_temp->get_mpf(); - ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); - if (ret != 0) { - throw std::runtime_error( - strprintf("BerkeleyBatch: Failed to configure for no " - "temp file backing for database %s", - strFilename)); - } - } - - ret = pdb_temp->open( - nullptr, // Txn pointer - fMockDb ? nullptr : strFilename.c_str(), // Filename - fMockDb ? strFilename.c_str() : "main", // Logical db name - DB_BTREE, // Database type - nFlags, // Flags - 0); - - if (ret != 0) { - throw std::runtime_error( - strprintf("BerkeleyBatch: Error %d, can't open database %s", - ret, strFilename)); - } - - // Call CheckUniqueFileid on the containing BDB environment to - // avoid BDB data consistency bugs that happen when different data - // files in the same environment have the same fileid. - // - // Also call CheckUniqueFileid on all the other g_dbenvs to prevent - // bitcoin from opening the same data file through another - // environment when the file is referenced through equivalent but - // not obviously identical symlinked or hard linked or bind mounted - // paths. In the future a more relaxed check for equal inode and - // device ids could be done instead, which would allow opening - // different backup copies of a wallet at the same time. Maybe even - // more ideally, an exclusive lock for accessing the database could - // be implemented, so no equality checks are needed at all. (Newer - // versions of BDB have an set_lk_exclusive method for this - // purpose, but the older version we use does not.) - for (const auto &dbenv : g_dbenvs) { - CheckUniqueFileid(*dbenv.second.lock().get(), strFilename, - *pdb_temp, this->env->m_fileids[strFilename]); - } - - pdb = pdb_temp.release(); - database.m_db.reset(pdb); - - if (fCreate && !Exists(std::string("version"))) { - bool fTmp = fReadOnly; - fReadOnly = false; - Write(std::string("version"), CLIENT_VERSION); - fReadOnly = fTmp; - } - } - ++env->mapFileUseCount[strFilename]; - strFile = strFilename; - } -} - -void BerkeleyBatch::Flush() { - if (activeTxn) { - return; - } - - // Flush database activity from memory pool to disk log - unsigned int nMinutes = 0; - if (fReadOnly) { - nMinutes = 1; - } - - // env is nullptr for dummy databases (i.e. in tests). Don't actually flush - // if env is nullptr so we don't segfault - if (env) { - env->dbenv->txn_checkpoint( - nMinutes - ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 - : 0, - nMinutes, 0); - } -} - -void BerkeleyDatabase::IncrementUpdateCounter() { - ++nUpdateCounter; -} - -void BerkeleyBatch::Close() { - if (!pdb) { - return; - } - if (activeTxn) { - activeTxn->abort(); - } - activeTxn = nullptr; - pdb = nullptr; - - if (fFlushOnClose) { - Flush(); - } - - { - LOCK(cs_db); - --env->mapFileUseCount[strFile]; - } - env->m_db_in_use.notify_all(); -} - -void BerkeleyEnvironment::CloseDb(const std::string &strFile) { - LOCK(cs_db); - auto it = m_databases.find(strFile); - assert(it != m_databases.end()); - BerkeleyDatabase &database = it->second.get(); - if (database.m_db) { - // Close the database handle - database.m_db->close(0); - database.m_db.reset(); - } -} - -void BerkeleyEnvironment::ReloadDbEnv() { - // Make sure that no Db's are in use - AssertLockNotHeld(cs_db); - std::unique_lock lock(cs_db); - m_db_in_use.wait(lock, [this]() { - for (auto &count : mapFileUseCount) { - if (count.second > 0) { - return false; - } - } - return true; - }); - - std::vector filenames; - for (auto it : m_databases) { - filenames.push_back(it.first); - } - // Close the individual Db's - for (const std::string &filename : filenames) { - CloseDb(filename); - } - // Reset the environment - // This will flush and close the environment - Flush(true); - Reset(); - Open(true); -} - -bool BerkeleyBatch::Rewrite(BerkeleyDatabase &database, const char *pszSkip) { - if (database.IsDummy()) { - return true; - } - BerkeleyEnvironment *env = database.env.get(); - const std::string &strFile = database.strFile; - while (true) { - { - LOCK(cs_db); - if (!env->mapFileUseCount.count(strFile) || - env->mapFileUseCount[strFile] == 0) { - // Flush log data to the dat file - env->CloseDb(strFile); - env->CheckpointLSN(strFile); - env->mapFileUseCount.erase(strFile); - - bool fSuccess = true; - LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile); - std::string strFileRes = strFile + ".rewrite"; - { - // surround usage of db with extra {} - BerkeleyBatch db(database, "r"); - std::unique_ptr pdbCopy = - std::make_unique(env->dbenv.get(), 0); - - int ret = pdbCopy->open(nullptr, // Txn pointer - strFileRes.c_str(), // Filename - "main", // Logical db name - DB_BTREE, // Database type - DB_CREATE, // Flags - 0); - if (ret > 0) { - LogPrintf("BerkeleyBatch::Rewrite: Can't create " - "database file %s\n", - strFileRes); - fSuccess = false; - } - - Dbc *pcursor = db.GetCursor(); - if (pcursor) { - while (fSuccess) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue); - if (ret1 == DB_NOTFOUND) { - pcursor->close(); - break; - } - if (ret1 != 0) { - pcursor->close(); - fSuccess = false; - break; - } - if (pszSkip && - strncmp(ssKey.data(), pszSkip, - std::min(ssKey.size(), - strlen(pszSkip))) == 0) { - continue; - } - if (strncmp(ssKey.data(), "\x07version", 8) == 0) { - // Update version: - ssValue.clear(); - ssValue << CLIENT_VERSION; - } - Dbt datKey(ssKey.data(), ssKey.size()); - Dbt datValue(ssValue.data(), ssValue.size()); - int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, - DB_NOOVERWRITE); - if (ret2 > 0) { - fSuccess = false; - } - } - } - if (fSuccess) { - db.Close(); - env->CloseDb(strFile); - if (pdbCopy->close(0)) { - fSuccess = false; - } - } else { - pdbCopy->close(0); - } - } - if (fSuccess) { - Db dbA(env->dbenv.get(), 0); - if (dbA.remove(strFile.c_str(), nullptr, 0)) { - fSuccess = false; - } - Db dbB(env->dbenv.get(), 0); - if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), - 0)) { - fSuccess = false; - } - } - if (!fSuccess) { - LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite " - "database file %s\n", - strFileRes); - } - return fSuccess; - } - } - UninterruptibleSleep(std::chrono::milliseconds{100}); - } -} - -void BerkeleyEnvironment::Flush(bool fShutdown) { - int64_t nStart = GetTimeMillis(); - // Flush log data to the actual data file on all files that are not in use - LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", - strPath, fShutdown ? "true" : "false", - fDbEnvInit ? "" : " database not started"); - if (!fDbEnvInit) { - return; - } - { - LOCK(cs_db); - std::map::iterator mi = mapFileUseCount.begin(); - while (mi != mapFileUseCount.end()) { - std::string strFile = (*mi).first; - int nRefCount = (*mi).second; - LogPrint( - BCLog::WALLETDB, - "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", - strFile, nRefCount); - if (nRefCount == 0) { - // Move log data to the dat file - CloseDb(strFile); - LogPrint(BCLog::WALLETDB, - "BerkeleyEnvironment::Flush: %s checkpoint\n", - strFile); - dbenv->txn_checkpoint(0, 0, 0); - LogPrint(BCLog::WALLETDB, - "BerkeleyEnvironment::Flush: %s detach\n", strFile); - if (!fMockDb) { - dbenv->lsn_reset(strFile.c_str(), 0); - } - LogPrint(BCLog::WALLETDB, - "BerkeleyEnvironment::Flush: %s closed\n", strFile); - mapFileUseCount.erase(mi++); - } else { - mi++; - } - } - LogPrint(BCLog::WALLETDB, - "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", - fShutdown ? "true" : "false", - fDbEnvInit ? "" : " database not started", - GetTimeMillis() - nStart); - if (fShutdown) { - char **listp; - if (mapFileUseCount.empty()) { - dbenv->log_archive(&listp, DB_ARCH_REMOVE); - Close(); - if (!fMockDb) { - fs::remove_all(fs::path(strPath) / "database"); - } - } - } - } -} - -bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase &database) { - if (database.IsDummy()) { - return true; - } - bool ret = false; - BerkeleyEnvironment *env = database.env.get(); - const std::string &strFile = database.strFile; - TRY_LOCK(cs_db, lockDb); - if (lockDb) { - // Don't do this if any databases are in use - int nRefCount = 0; - std::map::iterator mit = env->mapFileUseCount.begin(); - while (mit != env->mapFileUseCount.end()) { - nRefCount += (*mit).second; - mit++; - } - - if (nRefCount == 0) { - boost::this_thread::interruption_point(); - std::map::iterator mi = - env->mapFileUseCount.find(strFile); - if (mi != env->mapFileUseCount.end()) { - LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); - int64_t nStart = GetTimeMillis(); - - // Flush wallet file so it's self contained - env->CloseDb(strFile); - env->CheckpointLSN(strFile); - - env->mapFileUseCount.erase(mi++); - LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, - GetTimeMillis() - nStart); - ret = true; - } - } - } - - return ret; -} - -bool BerkeleyDatabase::Rewrite(const char *pszSkip) { - return BerkeleyBatch::Rewrite(*this, pszSkip); -} - -bool BerkeleyDatabase::Backup(const std::string &strDest) const { - if (IsDummy()) { - return false; - } - while (true) { - { - LOCK(cs_db); - if (!env->mapFileUseCount.count(strFile) || - env->mapFileUseCount[strFile] == 0) { - // Flush log data to the dat file - env->CloseDb(strFile); - env->CheckpointLSN(strFile); - env->mapFileUseCount.erase(strFile); - - // Copy wallet file. - fs::path pathSrc = env->Directory() / strFile; - fs::path pathDest(strDest); - if (fs::is_directory(pathDest)) { - pathDest /= strFile; - } - - try { - if (fs::equivalent(pathSrc, pathDest)) { - LogPrintf("cannot backup to wallet source file %s\n", - pathDest.string()); - return false; - } - - fs::copy_file(pathSrc, pathDest, - fs::copy_option::overwrite_if_exists); - LogPrintf("copied %s to %s\n", strFile, pathDest.string()); - return true; - } catch (const fs::filesystem_error &e) { - LogPrintf("error copying %s to %s - %s\n", strFile, - pathDest.string(), - fsbridge::get_filesystem_error_message(e)); - return false; - } - } - } - UninterruptibleSleep(std::chrono::milliseconds{100}); - } -} - -void BerkeleyDatabase::Flush(bool shutdown) { - if (!IsDummy()) { - env->Flush(shutdown); - if (shutdown) { - LOCK(cs_db); - g_dbenvs.erase(env->Directory().string()); - env = nullptr; - } else { - // TODO: To avoid g_dbenvs.erase erasing the environment prematurely - // after the first database shutdown when multiple databases are - // open in the same environment, should replace raw database `env` - // pointers with shared or weak pointers, or else separate the - // database and environment shutdowns so environments can be shut - // down after databases. - env->m_fileids.erase(strFile); - } - } -} - -void BerkeleyDatabase::ReloadDbEnv() { - if (!IsDummy()) { - env->ReloadDbEnv(); - } -} - -Dbc *BerkeleyBatch::GetCursor() { - if (!pdb) { - return nullptr; - } - Dbc *pcursor = nullptr; - int ret = pdb->cursor(nullptr, &pcursor, 0); - if (ret != 0) { - return nullptr; - } - return pcursor; -} - -int BerkeleyBatch::ReadAtCursor(Dbc *pcursor, CDataStream &ssKey, - CDataStream &ssValue) { - // Read at cursor - SafeDbt datKey; - SafeDbt datValue; - int ret = pcursor->get(datKey, datValue, DB_NEXT); - if (ret != 0) { - return ret; - } else if (datKey.get_data() == nullptr || datValue.get_data() == nullptr) { - return 99999; - } - - // Convert to streams - ssKey.SetType(SER_DISK); - ssKey.clear(); - ssKey.write((char *)datKey.get_data(), datKey.get_size()); - ssValue.SetType(SER_DISK); - ssValue.clear(); - ssValue.write((char *)datValue.get_data(), datValue.get_size()); - return 0; -} - -bool BerkeleyBatch::TxnBegin() { - if (!pdb || activeTxn) { - return false; - } - DbTxn *ptxn = env->TxnBegin(); - if (!ptxn) { - return false; - } - activeTxn = ptxn; - return true; -} - -bool BerkeleyBatch::TxnCommit() { - if (!pdb || !activeTxn) { - return false; - } - int ret = activeTxn->commit(0); - activeTxn = nullptr; - return (ret == 0); -} - -bool BerkeleyBatch::TxnAbort() { - if (!pdb || !activeTxn) { - return false; - } - int ret = activeTxn->abort(); - activeTxn = nullptr; - return (ret == 0); -} diff --git a/src/wallet/db.h b/src/wallet/db.h index 662a79071..874182b23 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -1,358 +1,20 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_WALLET_DB_H #define BITCOIN_WALLET_DB_H -#include #include -#include -#include -#include -#include - -#include -#include -#include #include -#include -#include - -struct bilingual_str; - -static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100; -static const bool DEFAULT_WALLET_PRIVDB = true; - -struct WalletDatabaseFileId { - u_int8_t value[DB_FILE_ID_LEN]; - bool operator==(const WalletDatabaseFileId &rhs) const; -}; - -class BerkeleyDatabase; - -class BerkeleyEnvironment { -private: - bool fDbEnvInit; - bool fMockDb; - // Don't change into fs::path, as that can result in - // shutdown problems/crashes caused by a static initialized internal - // pointer. - std::string strPath; - -public: - std::unique_ptr dbenv; - std::map mapFileUseCount; - std::map> m_databases; - std::unordered_map m_fileids; - std::condition_variable_any m_db_in_use; - - BerkeleyEnvironment(const fs::path &env_directory); - BerkeleyEnvironment(); - ~BerkeleyEnvironment(); - void Reset(); - - void MakeMock(); - bool IsMock() const { return fMockDb; } - bool IsInitialized() const { return fDbEnvInit; } - bool IsDatabaseLoaded(const std::string &db_filename) const { - return m_databases.find(db_filename) != m_databases.end(); - } - fs::path Directory() const { return strPath; } - - bool Verify(const std::string &strFile); - - bool Open(bool retry); - void Close(); - void Flush(bool fShutdown); - void CheckpointLSN(const std::string &strFile); - - void CloseDb(const std::string &strFile); - void ReloadDbEnv(); - - DbTxn *TxnBegin(int flags = DB_TXN_WRITE_NOSYNC) { - DbTxn *ptxn = nullptr; - int ret = dbenv->txn_begin(nullptr, &ptxn, flags); - if (!ptxn || ret != 0) return nullptr; - return ptxn; - } -}; - -/** Return whether a wallet database is currently loaded. */ -bool IsBDBWalletLoaded(const fs::path &wallet_path); /** * Given a wallet directory path or legacy file path, return path to main data * file in the wallet database. */ fs::path WalletDataFilePath(const fs::path &wallet_path); void SplitWalletPath(const fs::path &wallet_path, fs::path &env_directory, std::string &database_filename); -/** Get BerkeleyEnvironment and database filename given a wallet path. */ -std::shared_ptr -GetWalletEnv(const fs::path &wallet_path, std::string &database_filename); - -/** - * An instance of this class represents one database. - * For BerkeleyDB this is just a (env, strFile) tuple. - */ -class BerkeleyDatabase { - friend class BerkeleyBatch; - -public: - /** Create dummy DB handle */ - BerkeleyDatabase() - : nUpdateCounter(0), nLastSeen(0), nLastFlushed(0), - nLastWalletUpdate(0), env(nullptr) {} - - /** Create DB handle to real database */ - BerkeleyDatabase(std::shared_ptr envIn, - std::string filename) - : nUpdateCounter(0), nLastSeen(0), nLastFlushed(0), - nLastWalletUpdate(0), env(std::move(envIn)), - strFile(std::move(filename)) { - auto inserted = - this->env->m_databases.emplace(strFile, std::ref(*this)); - assert(inserted.second); - } - - ~BerkeleyDatabase() { - if (env) { - size_t erased = env->m_databases.erase(strFile); - assert(erased == 1); - } - } - - /** Return object for accessing database at specified path. */ - static std::unique_ptr Create(const fs::path &path) { - std::string filename; - return std::make_unique(GetWalletEnv(path, filename), - std::move(filename)); - } - - /** - * Return object for accessing dummy database with no read/write - * capabilities. - */ - static std::unique_ptr CreateDummy() { - return std::make_unique(); - } - - /** - * Return object for accessing temporary in-memory database. - */ - static std::unique_ptr CreateMock() { - return std::make_unique( - std::make_shared(), ""); - } - - /** - * Rewrite the entire database on disk, with the exception of key pszSkip if - * non-zero - */ - bool Rewrite(const char *pszSkip = nullptr); - - /** - * Back up the entire database to a file. - */ - bool Backup(const std::string &strDest) const; - - /** - * Make sure all changes are flushed to disk. - */ - void Flush(bool shutdown); - - void IncrementUpdateCounter(); - - void ReloadDbEnv(); - - std::atomic nUpdateCounter; - unsigned int nLastSeen; - unsigned int nLastFlushed; - int64_t nLastWalletUpdate; - - /** - * Pointer to shared database environment. - * - * Normally there is only one BerkeleyDatabase object per - * BerkeleyEnvivonment, but in the special, backwards compatible case where - * multiple wallet BDB data files are loaded from the same directory, this - * will point to a shared instance that gets freed when the last data file - * is closed. - */ - std::shared_ptr env; - - /** - * Database pointer. This is initialized lazily and reset during flushes, - * so it can be null. - */ - std::unique_ptr m_db; - -private: - std::string strFile; - - /** - * Return whether this database handle is a dummy for testing. - * Only to be used at a low level, application should ideally not care - * about this. - */ - bool IsDummy() const { return env == nullptr; } -}; - -/** RAII class that provides access to a Berkeley database */ -class BerkeleyBatch { - /** RAII class that automatically cleanses its data on destruction */ - class SafeDbt final { - Dbt m_dbt; - - public: - // construct Dbt with internally-managed data - SafeDbt(); - // construct Dbt with provided data - SafeDbt(void *data, size_t size); - ~SafeDbt(); - - // delegate to Dbt - const void *get_data() const; - u_int32_t get_size() const; - - // conversion operator to access the underlying Dbt - operator Dbt *(); - }; - -protected: - Db *pdb; - std::string strFile; - DbTxn *activeTxn; - bool fReadOnly; - bool fFlushOnClose; - BerkeleyEnvironment *env; - -public: - explicit BerkeleyBatch(BerkeleyDatabase &database, - const char *pszMode = "r+", - bool fFlushOnCloseIn = true); - ~BerkeleyBatch() { Close(); } - - BerkeleyBatch(const BerkeleyBatch &) = delete; - BerkeleyBatch &operator=(const BerkeleyBatch &) = delete; - - void Flush(); - void Close(); - - /* flush the wallet passively (TRY_LOCK) - ideal to be called periodically */ - static bool PeriodicFlush(BerkeleyDatabase &database); - /* verifies the database environment */ - static bool VerifyEnvironment(const fs::path &file_path, - bilingual_str &errorStr); - /* verifies the database file */ - static bool VerifyDatabaseFile(const fs::path &file_path, - bilingual_str &errorStr); - - template bool Read(const K &key, T &value) { - if (!pdb) { - return false; - } - - // Key - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - - // Read - SafeDbt datValue; - int ret = pdb->get(activeTxn, datKey, datValue, 0); - bool success = false; - if (datValue.get_data() != nullptr) { - // Unserialize value - try { - CDataStream ssValue((char *)datValue.get_data(), - (char *)datValue.get_data() + - datValue.get_size(), - SER_DISK, CLIENT_VERSION); - ssValue >> value; - success = true; - } catch (const std::exception &) { - // In this case success remains 'false' - } - } - return ret == 0 && success; - } - - template - bool Write(const K &key, const T &value, bool fOverwrite = true) { - if (!pdb) { - return true; - } - if (fReadOnly) { - assert(!"Write called on database in read-only mode"); - } - - // Key - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - - // Value - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - ssValue.reserve(10000); - ssValue << value; - SafeDbt datValue(ssValue.data(), ssValue.size()); - - // Write - int ret = pdb->put(activeTxn, datKey, datValue, - (fOverwrite ? 0 : DB_NOOVERWRITE)); - return (ret == 0); - } - - template bool Erase(const K &key) { - if (!pdb) { - return false; - } - if (fReadOnly) { - assert(!"Erase called on database in read-only mode"); - } - - // Key - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - - // Erase - int ret = pdb->del(activeTxn, datKey, 0); - return (ret == 0 || ret == DB_NOTFOUND); - } - - template bool Exists(const K &key) { - if (!pdb) { - return false; - } - - // Key - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - ssKey.reserve(1000); - ssKey << key; - SafeDbt datKey(ssKey.data(), ssKey.size()); - - // Exists - int ret = pdb->exists(activeTxn, datKey, 0); - return (ret == 0); - } - - Dbc *GetCursor(); - int ReadAtCursor(Dbc *pcursor, CDataStream &ssKey, CDataStream &ssValue); - bool TxnBegin(); - bool TxnCommit(); - bool TxnAbort(); - - static bool Rewrite(BerkeleyDatabase &database, - const char *pszSkip = nullptr); -}; - #endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index c1bca1e07..48583f6d3 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -1,77 +1,77 @@ // Copyright (c) 2018 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include #include #include #include #include BOOST_FIXTURE_TEST_SUITE(db_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(getwalletenv_file) { std::string test_name = "test_name.dat"; const fs::path datadir = GetDataDir(); fs::path file_path = datadir / test_name; std::ofstream f(file_path.BOOST_FILESYSTEM_C_STR); f.close(); std::string filename; std::shared_ptr env = GetWalletEnv(file_path, filename); BOOST_CHECK(filename == test_name); BOOST_CHECK(env->Directory() == datadir); } BOOST_AUTO_TEST_CASE(getwalletenv_directory) { std::string expected_name = "wallet.dat"; const fs::path datadir = GetDataDir(); std::string filename; std::shared_ptr env = GetWalletEnv(datadir, filename); BOOST_CHECK(filename == expected_name); BOOST_CHECK(env->Directory() == datadir); } BOOST_AUTO_TEST_CASE(getwalletenv_g_dbenvs_multiple) { fs::path datadir = GetDataDir() / "1"; fs::path datadir_2 = GetDataDir() / "2"; std::string filename; std::shared_ptr env_1 = GetWalletEnv(datadir, filename); std::shared_ptr env_2 = GetWalletEnv(datadir, filename); std::shared_ptr env_3 = GetWalletEnv(datadir_2, filename); BOOST_CHECK(env_1 == env_2); BOOST_CHECK(env_2 != env_3); } BOOST_AUTO_TEST_CASE(getwalletenv_g_dbenvs_free_instance) { fs::path datadir = GetDataDir() / "1"; fs::path datadir_2 = GetDataDir() / "2"; std::string filename; std::shared_ptr env_1_a = GetWalletEnv(datadir, filename); std::shared_ptr env_2_a = GetWalletEnv(datadir_2, filename); env_1_a.reset(); std::shared_ptr env_1_b = GetWalletEnv(datadir, filename); std::shared_ptr env_2_b = GetWalletEnv(datadir_2, filename); BOOST_CHECK(env_1_a != env_1_b); BOOST_CHECK(env_2_a == env_2_b); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 8ab2fedcb..d796962ef 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -1,317 +1,317 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Copyright (c) 2017-2020 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_WALLET_WALLETDB_H #define BITCOIN_WALLET_WALLETDB_H #include #include #include