diff --git a/src/banman.cpp b/src/banman.cpp index 9ccfd1482..206fe3fa2 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -1,243 +1,243 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include BanMan::BanMan(fs::path ban_file, const CChainParams &chainparams, CClientUIInterface *client_interface, int64_t default_ban_time) : m_client_interface(client_interface), m_ban_db(std::move(ban_file), chainparams), m_default_ban_time(default_ban_time) { if (m_client_interface) { - m_client_interface->InitMessage(_("Loading banlist...")); + m_client_interface->InitMessage(_("Loading banlist...").translated); } int64_t n_start = GetTimeMillis(); m_is_dirty = false; banmap_t banmap; if (m_ban_db.Read(banmap)) { // thread save setter SetBanned(banmap); // no need to write down, just read data SetBannedSetDirty(false); // sweep out unused entries SweepBanned(); LogPrint(BCLog::NET, "Loaded %d banned node ips/subnets from banlist.dat %dms\n", banmap.size(), GetTimeMillis() - n_start); } else { LogPrintf("Invalid or missing banlist.dat; recreating\n"); // force write SetBannedSetDirty(true); DumpBanlist(); } } BanMan::~BanMan() { DumpBanlist(); } void BanMan::DumpBanlist() { // clean unused entries (if bantime has expired) SweepBanned(); if (!BannedSetIsDirty()) { return; } int64_t n_start = GetTimeMillis(); banmap_t banmap; GetBanned(banmap); if (m_ban_db.Write(banmap)) { SetBannedSetDirty(false); } LogPrint(BCLog::NET, "Flushed %d banned node ips/subnets to banlist.dat %dms\n", banmap.size(), GetTimeMillis() - n_start); } void BanMan::ClearBanned() { { LOCK(m_cs_banned); m_banned.clear(); m_is_dirty = true; } // store banlist to disk DumpBanlist(); if (m_client_interface) { m_client_interface->BannedListChanged(); } } int BanMan::IsBannedLevel(CNetAddr net_addr) { // Returns the most severe level of banning that applies to this address. // 0 - Not banned // 1 - Automatic misbehavior ban // 2 - Any other ban int level = 0; auto current_time = GetTime(); LOCK(m_cs_banned); for (const auto &it : m_banned) { CSubNet sub_net = it.first; CBanEntry ban_entry = it.second; if (current_time < ban_entry.nBanUntil && sub_net.Match(net_addr)) { if (ban_entry.banReason != BanReasonNodeMisbehaving) { return 2; } level = 1; } } return level; } bool BanMan::IsBanned(CNetAddr net_addr) { auto current_time = GetTime(); LOCK(m_cs_banned); for (const auto &it : m_banned) { CSubNet sub_net = it.first; CBanEntry ban_entry = it.second; if (current_time < ban_entry.nBanUntil && sub_net.Match(net_addr)) { return true; } } return false; } bool BanMan::IsBanned(CSubNet sub_net) { auto current_time = GetTime(); LOCK(m_cs_banned); banmap_t::iterator i = m_banned.find(sub_net); if (i != m_banned.end()) { CBanEntry ban_entry = (*i).second; if (current_time < ban_entry.nBanUntil) { return true; } } return false; } void BanMan::Ban(const CNetAddr &net_addr, const BanReason &ban_reason, int64_t ban_time_offset, bool since_unix_epoch) { CSubNet sub_net(net_addr); Ban(sub_net, ban_reason, ban_time_offset, since_unix_epoch); } void BanMan::Ban(const CSubNet &sub_net, const BanReason &ban_reason, int64_t ban_time_offset, bool since_unix_epoch) { CBanEntry ban_entry(GetTime(), ban_reason); int64_t normalized_ban_time_offset = ban_time_offset; bool normalized_since_unix_epoch = since_unix_epoch; if (ban_time_offset <= 0) { normalized_ban_time_offset = m_default_ban_time; normalized_since_unix_epoch = false; } ban_entry.nBanUntil = (normalized_since_unix_epoch ? 0 : GetTime()) + normalized_ban_time_offset; { LOCK(m_cs_banned); if (m_banned[sub_net].nBanUntil < ban_entry.nBanUntil) { m_banned[sub_net] = ban_entry; m_is_dirty = true; } else { return; } } if (m_client_interface) { m_client_interface->BannedListChanged(); } // store banlist to disk immediately if user requested ban if (ban_reason == BanReasonManuallyAdded) { DumpBanlist(); } } bool BanMan::Unban(const CNetAddr &net_addr) { CSubNet sub_net(net_addr); return Unban(sub_net); } bool BanMan::Unban(const CSubNet &sub_net) { { LOCK(m_cs_banned); if (m_banned.erase(sub_net) == 0) { return false; } m_is_dirty = true; } if (m_client_interface) { m_client_interface->BannedListChanged(); } // store banlist to disk immediately DumpBanlist(); return true; } void BanMan::GetBanned(banmap_t &banmap) { LOCK(m_cs_banned); // Sweep the banlist so expired bans are not returned SweepBanned(); // create a thread safe copy banmap = m_banned; } void BanMan::SetBanned(const banmap_t &banmap) { LOCK(m_cs_banned); m_banned = banmap; m_is_dirty = true; } void BanMan::SweepBanned() { int64_t now = GetTime(); bool notify_ui = false; { LOCK(m_cs_banned); banmap_t::iterator it = m_banned.begin(); while (it != m_banned.end()) { CSubNet sub_net = (*it).first; CBanEntry ban_entry = (*it).second; if (now > ban_entry.nBanUntil) { m_banned.erase(it++); m_is_dirty = true; notify_ui = true; LogPrint( BCLog::NET, "%s: Removed banned node ip/subnet from banlist.dat: %s\n", __func__, sub_net.ToString()); } else { ++it; } } } // update UI if (notify_ui && m_client_interface) { m_client_interface->BannedListChanged(); } } bool BanMan::BannedSetIsDirty() { LOCK(m_cs_banned); return m_is_dirty; } void BanMan::SetBannedSetDirty(bool dirty) { // reuse m_banned lock for the m_is_dirty flag LOCK(m_cs_banned); m_is_dirty = dirty; } diff --git a/src/httprpc.cpp b/src/httprpc.cpp index f08d6b853..04f7f2792 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -1,491 +1,492 @@ // Copyright (c) 2015-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // boost::trim #include #include #include #include #include #include #include /** WWW-Authenticate to present with 401 Unauthorized response */ static const char *WWW_AUTH_HEADER_DATA = "Basic realm=\"jsonrpc\""; /** RPC auth failure delay to make brute-forcing expensive */ static const int64_t RPC_AUTH_BRUTE_FORCE_DELAY = 250; /** * Simple one-shot callback timer to be used by the RPC mechanism to e.g. * re-lock the wallet. */ class HTTPRPCTimer : public RPCTimerBase { public: HTTPRPCTimer(struct event_base *eventBase, std::function &func, int64_t millis) : ev(eventBase, false, func) { struct timeval tv; tv.tv_sec = millis / 1000; tv.tv_usec = (millis % 1000) * 1000; ev.trigger(&tv); } private: HTTPEvent ev; }; class HTTPRPCTimerInterface : public RPCTimerInterface { public: explicit HTTPRPCTimerInterface(struct event_base *_base) : base(_base) {} const char *Name() override { return "HTTP"; } RPCTimerBase *NewTimer(std::function &func, int64_t millis) override { return new HTTPRPCTimer(base, func, millis); } private: struct event_base *base; }; /* Pre-base64-encoded authentication token */ static std::string strRPCUserColonPass; /* Pre-base64-encoded authentication token */ static std::string strRPCCORSDomain; /* Stored RPC timer interface (for unregistration) */ static std::unique_ptr httpRPCTimerInterface; /* RPC Auth Whitelist */ static std::map> g_rpc_whitelist; static bool g_rpc_whitelist_default = false; static void JSONErrorReply(HTTPRequest *req, const UniValue &objError, const UniValue &id) { // Send error reply from json-rpc error object. int nStatus = HTTP_INTERNAL_SERVER_ERROR; int code = find_value(objError, "code").get_int(); if (code == RPC_INVALID_REQUEST) { nStatus = HTTP_BAD_REQUEST; } else if (code == RPC_METHOD_NOT_FOUND) { nStatus = HTTP_NOT_FOUND; } std::string strReply = JSONRPCReply(NullUniValue, objError, id); req->WriteHeader("Content-Type", "application/json"); req->WriteReply(nStatus, strReply); } /* * This function checks username and password against -rpcauth entries from * config file. */ static bool multiUserAuthorized(std::string strUserPass) { if (strUserPass.find(':') == std::string::npos) { return false; } std::string strUser = strUserPass.substr(0, strUserPass.find(':')); std::string strPass = strUserPass.substr(strUserPass.find(':') + 1); for (const std::string &strRPCAuth : gArgs.GetArgs("-rpcauth")) { // Search for multi-user login/pass "rpcauth" from config std::vector vFields; boost::split(vFields, strRPCAuth, boost::is_any_of(":$")); if (vFields.size() != 3) { // Incorrect formatting in config file continue; } std::string strName = vFields[0]; if (!TimingResistantEqual(strName, strUser)) { continue; } std::string strSalt = vFields[1]; std::string strHash = vFields[2]; static const unsigned int KEY_SIZE = 32; uint8_t out[KEY_SIZE]; CHMAC_SHA256(reinterpret_cast(strSalt.c_str()), strSalt.size()) .Write(reinterpret_cast(strPass.c_str()), strPass.size()) .Finalize(out); std::vector hexvec(out, out + KEY_SIZE); std::string strHashFromPass = HexStr(hexvec); if (TimingResistantEqual(strHashFromPass, strHash)) { return true; } } return false; } static bool RPCAuthorized(const std::string &strAuth, std::string &strAuthUsernameOut) { // Belt-and-suspenders measure if InitRPCAuthentication was not called. if (strRPCUserColonPass.empty()) { return false; } if (strAuth.substr(0, 6) != "Basic ") { return false; } std::string strUserPass64 = strAuth.substr(6); boost::trim(strUserPass64); std::string strUserPass = DecodeBase64(strUserPass64); if (strUserPass.find(':') != std::string::npos) { strAuthUsernameOut = strUserPass.substr(0, strUserPass.find(':')); } // Check if authorized under single-user field if (TimingResistantEqual(strUserPass, strRPCUserColonPass)) { return true; } return multiUserAuthorized(strUserPass); } static bool checkCORS(HTTPRequest *req) { // https://www.w3.org/TR/cors/#resource-requests // 1. If the Origin header is not present terminate this set of steps. // The request is outside the scope of this specification. std::pair origin = req->GetHeader("origin"); if (!origin.first) { return false; } // 2. If the value of the Origin header is not a case-sensitive match for // any of the values in list of origins do not set any additional headers // and terminate this set of steps. // Note: Always matching is acceptable since the list of origins can be // unbounded. if (origin.second != strRPCCORSDomain) { return false; } if (req->GetRequestMethod() == HTTPRequest::OPTIONS) { // 6.2 Preflight Request // In response to a preflight request the resource indicates which // methods and headers (other than simple methods and simple // headers) it is willing to handle and whether it supports // credentials. // Resources must use the following set of steps to determine which // additional headers to use in the response: // 3. Let method be the value as result of parsing the // Access-Control-Request-Method header. // If there is no Access-Control-Request-Method header or if parsing // failed, do not set any additional headers and terminate this set // of steps. The request is outside the scope of this specification. std::pair method = req->GetHeader("access-control-request-method"); if (!method.first) { return false; } // 4. Let header field-names be the values as result of parsing // the Access-Control-Request-Headers headers. // If there are no Access-Control-Request-Headers headers let header // field-names be the empty list. // If parsing failed do not set any additional headers and terminate // this set of steps. The request is outside the scope of this // specification. std::pair header_field_names = req->GetHeader("access-control-request-headers"); // 5. If method is not a case-sensitive match for any of the // values in list of methods do not set any additional headers // and terminate this set of steps. // Note: Always matching is acceptable since the list of methods // can be unbounded. if (method.second != "POST") { return false; } // 6. If any of the header field-names is not a ASCII case- // insensitive match for any of the values in list of headers do not // set any additional headers and terminate this set of steps. // Note: Always matching is acceptable since the list of headers can // be unbounded. const std::string &list_of_headers = "authorization,content-type"; // 7. If the resource supports credentials add a single // Access-Control-Allow-Origin header, with the value of the Origin // header as value, and add a single // Access-Control-Allow-Credentials header with the case-sensitive // string "true" as value. req->WriteHeader("Access-Control-Allow-Origin", origin.second); req->WriteHeader("Access-Control-Allow-Credentials", "true"); // 8. Optionally add a single Access-Control-Max-Age header with as // value the amount of seconds the user agent is allowed to cache // the result of the request. // 9. If method is a simple method this step may be skipped. // Add one or more Access-Control-Allow-Methods headers consisting // of (a subset of) the list of methods. // If a method is a simple method it does not need to be listed, but // this is not prohibited. // Note: Since the list of methods can be unbounded, simply // returning the method indicated by // Access-Control-Request-Method (if supported) can be enough. req->WriteHeader("Access-Control-Allow-Methods", method.second); // 10. If each of the header field-names is a simple header and none // is Content-Type, this step may be skipped. // Add one or more Access-Control-Allow-Headers headers consisting // of (a subset of) the list of headers. req->WriteHeader("Access-Control-Allow-Headers", header_field_names.first ? header_field_names.second : list_of_headers); req->WriteReply(HTTP_OK); return true; } // 6.1 Simple Cross-Origin Request, Actual Request, and Redirects // In response to a simple cross-origin request or actual request the // resource indicates whether or not to share the response. // If the resource has been relocated, it indicates whether to share its // new URL. // Resources must use the following set of steps to determine which // additional headers to use in the response: // 3. If the resource supports credentials add a single // Access-Control-Allow-Origin header, with the value of the Origin // header as value, and add a single Access-Control-Allow-Credentials // header with the case-sensitive string "true" as value. req->WriteHeader("Access-Control-Allow-Origin", origin.second); req->WriteHeader("Access-Control-Allow-Credentials", "true"); // 4. If the list of exposed headers is not empty add one or more // Access-Control-Expose-Headers headers, with as values the header // field names given in the list of exposed headers. req->WriteHeader("Access-Control-Expose-Headers", "WWW-Authenticate"); return false; } bool HTTPRPCRequestProcessor::ProcessHTTPRequest(HTTPRequest *req) { // First, check and/or set CORS headers if (checkCORS(req)) { return true; } // JSONRPC handles only POST if (req->GetRequestMethod() != HTTPRequest::POST) { req->WriteReply(HTTP_BAD_METHOD, "JSONRPC server handles only POST requests"); return false; } // Check authorization std::pair authHeader = req->GetHeader("authorization"); if (!authHeader.first) { req->WriteHeader("WWW-Authenticate", WWW_AUTH_HEADER_DATA); req->WriteReply(HTTP_UNAUTHORIZED); return false; } JSONRPCRequest jreq; if (!RPCAuthorized(authHeader.second, jreq.authUser)) { LogPrintf("ThreadRPCServer incorrect password attempt from %s\n", req->GetPeer().ToString()); /** * Deter brute-forcing. * If this results in a DoS the user really shouldn't have their RPC * port exposed. */ MilliSleep(RPC_AUTH_BRUTE_FORCE_DELAY); req->WriteHeader("WWW-Authenticate", WWW_AUTH_HEADER_DATA); req->WriteReply(HTTP_UNAUTHORIZED); return false; } try { // Parse request UniValue valRequest; if (!valRequest.read(req->ReadBody())) { throw JSONRPCError(RPC_PARSE_ERROR, "Parse error"); } // Set the URI jreq.URI = req->GetURI(); std::string strReply; bool user_has_whitelist = g_rpc_whitelist.count(jreq.authUser); if (!user_has_whitelist && g_rpc_whitelist_default) { LogPrintf("RPC User %s not allowed to call any methods\n", jreq.authUser); req->WriteReply(HTTP_FORBIDDEN); return false; // singleton request } else if (valRequest.isObject()) { jreq.parse(valRequest); if (user_has_whitelist && !g_rpc_whitelist[jreq.authUser].count(jreq.strMethod)) { LogPrintf("RPC User %s not allowed to call method %s\n", jreq.authUser, jreq.strMethod); req->WriteReply(HTTP_FORBIDDEN); return false; } UniValue result = rpcServer.ExecuteCommand(config, jreq); // Send reply strReply = JSONRPCReply(result, NullUniValue, jreq.id); // array of requests } else if (valRequest.isArray()) { if (user_has_whitelist) { for (unsigned int reqIdx = 0; reqIdx < valRequest.size(); reqIdx++) { if (!valRequest[reqIdx].isObject()) { throw JSONRPCError(RPC_INVALID_REQUEST, "Invalid Request object"); } else { const UniValue &request = valRequest[reqIdx].get_obj(); // Parse method std::string strMethod = find_value(request, "method").get_str(); if (!g_rpc_whitelist[jreq.authUser].count(strMethod)) { LogPrintf( "RPC User %s not allowed to call method %s\n", jreq.authUser, strMethod); req->WriteReply(HTTP_FORBIDDEN); return false; } } } } strReply = JSONRPCExecBatch(config, rpcServer, jreq, valRequest.get_array()); } else { throw JSONRPCError(RPC_PARSE_ERROR, "Top-level object parse error"); } req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strReply); } catch (const UniValue &objError) { JSONErrorReply(req, objError, jreq.id); return false; } catch (const std::exception &e) { JSONErrorReply(req, JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq.id); return false; } return true; } static bool InitRPCAuthentication() { if (gArgs.GetArg("-rpcpassword", "") == "") { LogPrintf("No rpcpassword set - using random cookie authentication.\n"); if (!GenerateAuthCookie(&strRPCUserColonPass)) { // Same message as AbortNode. uiInterface.ThreadSafeMessageBox( _("Error: A fatal internal error occurred, see debug.log for " - "details"), + "details") + .translated, "", CClientUIInterface::MSG_ERROR); return false; } } else { LogPrintf("Config options rpcuser and rpcpassword will soon be " "deprecated. Locally-run instances may remove rpcuser to use " "cookie-based auth, or may be replaced with rpcauth. Please " "see share/rpcauth for rpcauth auth generation.\n"); strRPCUserColonPass = gArgs.GetArg("-rpcuser", "") + ":" + gArgs.GetArg("-rpcpassword", ""); } strRPCCORSDomain = gArgs.GetArg("-rpccorsdomain", ""); if (gArgs.GetArg("-rpcauth", "") != "") { LogPrintf("Using rpcauth authentication.\n"); } g_rpc_whitelist_default = gArgs.GetBoolArg("-rpcwhitelistdefault", gArgs.IsArgSet("-rpcwhitelist")); for (const std::string &strRPCWhitelist : gArgs.GetArgs("-rpcwhitelist")) { auto pos = strRPCWhitelist.find(':'); std::string strUser = strRPCWhitelist.substr(0, pos); bool intersect = g_rpc_whitelist.count(strUser); std::set &whitelist = g_rpc_whitelist[strUser]; if (pos != std::string::npos) { std::string strWhitelist = strRPCWhitelist.substr(pos + 1); std::set new_whitelist; boost::split(new_whitelist, strWhitelist, boost::is_any_of(", ")); if (intersect) { std::set tmp_whitelist; std::set_intersection( new_whitelist.begin(), new_whitelist.end(), whitelist.begin(), whitelist.end(), std::inserter(tmp_whitelist, tmp_whitelist.end())); new_whitelist = std::move(tmp_whitelist); } whitelist = std::move(new_whitelist); } } return true; } bool StartHTTPRPC(HTTPRPCRequestProcessor &httpRPCRequestProcessor) { LogPrint(BCLog::RPC, "Starting HTTP RPC server\n"); if (!InitRPCAuthentication()) { return false; } const std::function &rpcFunction = std::bind(&HTTPRPCRequestProcessor::DelegateHTTPRequest, &httpRPCRequestProcessor, std::placeholders::_2); RegisterHTTPHandler("/", true, rpcFunction); if (g_wallet_init_interface.HasWalletSupport()) { RegisterHTTPHandler("/wallet/", false, rpcFunction); } struct event_base *eventBase = EventBase(); assert(eventBase); httpRPCTimerInterface = std::make_unique(eventBase); RPCSetTimerInterface(httpRPCTimerInterface.get()); return true; } void InterruptHTTPRPC() { LogPrint(BCLog::RPC, "Interrupting HTTP RPC server\n"); } void StopHTTPRPC() { LogPrint(BCLog::RPC, "Stopping HTTP RPC server\n"); UnregisterHTTPHandler("/", true); if (g_wallet_init_interface.HasWalletSupport()) { UnregisterHTTPHandler("/wallet/", false); } if (httpRPCTimerInterface) { RPCUnsetTimerInterface(httpRPCTimerInterface.get()); httpRPCTimerInterface.reset(); } } diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 97d1bb5ab..fc0b012a2 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -1,296 +1,297 @@ // Copyright (c) 2017-2018 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include constexpr char DB_BEST_BLOCK = 'B'; constexpr char DB_TXINDEX = 't'; constexpr char DB_TXINDEX_BLOCK = 'T'; std::unique_ptr g_txindex; struct CDiskTxPos : public FlatFilePos { unsigned int nTxOffset; // after header ADD_SERIALIZE_METHODS; template inline void SerializationOp(Stream &s, Operation ser_action) { READWRITEAS(FlatFilePos, *this); READWRITE(VARINT(nTxOffset)); } CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {} CDiskTxPos() { SetNull(); } void SetNull() { FlatFilePos::SetNull(); nTxOffset = 0; } }; /** * Access to the txindex database (indexes/txindex/) * * The database stores a block locator of the chain the database is synced to * so that the TxIndex can efficiently determine the point it last stopped at. * A locator is used instead of a simple hash of the chain tip because blocks * and block index entries may not be flushed to disk until after this database * is updated. */ class TxIndex::DB : public BaseIndex::DB { public: explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); /// Read the disk location of the transaction data with the given ID. /// Returns false if the transaction ID is not indexed. bool ReadTxPos(const TxId &txid, CDiskTxPos &pos) const; /// Write a batch of transaction positions to the DB. bool WriteTxs(const std::vector> &v_pos); /// Migrate txindex data from the block tree DB, where it may be for older /// nodes that have not been upgraded yet to the new database. bool MigrateData(CBlockTreeDB &block_tree_db, const CBlockLocator &best_locator); }; TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) : BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe) {} bool TxIndex::DB::ReadTxPos(const TxId &txid, CDiskTxPos &pos) const { return Read(std::make_pair(DB_TXINDEX, txid), pos); } bool TxIndex::DB::WriteTxs( const std::vector> &v_pos) { CDBBatch batch(*this); for (const auto &tuple : v_pos) { batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second); } return WriteBatch(batch); } /* * Safely persist a transfer of data from the old txindex database to the new * one, and compact the range of keys updated. This is used internally by * MigrateData. */ static void WriteTxIndexMigrationBatches(CDBWrapper &newdb, CDBWrapper &olddb, CDBBatch &batch_newdb, CDBBatch &batch_olddb, const std::pair &begin_key, const std::pair &end_key) { // Sync new DB changes to disk before deleting from old DB. newdb.WriteBatch(batch_newdb, /*fSync=*/true); olddb.WriteBatch(batch_olddb); olddb.CompactRange(begin_key, end_key); batch_newdb.Clear(); batch_olddb.Clear(); } bool TxIndex::DB::MigrateData(CBlockTreeDB &block_tree_db, const CBlockLocator &best_locator) { // The prior implementation of txindex was always in sync with block index // and presence was indicated with a boolean DB flag. If the flag is set, // this means the txindex from a previous version is valid and in sync with // the chain tip. The first step of the migration is to unset the flag and // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the // index entries are copied over in batches to the new database. Finally, // DB_TXINDEX_BLOCK is erased from the old database and the block hash is // written to the new database. // // Unsetting the boolean flag ensures that if the node is downgraded to a // previous version, it will not see a corrupted, partially migrated index // -- it will see that the txindex is disabled. When the node is upgraded // again, the migration will pick up where it left off and sync to the block // with hash DB_TXINDEX_BLOCK. bool f_legacy_flag = false; block_tree_db.ReadFlag("txindex", f_legacy_flag); if (f_legacy_flag) { if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) { return error("%s: cannot write block indicator", __func__); } if (!block_tree_db.WriteFlag("txindex", false)) { return error("%s: cannot write block index db flag", __func__); } } CBlockLocator locator; if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) { return true; } int64_t count = 0; - uiInterface.InitMessage(_("Upgrading txindex database")); + uiInterface.InitMessage(_("Upgrading txindex database").translated); LogPrintf("Upgrading txindex database... [0%%]\n"); - uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true); + uiInterface.ShowProgress(_("Upgrading txindex database").translated, 0, + true); int report_done = 0; const size_t batch_size = 1 << 24; // 16 MiB CDBBatch batch_newdb(*this); CDBBatch batch_olddb(block_tree_db); std::pair key; std::pair begin_key{DB_TXINDEX, uint256()}; std::pair prev_key = begin_key; bool interrupted = false; std::unique_ptr cursor(block_tree_db.NewIterator()); for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) { boost::this_thread::interruption_point(); if (ShutdownRequested()) { interrupted = true; break; } if (!cursor->GetKey(key)) { return error("%s: cannot get key from valid cursor", __func__); } if (key.first != DB_TXINDEX) { break; } // Log progress every 10%. if (++count % 256 == 0) { // Since txids are uniformly random and traversed in increasing // order, the high 16 bits of the ID can be used to estimate the // current progress. const uint256 &txid = key.second; uint32_t high_nibble = (static_cast(*(txid.begin() + 0)) << 8) + (static_cast(*(txid.begin() + 1)) << 0); int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5); - uiInterface.ShowProgress(_("Upgrading txindex database"), + uiInterface.ShowProgress(_("Upgrading txindex database").translated, percentage_done, true); if (report_done < percentage_done / 10) { LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done); report_done = percentage_done / 10; } } CDiskTxPos value; if (!cursor->GetValue(value)) { return error("%s: cannot parse txindex record", __func__); } batch_newdb.Write(key, value); batch_olddb.Erase(key); if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) { // NOTE: it's OK to delete the key pointed at by the current DB // cursor while iterating because LevelDB iterators are guaranteed // to provide a consistent view of the underlying data, like a // lightweight snapshot. WriteTxIndexMigrationBatches(*this, block_tree_db, batch_newdb, batch_olddb, prev_key, key); prev_key = key; } } // If these final DB batches complete the migration, write the best block // hash marker to the new database and delete from the old one. This signals // that the former is fully caught up to that point in the blockchain and // that all txindex entries have been removed from the latter. if (!interrupted) { batch_olddb.Erase(DB_TXINDEX_BLOCK); batch_newdb.Write(DB_BEST_BLOCK, locator); } WriteTxIndexMigrationBatches(*this, block_tree_db, batch_newdb, batch_olddb, begin_key, key); if (interrupted) { LogPrintf("[CANCELLED].\n"); return false; } uiInterface.ShowProgress("", 100, false); LogPrintf("[DONE].\n"); return true; } TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe) : m_db(std::make_unique(n_cache_size, f_memory, f_wipe)) {} TxIndex::~TxIndex() {} bool TxIndex::Init() { LOCK(cs_main); // Attempt to migrate txindex from the old database to the new one. Even if // chain_tip is null, the node could be reindexing and we still want to // delete txindex records in the old database. if (!m_db->MigrateData(*pblocktree, ::ChainActive().GetLocator())) { return false; } return BaseIndex::Init(); } bool TxIndex::WriteBlock(const CBlock &block, const CBlockIndex *pindex) { // Exclude genesis block transaction because outputs are not spendable. if (pindex->nHeight == 0) { return true; } CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); std::vector> vPos; vPos.reserve(block.vtx.size()); for (const auto &tx : block.vtx) { vPos.emplace_back(tx->GetId(), pos); pos.nTxOffset += ::GetSerializeSize(*tx, CLIENT_VERSION); } return m_db->WriteTxs(vPos); } BaseIndex::DB &TxIndex::GetDB() const { return *m_db; } bool TxIndex::FindTx(const TxId &txid, BlockHash &block_hash, CTransactionRef &tx) const { CDiskTxPos postx; if (!m_db->ReadTxPos(txid, postx)) { return false; } CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); if (file.IsNull()) { return error("%s: OpenBlockFile failed", __func__); } CBlockHeader header; try { file >> header; if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) { return error("%s: fseek(...) failed", __func__); } file >> tx; } catch (const std::exception &e) { return error("%s: Deserialize or I/O error - %s", __func__, e.what()); } if (tx->GetId() != txid) { return error("%s: txid mismatch", __func__); } block_hash = header.GetHash(); return true; } diff --git a/src/init.cpp b/src/init.cpp index 1ddb5319a..f8f19b3fa 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1,2652 +1,2689 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2018 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include