diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 6e2a5d113..2a667f98d 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -1,470 +1,483 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) #include "config/bitcoin-config.h" #endif #include "chainparamsbase.h" #include "clientversion.h" #include "fs.h" #include "rpc/client.h" #include "rpc/protocol.h" #include "support/events.h" #include "util.h" #include "utilstrencodings.h" #include #include #include #include #include static const char DEFAULT_RPCCONNECT[] = "127.0.0.1"; static const int DEFAULT_HTTP_CLIENT_TIMEOUT = 900; static const bool DEFAULT_NAMED = false; static const int CONTINUE_EXECUTION = -1; std::string HelpMessageCli() { std::string strUsage; strUsage += HelpMessageGroup(_("Options:")); strUsage += HelpMessageOpt("-?", _("This help message")); strUsage += HelpMessageOpt( "-conf=", strprintf(_("Specify configuration file (default: %s)"), BITCOIN_CONF_FILENAME)); strUsage += HelpMessageOpt("-datadir=", _("Specify data directory")); AppendParamsHelpMessages(strUsage); strUsage += HelpMessageOpt( "-named", strprintf(_("Pass named instead of positional arguments (default: %s)"), DEFAULT_NAMED)); strUsage += HelpMessageOpt( "-rpcconnect=", strprintf(_("Send commands to node running on (default: %s)"), DEFAULT_RPCCONNECT)); strUsage += HelpMessageOpt( "-rpcport=", strprintf( _("Connect to JSON-RPC on (default: %u or testnet: %u)"), BaseParams(CBaseChainParams::MAIN).RPCPort(), BaseParams(CBaseChainParams::TESTNET).RPCPort())); strUsage += HelpMessageOpt("-rpcwait", _("Wait for RPC server to start")); strUsage += HelpMessageOpt("-rpcuser=", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=", _("Password for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcclienttimeout=", strprintf(_("Timeout in seconds during HTTP requests, " "or 0 for no timeout. (default: %d)"), DEFAULT_HTTP_CLIENT_TIMEOUT)); + + strUsage += HelpMessageOpt( + "-stdinrpcpass", + strprintf(_("Read RPC password from standard input as a single line. " + "When combined with -stdin, the first line from standard " + "input is used for the RPC password."))); strUsage += HelpMessageOpt( "-stdin", _("Read extra arguments from standard input, one per line " "until EOF/Ctrl-D (recommended for sensitive information " "such as passphrases)")); strUsage += HelpMessageOpt( "-rpcwallet=", _("Send RPC for non-default wallet on RPC server (argument is wallet " "filename in bitcoind directory, required if bitcoind/-Qt runs with " "multiple wallets)")); return strUsage; } ////////////////////////////////////////////////////////////////////////////// // // Start // // // Exception thrown on connection error. This error is used to determine when // to wait if -rpcwait is given. // class CConnectionFailed : public std::runtime_error { public: explicit inline CConnectionFailed(const std::string &msg) : std::runtime_error(msg) {} }; // // This function returns either one of EXIT_ codes when it's expected to stop // the process or CONTINUE_EXECUTION when it's expected to continue further. // static int AppInitRPC(int argc, char *argv[]) { // // Parameters // gArgs.ParseParameters(argc, argv); if (argc < 2 || gArgs.IsArgSet("-?") || gArgs.IsArgSet("-h") || gArgs.IsArgSet("-help") || gArgs.IsArgSet("-version")) { std::string strUsage = strprintf(_("%s RPC client version"), _(PACKAGE_NAME)) + " " + FormatFullVersion() + "\n"; if (!gArgs.IsArgSet("-version")) { strUsage += "\n" + _("Usage:") + "\n" + " bitcoin-cli [options] [params] " + strprintf(_("Send command to %s"), _(PACKAGE_NAME)) + "\n" + " bitcoin-cli [options] -named [name=value] ... " + strprintf(_("Send command to %s (with named arguments)"), _(PACKAGE_NAME)) + "\n" + " bitcoin-cli [options] help " + _("List commands") + "\n" + " bitcoin-cli [options] help " + _("Get help for a command") + "\n"; strUsage += "\n" + HelpMessageCli(); } fprintf(stdout, "%s", strUsage.c_str()); if (argc < 2) { fprintf(stderr, "Error: too few parameters\n"); return EXIT_FAILURE; } return EXIT_SUCCESS; } if (!fs::is_directory(GetDataDir(false))) { fprintf(stderr, "Error: Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "").c_str()); return EXIT_FAILURE; } try { gArgs.ReadConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME)); } catch (const std::exception &e) { fprintf(stderr, "Error reading configuration file: %s\n", e.what()); return EXIT_FAILURE; } // Check for -testnet or -regtest parameter (BaseParams() calls are only // valid after this clause) try { SelectBaseParams(ChainNameFromCommandLine()); } catch (const std::exception &e) { fprintf(stderr, "Error: %s\n", e.what()); return EXIT_FAILURE; } if (gArgs.GetBoolArg("-rpcssl", false)) { fprintf(stderr, "Error: SSL mode for RPC (-rpcssl) is no longer supported.\n"); return EXIT_FAILURE; } return CONTINUE_EXECUTION; } /** Reply structure for request_done to fill in */ struct HTTPReply { HTTPReply() : status(0), error(-1) {} int status; int error; std::string body; }; const char *http_errorstring(int code) { switch (code) { #if LIBEVENT_VERSION_NUMBER >= 0x02010300 case EVREQ_HTTP_TIMEOUT: return "timeout reached"; case EVREQ_HTTP_EOF: return "EOF reached"; case EVREQ_HTTP_INVALID_HEADER: return "error while reading header, or invalid header"; case EVREQ_HTTP_BUFFER_ERROR: return "error encountered while reading or writing"; case EVREQ_HTTP_REQUEST_CANCEL: return "request was canceled"; case EVREQ_HTTP_DATA_TOO_LONG: return "response body is larger than allowed"; #endif default: return "unknown"; } } static void http_request_done(struct evhttp_request *req, void *ctx) { HTTPReply *reply = static_cast(ctx); if (req == nullptr) { /** * If req is nullptr, it means an error occurred while connecting: the * error code will have been passed to http_error_cb. */ reply->status = 0; return; } reply->status = evhttp_request_get_response_code(req); struct evbuffer *buf = evhttp_request_get_input_buffer(req); if (buf) { size_t size = evbuffer_get_length(buf); const char *data = (const char *)evbuffer_pullup(buf, size); if (data) reply->body = std::string(data, size); evbuffer_drain(buf, size); } } #if LIBEVENT_VERSION_NUMBER >= 0x02010300 static void http_error_cb(enum evhttp_request_error err, void *ctx) { HTTPReply *reply = static_cast(ctx); reply->error = err; } #endif -UniValue CallRPC(const std::string &strMethod, const UniValue ¶ms) { +static UniValue CallRPC(const std::string &strMethod, const UniValue ¶ms) { std::string host; // In preference order, we choose the following for the port: // 1. -rpcport // 2. port in -rpcconnect (ie following : in ipv4 or ]: in ipv6) // 3. default port for chain int port = BaseParams().RPCPort(); SplitHostPort(gArgs.GetArg("-rpcconnect", DEFAULT_RPCCONNECT), port, host); port = gArgs.GetArg("-rpcport", port); // Obtain event base raii_event_base base = obtain_event_base(); // Synchronously look up hostname raii_evhttp_connection evcon = obtain_evhttp_connection_base(base.get(), host, port); evhttp_connection_set_timeout( evcon.get(), gArgs.GetArg("-rpcclienttimeout", DEFAULT_HTTP_CLIENT_TIMEOUT)); HTTPReply response; raii_evhttp_request req = obtain_evhttp_request(http_request_done, (void *)&response); if (req == nullptr) throw std::runtime_error("create http request failed"); #if LIBEVENT_VERSION_NUMBER >= 0x02010300 evhttp_request_set_error_cb(req.get(), http_error_cb); #endif // Get credentials std::string strRPCUserColonPass; if (gArgs.GetArg("-rpcpassword", "") == "") { // Try fall back to cookie-based authentication if no password is // provided if (!GetAuthCookie(&strRPCUserColonPass)) { throw std::runtime_error(strprintf( _("Could not locate RPC credentials. No authentication cookie " - "could be found, and no rpcpassword is set in the " - "configuration file (%s)"), + "could be found, and RPC password is not set. See " + "-rpcpassword and -stdinrpcpass. Configuration file: (%s)"), GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME)) .string() .c_str())); } } else { strRPCUserColonPass = gArgs.GetArg("-rpcuser", "") + ":" + gArgs.GetArg("-rpcpassword", ""); } struct evkeyvalq *output_headers = evhttp_request_get_output_headers(req.get()); assert(output_headers); evhttp_add_header(output_headers, "Host", host.c_str()); evhttp_add_header(output_headers, "Connection", "close"); evhttp_add_header( output_headers, "Authorization", (std::string("Basic ") + EncodeBase64(strRPCUserColonPass)).c_str()); // Attach request data std::string strRequest = JSONRPCRequestObj(strMethod, params, 1).write() + "\n"; struct evbuffer *output_buffer = evhttp_request_get_output_buffer(req.get()); assert(output_buffer); evbuffer_add(output_buffer, strRequest.data(), strRequest.size()); // check if we should use a special wallet endpoint std::string endpoint = "/"; std::string walletName = gArgs.GetArg("-rpcwallet", ""); if (!walletName.empty()) { char *encodedURI = evhttp_uriencode(walletName.c_str(), walletName.size(), false); if (encodedURI) { endpoint = "/wallet/" + std::string(encodedURI); free(encodedURI); } else { throw CConnectionFailed("uri-encode failed"); } } int r = evhttp_make_request(evcon.get(), req.get(), EVHTTP_REQ_POST, endpoint.c_str()); // ownership moved to evcon in above call req.release(); if (r != 0) { throw CConnectionFailed("send http request failed"); } event_base_dispatch(base.get()); if (response.status == 0) { throw CConnectionFailed(strprintf( "couldn't connect to server: %s (code %d)\n(make sure server is " "running and you are connecting to the correct RPC port)", http_errorstring(response.error), response.error)); } else if (response.status == HTTP_UNAUTHORIZED) { throw std::runtime_error( "incorrect rpcuser or rpcpassword (authorization failed)"); } else if (response.status >= 400 && response.status != HTTP_BAD_REQUEST && response.status != HTTP_NOT_FOUND && response.status != HTTP_INTERNAL_SERVER_ERROR) { throw std::runtime_error( strprintf("server returned HTTP error %d", response.status)); } else if (response.body.empty()) { throw std::runtime_error("no response from server"); } // Parse reply UniValue valReply(UniValue::VSTR); if (!valReply.read(response.body)) { throw std::runtime_error("couldn't parse reply from server"); } const UniValue &reply = valReply.get_obj(); if (reply.empty()) { throw std::runtime_error( "expected reply to have result, error and id properties"); } return reply; } int CommandLineRPC(int argc, char *argv[]) { std::string strPrint; int nRet = 0; try { // Skip switches while (argc > 1 && IsSwitchChar(argv[1][0])) { argc--; argv++; } + std::string rpcPass; + if (gArgs.GetBoolArg("-stdinrpcpass", false)) { + if (!std::getline(std::cin, rpcPass)) + throw std::runtime_error("-stdinrpcpass specified but failed " + "to read from standard input"); + gArgs.ForceSetArg("-rpcpassword", rpcPass); + } std::vector args = std::vector(&argv[1], &argv[argc]); if (gArgs.GetBoolArg("-stdin", false)) { // Read one arg per line from stdin and append std::string line; while (std::getline(std::cin, line)) { args.push_back(line); } } if (args.size() < 1) { throw std::runtime_error( "too few parameters (need at least command)"); } std::string strMethod = args[0]; // Remove trailing method name from arguments vector args.erase(args.begin()); UniValue params; if (gArgs.GetBoolArg("-named", DEFAULT_NAMED)) { params = RPCConvertNamedValues(strMethod, args); } else { params = RPCConvertValues(strMethod, args); } // Execute and handle connection failures with -rpcwait const bool fWait = gArgs.GetBoolArg("-rpcwait", false); do { try { const UniValue reply = CallRPC(strMethod, params); // Parse reply const UniValue &result = find_value(reply, "result"); const UniValue &error = find_value(reply, "error"); if (!error.isNull()) { // Error int code = error["code"].get_int(); if (fWait && code == RPC_IN_WARMUP) throw CConnectionFailed("server in warmup"); strPrint = "error: " + error.write(); nRet = abs(code); if (error.isObject()) { UniValue errCode = find_value(error, "code"); UniValue errMsg = find_value(error, "message"); strPrint = errCode.isNull() ? "" : "error code: " + errCode.getValStr() + "\n"; if (errMsg.isStr()) { strPrint += "error message:\n" + errMsg.get_str(); } if (errCode.isNum() && errCode.get_int() == RPC_WALLET_NOT_SPECIFIED) { strPrint += "\nTry adding " "\"-rpcwallet=\" option to " "bitcoin-cli command line."; } } } else { // Result if (result.isNull()) { strPrint = ""; } else if (result.isStr()) { strPrint = result.get_str(); } else { strPrint = result.write(2); } } // Connection succeeded, no need to retry. break; } catch (const CConnectionFailed &) { if (fWait) { MilliSleep(1000); } else { throw; } } } while (fWait); } catch (const boost::thread_interrupted &) { throw; } catch (const std::exception &e) { strPrint = std::string("error: ") + e.what(); nRet = EXIT_FAILURE; } catch (...) { PrintExceptionContinue(nullptr, "CommandLineRPC()"); throw; } if (strPrint != "") { fprintf((nRet == 0 ? stdout : stderr), "%s\n", strPrint.c_str()); } return nRet; } int main(int argc, char *argv[]) { SetupEnvironment(); if (!SetupNetworking()) { fprintf(stderr, "Error: Initializing networking failed\n"); return EXIT_FAILURE; } try { int ret = AppInitRPC(argc, argv); if (ret != CONTINUE_EXECUTION) { return ret; } } catch (const std::exception &e) { PrintExceptionContinue(&e, "AppInitRPC()"); return EXIT_FAILURE; } catch (...) { PrintExceptionContinue(nullptr, "AppInitRPC()"); return EXIT_FAILURE; } int ret = EXIT_FAILURE; try { ret = CommandLineRPC(argc, argv); } catch (const std::exception &e) { PrintExceptionContinue(&e, "CommandLineRPC()"); } catch (...) { PrintExceptionContinue(nullptr, "CommandLineRPC()"); } return ret; } diff --git a/test/functional/bitcoin_cli.py b/test/functional/bitcoin_cli.py index bf31a8860..3e00e4482 100755 --- a/test/functional/bitcoin_cli.py +++ b/test/functional/bitcoin_cli.py @@ -1,29 +1,48 @@ #!/usr/bin/env python3 # Copyright (c) 2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoin-cli""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal +from test_framework.util import assert_equal, assert_raises_process_error, get_auth_cookie class TestBitcoinCli(BitcoinTestFramework): def __init__(self): super().__init__() self.setup_clean_chain = True self.num_nodes = 1 def run_test(self): """Main test logic""" self.log.info( - "Compare responses from getinfo RPC and `bitcoin-cli getinfo`") - cli_get_info = self.nodes[0].cli.getinfo() - rpc_get_info = self.nodes[0].getinfo() + "Compare responses from gewalletinfo RPC and `bitcoin-cli getwalletinfo`") + cli_response = self.nodes[0].cli.getwalletinfo() + rpc_response = self.nodes[0].getwalletinfo() + assert_equal(cli_response, rpc_response) - assert_equal(cli_get_info, rpc_get_info) + self.log.info( + "Compare responses from getblockchaininfo RPC and `bitcoin-cli getblockchaininfo`") + cli_response = self.nodes[0].cli.getblockchaininfo() + rpc_response = self.nodes[0].getblockchaininfo() + assert_equal(cli_response, rpc_response) + + user, password = get_auth_cookie(self.nodes[0].datadir) + + self.log.info("Test -stdinrpcpass option") + assert_equal(0, self.nodes[0].cli( + '-rpcuser=%s' % user, '-stdinrpcpass', input=password).getblockcount()) + assert_raises_process_error(1, "incorrect rpcuser or rpcpassword", self.nodes[0].cli( + '-rpcuser=%s' % user, '-stdinrpcpass', input="foo").echo) + + self.log.info("Test -stdin and -stdinrpcpass") + assert_equal(["foo", "bar"], self.nodes[0].cli('-rpcuser=%s' % user, + '-stdin', '-stdinrpcpass', input=password + "\nfoo\nbar").echo()) + assert_raises_process_error(1, "incorrect rpcuser or rpcpassword", self.nodes[0].cli( + '-rpcuser=%s' % user, '-stdin', '-stdinrpcpass', input="foo").echo) if __name__ == '__main__': TestBitcoinCli().main() diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index c5ecc89a8..110d2f9d2 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -1,176 +1,192 @@ #!/usr/bin/env python3 # Copyright (c) 2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Class for bitcoind node under test""" import decimal import errno import http.client import json import logging import os import subprocess import time from .util import ( assert_equal, get_rpc_proxy, rpc_url, ) from .authproxy import JSONRPCException class TestNode(): """A class for representing a bitcoind node under test. This class contains: - state about the node (whether it's running, etc) - a Python subprocess.Popen object representing the running process - an RPC connection to the node To make things easier for the test writer, a bit of magic is happening under the covers. Any unrecognised messages will be dispatched to the RPC connection.""" def __init__(self, i, dirname, extra_args, rpchost, timewait, binary, stderr, mocktime, coverage_dir): self.index = i self.datadir = os.path.join(dirname, "node" + str(i)) self.rpchost = rpchost if timewait: self.rpc_timeout = timewait else: # Wait for up to 60 seconds for the RPC server to respond self.rpc_timeout = 60 if binary is None: self.binary = os.getenv("BITCOIND", "bitcoind") else: self.binary = binary self.stderr = stderr self.coverage_dir = coverage_dir # Most callers will just need to add extra args to the standard list below. For those callers that need more flexibity, they can just set the args property directly. self.extra_args = extra_args self.args = [self.binary, "-datadir=" + self.datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", "-mocktime=" + str(mocktime), "-uacomment=testnode%d" % i] self.cli = TestNodeCLI( os.getenv("BITCOINCLI", "bitcoin-cli"), self.datadir) self.running = False self.process = None self.rpc_connected = False self.rpc = None self.url = None self.log = logging.getLogger('TestFramework.node%d' % i) def __getattr__(self, *args, **kwargs): """Dispatches any unrecognised messages to the RPC connection.""" assert self.rpc_connected and self.rpc is not None, "Error: no RPC connection" return self.rpc.__getattr__(*args, **kwargs) def start(self): """Start the node.""" self.process = subprocess.Popen( self.args + self.extra_args, stderr=self.stderr) self.running = True self.log.debug("bitcoind started, waiting for RPC to come up") def wait_for_rpc_connection(self): """Sets up an RPC connection to the bitcoind process. Returns False if unable to connect.""" # Poll at a rate of four times per second poll_per_s = 4 for _ in range(poll_per_s * self.rpc_timeout): assert self.process.poll( ) is None, "bitcoind exited with status %i during initialization" % self.process.returncode try: self.rpc = get_rpc_proxy(rpc_url( self.datadir, self.index, self.rpchost), self.index, coveragedir=self.coverage_dir) self.rpc.getblockcount() # If the call to getblockcount() succeeds then the RPC connection is up self.rpc_connected = True self.url = self.rpc.url self.log.debug("RPC successfully started") return except IOError as e: if e.errno != errno.ECONNREFUSED: # Port not yet open? raise # unknown IO error except JSONRPCException as e: # Initialization phase if e.error['code'] != -28: # RPC in warmup? raise # unknown JSON RPC exception except ValueError as e: # cookie file not found and no rpcuser or rpcassword. bitcoind still starting if "No RPC credentials" not in str(e): raise time.sleep(1.0 / poll_per_s) raise AssertionError("Unable to connect to bitcoind") def get_wallet_rpc(self, wallet_name): assert self.rpc_connected assert self.rpc wallet_path = "wallet/%s" % wallet_name return self.rpc / wallet_path def stop_node(self): """Stop the node.""" if not self.running: return self.log.debug("Stopping node") try: self.stop() except http.client.CannotSendRequest: self.log.exception("Unable to stop node.") def is_node_stopped(self): """Checks whether the node has stopped. Returns True if the node has stopped. False otherwise. This method is responsible for freeing resources (self.process).""" if not self.running: return True return_code = self.process.poll() if return_code is not None: # process has stopped. Assert that it didn't return an error code. assert_equal(return_code, 0) self.running = False self.process = None self.log.debug("Node stopped") return True return False def node_encrypt_wallet(self, passphrase): """"Encrypts the wallet. This causes bitcoind to shutdown, so this method takes care of cleaning up resources.""" self.encryptwallet(passphrase) while not self.is_node_stopped(): time.sleep(0.1) self.rpc = None self.rpc_connected = False class TestNodeCLI(): """Interface to bitcoin-cli for an individual node""" def __init__(self, binary, datadir): + self.args = [] self.binary = binary self.datadir = datadir + self.input = None + + def __call__(self, *args, input=None): + # TestNodeCLI is callable with bitcoin-cli command-line args + self.args = [str(arg) for arg in args] + self.input = input + return self def __getattr__(self, command): def dispatcher(*args, **kwargs): return self.send_cli(command, *args, **kwargs) return dispatcher def send_cli(self, command, *args, **kwargs): """Run bitcoin-cli command. Deserializes returned string as python object.""" pos_args = [str(arg) for arg in args] named_args = [str(key) + "=" + str(value) for (key, value) in kwargs.items()] assert not ( pos_args and named_args), "Cannot use positional arguments and named arguments in the same bitcoin-cli call" - p_args = [self.binary, "-datadir=" + self.datadir] + + p_args = [self.binary, "-datadir=" + self.datadir] + self.args if named_args: p_args += ["-named"] p_args += [command] + pos_args + named_args - cli_output = subprocess.check_output(p_args, universal_newlines=True) - return json.loads(cli_output, parse_float=decimal.Decimal) + process = subprocess.Popen(p_args, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + cli_stdout, cli_stderr = process.communicate(input=self.input) + returncode = process.poll() + if returncode: + # Ignore cli_stdout, raise with cli_stderr + raise subprocess.CalledProcessError( + returncode, self.binary, output=cli_stderr) + return json.loads(cli_stdout, parse_float=decimal.Decimal) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 3643ae1f7..d5dc98889 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -1,654 +1,680 @@ #!/usr/bin/env python3 # Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. from base64 import b64encode from binascii import hexlify, unhexlify from decimal import Decimal, ROUND_DOWN import json import logging import os import random import re +from subprocess import CalledProcessError import time from . import coverage from .authproxy import AuthServiceProxy, JSONRPCException logger = logging.getLogger("TestFramework.utils") # Assert functions ################## def assert_fee_amount(fee, tx_size, fee_per_kB): """Assert the fee was in range""" target_fee = tx_size * fee_per_kB / 1000 if fee < target_fee: raise AssertionError( "Fee of %s BTC too low! (Should be %s BTC)" % (str(fee), str(target_fee))) # allow the wallet's estimation to be at most 2 bytes off if fee > (tx_size + 2) * fee_per_kB / 1000: raise AssertionError( "Fee of %s BTC too high! (Should be %s BTC)" % (str(fee), str(target_fee))) def assert_equal(thing1, thing2, *args): if thing1 != thing2 or any(thing1 != arg for arg in args): raise AssertionError("not(%s)" % " == ".join(str(arg) for arg in (thing1, thing2) + args)) def assert_greater_than(thing1, thing2): if thing1 <= thing2: raise AssertionError("%s <= %s" % (str(thing1), str(thing2))) def assert_greater_than_or_equal(thing1, thing2): if thing1 < thing2: raise AssertionError("%s < %s" % (str(thing1), str(thing2))) def assert_raises(exc, fun, *args, **kwds): assert_raises_message(exc, None, fun, *args, **kwds) def assert_raises_message(exc, message, fun, *args, **kwds): try: fun(*args, **kwds) except exc as e: if message is not None and message not in e.error['message']: raise AssertionError( "Expected substring not found:" + e.error['message']) except Exception as e: raise AssertionError( "Unexpected exception raised: " + type(e).__name__) else: raise AssertionError("No exception raised") +def assert_raises_process_error(returncode, output, fun, *args, **kwds): + """Execute a process and asserts the process return code and output. + + Calls function `fun` with arguments `args` and `kwds`. Catches a CalledProcessError + and verifies that the return code and output are as expected. Throws AssertionError if + no CalledProcessError was raised or if the return code and output are not as expected. + + Args: + returncode (int): the process return code. + output (string): [a substring of] the process output. + fun (function): the function to call. This should execute a process. + args*: positional arguments for the function. + kwds**: named arguments for the function. + """ + try: + fun(*args, **kwds) + except CalledProcessError as e: + if returncode != e.returncode: + raise AssertionError("Unexpected returncode %i" % e.returncode) + if output not in e.output: + raise AssertionError("Expected substring not found:" + e.output) + else: + raise AssertionError("No exception raised") + + def assert_raises_jsonrpc(code, message, fun, *args, **kwds): """Run an RPC and verify that a specific JSONRPC exception code and message is raised. Calls function `fun` with arguments `args` and `kwds`. Catches a JSONRPCException and verifies that the error code and message are as expected. Throws AssertionError if - no JSONRPCException was returned or if the error code/message are not as expected. + no JSONRPCException was raised or if the error code/message are not as expected. Args: code (int), optional: the error code returned by the RPC call (defined in src/rpc/protocol.h). Set to None if checking the error code is not required. message (string), optional: [a substring of] the error string returned by the - RPC call. Set to None if checking the error string is not required + RPC call. Set to None if checking the error string is not required. fun (function): the function to call. This should be the name of an RPC. args*: positional arguments for the function. kwds**: named arguments for the function. """ try: fun(*args, **kwds) except JSONRPCException as e: # JSONRPCException was thrown as expected. Check the code and message values are correct. if (code is not None) and (code != e.error["code"]): raise AssertionError( "Unexpected JSONRPC error code %i" % e.error["code"]) if (message is not None) and (message not in e.error['message']): raise AssertionError( "Expected substring not found:" + e.error['message']) except Exception as e: raise AssertionError( "Unexpected exception raised: " + type(e).__name__) else: raise AssertionError("No exception raised") def assert_is_hex_string(string): try: int(string, 16) except Exception as e: raise AssertionError( "Couldn't interpret %r as hexadecimal; raised: %s" % (string, e)) def assert_is_hash_string(string, length=64): if not isinstance(string, str): raise AssertionError("Expected a string, got type %r" % type(string)) elif length and len(string) != length: raise AssertionError( "String of length %d expected; got %d" % (length, len(string))) elif not re.match('[abcdef0-9]+$', string): raise AssertionError( "String %r contains invalid characters for a hash." % string) def assert_array_result(object_array, to_match, expected, should_not_find=False): """ Pass in array of JSON objects, a dictionary with key/value pairs to match against, and another dictionary with expected key/value pairs. If the should_not_find flag is true, to_match should not be found in object_array """ if should_not_find: assert_equal(expected, {}) num_matched = 0 for item in object_array: all_match = True for key, value in to_match.items(): if item[key] != value: all_match = False if not all_match: continue elif should_not_find: num_matched = num_matched + 1 for key, value in expected.items(): if item[key] != value: raise AssertionError("%s : expected %s=%s" % (str(item), str(key), str(value))) num_matched = num_matched + 1 if num_matched == 0 and not should_not_find: raise AssertionError("No objects matched %s" % (str(to_match))) if num_matched > 0 and should_not_find: raise AssertionError("Objects were found %s" % (str(to_match))) # Utility functions ################### def check_json_precision(): """Make sure json library being used does not lose precision converting BTC values""" n = Decimal("20000000.00000003") satoshis = int(json.loads(json.dumps(float(n))) * 1.0e8) if satoshis != 2000000000000003: raise RuntimeError("JSON encode/decode loses precision") def count_bytes(hex_string): return len(bytearray.fromhex(hex_string)) def bytes_to_hex_str(byte_str): return hexlify(byte_str).decode('ascii') def hex_str_to_bytes(hex_str): return unhexlify(hex_str.encode('ascii')) def str_to_b64str(string): return b64encode(string.encode('utf-8')).decode('ascii') def satoshi_round(amount): return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) # RPC/P2P connection constants and functions ############################################ # The maximum number of nodes a single test can spawn MAX_NODES = 8 # Don't assign rpc or p2p ports lower than this PORT_MIN = 11000 # The number of ports to "reserve" for p2p and rpc, each PORT_RANGE = 5000 class PortSeed: # Must be initialized with a unique integer for each process n = None def get_rpc_proxy(url, node_number, timeout=None, coveragedir=None): """ Args: url (str): URL of the RPC server to call node_number (int): the node number (or id) that this calls to Kwargs: timeout (int): HTTP timeout in seconds Returns: AuthServiceProxy. convenience object for making RPC calls. """ proxy_kwargs = {} if timeout is not None: proxy_kwargs['timeout'] = timeout proxy = AuthServiceProxy(url, **proxy_kwargs) proxy.url = url # store URL on proxy for info coverage_logfile = coverage.get_filename( coveragedir, node_number) if coveragedir else None return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile) def p2p_port(n): assert(n <= MAX_NODES) return PORT_MIN + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) def rpc_port(n): return PORT_MIN + PORT_RANGE + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) def rpc_url(datadir, i, rpchost=None): rpc_u, rpc_p = get_auth_cookie(datadir) host = '127.0.0.1' port = rpc_port(i) if rpchost: parts = rpchost.split(':') if len(parts) == 2: host, port = parts else: host = rpchost return "http://%s:%s@%s:%d" % (rpc_u, rpc_p, host, int(port)) # Node functions ################ def initialize_datadir(dirname, n): datadir = os.path.join(dirname, "node" + str(n)) if not os.path.isdir(datadir): os.makedirs(datadir) with open(os.path.join(datadir, "bitcoin.conf"), 'w', encoding='utf8') as f: f.write("regtest=1\n") f.write("port=" + str(p2p_port(n)) + "\n") f.write("rpcport=" + str(rpc_port(n)) + "\n") f.write("listenonion=0\n") f.write("usecashaddr=1\n") return datadir def get_datadir_path(dirname, n): return os.path.join(dirname, "node" + str(n)) def get_auth_cookie(datadir): user = None password = None if os.path.isfile(os.path.join(datadir, "bitcoin.conf")): with open(os.path.join(datadir, "bitcoin.conf"), 'r') as f: for line in f: if line.startswith("rpcuser="): assert user is None # Ensure that there is only one rpcuser line user = line.split("=")[1].strip("\n") if line.startswith("rpcpassword="): assert password is None # Ensure that there is only one rpcpassword line password = line.split("=")[1].strip("\n") if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")): with open(os.path.join(datadir, "regtest", ".cookie"), 'r') as f: userpass = f.read() split_userpass = userpass.split(':') user = split_userpass[0] password = split_userpass[1] if user is None or password is None: raise ValueError("No RPC credentials") return user, password def log_filename(dirname, n_node, logname): return os.path.join(dirname, "node" + str(n_node), "regtest", logname) def get_bip9_status(node, key): info = node.getblockchaininfo() return info['bip9_softforks'][key] def set_node_times(nodes, t): for node in nodes: node.setmocktime(t) def disconnect_nodes(from_connection, node_num): for peer_id in [peer['id'] for peer in from_connection.getpeerinfo() if "testnode%d" % node_num in peer['subver']]: from_connection.disconnectnode(nodeid=peer_id) for _ in range(50): if [peer['id'] for peer in from_connection.getpeerinfo() if "testnode%d" % node_num in peer['subver']] == []: break time.sleep(0.1) else: raise AssertionError("timed out waiting for disconnect") def connect_nodes(from_connection, node_num): ip_port = "127.0.0.1:" + str(p2p_port(node_num)) from_connection.addnode(ip_port, "onetry") # poll until version handshake complete to avoid race conditions # with transaction relaying while any(peer['version'] == 0 for peer in from_connection.getpeerinfo()): time.sleep(0.1) def connect_nodes_bi(nodes, a, b): connect_nodes(nodes[a], b) connect_nodes(nodes[b], a) def sync_blocks(rpc_connections, *, wait=1, timeout=60): """ Wait until everybody has the same tip. sync_blocks needs to be called with an rpc_connections set that has least one node already synced to the latest, stable tip, otherwise there's a chance it might return before all nodes are stably synced. """ # Use getblockcount() instead of waitforblockheight() to determine the # initial max height because the two RPCs look at different internal global # variables (chainActive vs latestBlock) and the former gets updated # earlier. maxheight = max(x.getblockcount() for x in rpc_connections) start_time = cur_time = time.time() while cur_time <= start_time + timeout: tips = [r.waitforblockheight(maxheight, int(wait * 1000)) for r in rpc_connections] if all(t["height"] == maxheight for t in tips): if all(t["hash"] == tips[0]["hash"] for t in tips): return raise AssertionError("Block sync failed, mismatched block hashes:{}".format( "".join("\n {!r}".format(tip) for tip in tips))) cur_time = time.time() raise AssertionError("Block sync to height {} timed out:{}".format( maxheight, "".join("\n {!r}".format(tip) for tip in tips))) def sync_chain(rpc_connections, *, wait=1, timeout=60): """ Wait until everybody has the same best block """ while timeout > 0: best_hash = [x.getbestblockhash() for x in rpc_connections] if best_hash == [best_hash[0]] * len(best_hash): return time.sleep(wait) timeout -= wait raise AssertionError("Chain sync failed: Best block hashes don't match") def sync_mempools(rpc_connections, *, wait=1, timeout=60): """ Wait until everybody has the same transactions in their memory pools """ while timeout > 0: pool = set(rpc_connections[0].getrawmempool()) num_match = 1 for i in range(1, len(rpc_connections)): if set(rpc_connections[i].getrawmempool()) == pool: num_match = num_match + 1 if num_match == len(rpc_connections): return time.sleep(wait) timeout -= wait raise AssertionError("Mempool sync failed") # Transaction/Block functions ############################# def find_output(node, txid, amount): """ Return index to output of txid with value amount Raises exception if there is none. """ txdata = node.getrawtransaction(txid, 1) for i in range(len(txdata["vout"])): if txdata["vout"][i]["value"] == amount: return i raise RuntimeError("find_output txid %s : %s not found" % (txid, str(amount))) def gather_inputs(from_node, amount_needed, confirmations_required=1): """ Return a random set of unspent txouts that are enough to pay amount_needed """ assert(confirmations_required >= 0) utxo = from_node.listunspent(confirmations_required) random.shuffle(utxo) inputs = [] total_in = Decimal("0.00000000") while total_in < amount_needed and len(utxo) > 0: t = utxo.pop() total_in += t["amount"] inputs.append( {"txid": t["txid"], "vout": t["vout"], "address": t["address"]}) if total_in < amount_needed: raise RuntimeError("Insufficient funds: need %d, have %d" % (amount_needed, total_in)) return (total_in, inputs) def make_change(from_node, amount_in, amount_out, fee): """ Create change output(s), return them """ outputs = {} amount = amount_out + fee change = amount_in - amount if change > amount * 2: # Create an extra change output to break up big inputs change_address = from_node.getnewaddress() # Split change in two, being careful of rounding: outputs[change_address] = Decimal( change / 2).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) change = amount_in - amount - outputs[change_address] if change > 0: outputs[from_node.getnewaddress()] = change return outputs def send_zeropri_transaction(from_node, to_node, amount, fee): """ Create&broadcast a zero-priority transaction. Returns (txid, hex-encoded-txdata) Ensures transaction is zero-priority by first creating a send-to-self, then using its output """ # Create a send-to-self with confirmed inputs: self_address = from_node.getnewaddress() (total_in, inputs) = gather_inputs(from_node, amount + fee * 2) outputs = make_change(from_node, total_in, amount + fee, fee) outputs[self_address] = float(amount + fee) self_rawtx = from_node.createrawtransaction(inputs, outputs) self_signresult = from_node.signrawtransaction( self_rawtx, None, None, "ALL|FORKID") self_txid = from_node.sendrawtransaction(self_signresult["hex"], True) vout = find_output(from_node, self_txid, amount + fee) # Now immediately spend the output to create a 1-input, 1-output # zero-priority transaction: inputs = [{"txid": self_txid, "vout": vout}] outputs = {to_node.getnewaddress(): float(amount)} rawtx = from_node.createrawtransaction(inputs, outputs) signresult = from_node.signrawtransaction(rawtx, None, None, "ALL|FORKID") txid = from_node.sendrawtransaction(signresult["hex"], True) return (txid, signresult["hex"]) def random_zeropri_transaction(nodes, amount, min_fee, fee_increment, fee_variants): """ Create a random zero-priority transaction. Returns (txid, hex-encoded-transaction-data, fee) """ from_node = random.choice(nodes) to_node = random.choice(nodes) fee = min_fee + fee_increment * random.randint(0, fee_variants) (txid, txhex) = send_zeropri_transaction(from_node, to_node, amount, fee) return (txid, txhex, fee) def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants): """ Create a random transaction. Returns (txid, hex-encoded-transaction-data, fee) """ from_node = random.choice(nodes) to_node = random.choice(nodes) fee = min_fee + fee_increment * random.randint(0, fee_variants) (total_in, inputs) = gather_inputs(from_node, amount + fee) outputs = make_change(from_node, total_in, amount, fee) outputs[to_node.getnewaddress()] = float(amount) rawtx = from_node.createrawtransaction(inputs, outputs) signresult = from_node.signrawtransaction(rawtx, None, None, "ALL|FORKID") txid = from_node.sendrawtransaction(signresult["hex"], True) return (txid, signresult["hex"], fee) # Helper to create at least "count" utxos # Pass in a fee that is sufficient for relay and mining new transactions. def create_confirmed_utxos(fee, node, count, age=101): to_generate = int(0.5 * count) + age while to_generate > 0: node.generate(min(25, to_generate)) to_generate -= 25 utxos = node.listunspent() iterations = count - len(utxos) addr1 = node.getnewaddress() addr2 = node.getnewaddress() if iterations <= 0: return utxos for i in range(iterations): t = utxos.pop() inputs = [] inputs.append({"txid": t["txid"], "vout": t["vout"]}) outputs = {} send_value = t['amount'] - fee outputs[addr1] = satoshi_round(send_value / 2) outputs[addr2] = satoshi_round(send_value / 2) raw_tx = node.createrawtransaction(inputs, outputs) signed_tx = node.signrawtransaction( raw_tx, None, None, "ALL|FORKID")["hex"] node.sendrawtransaction(signed_tx) while (node.getmempoolinfo()['size'] > 0): node.generate(1) utxos = node.listunspent() assert(len(utxos) >= count) return utxos # Create large OP_RETURN txouts that can be appended to a transaction # to make it large (helper for constructing large transactions). def gen_return_txouts(): # Some pre-processing to create a bunch of OP_RETURN txouts to insert into transactions we create # So we have big transactions (and therefore can't fit very many into each block) # create one script_pubkey script_pubkey = "6a4d0200" # OP_RETURN OP_PUSH2 512 bytes for i in range(512): script_pubkey = script_pubkey + "01" # concatenate 128 txouts of above script_pubkey which we'll insert before # the txout for change txouts = "81" for k in range(128): # add txout value txouts = txouts + "0000000000000000" # add length of script_pubkey txouts = txouts + "fd0402" # add script_pubkey txouts = txouts + script_pubkey return txouts def create_tx(node, coinbase, to_address, amount): inputs = [{"txid": coinbase, "vout": 0}] outputs = {to_address: amount} rawtx = node.createrawtransaction(inputs, outputs) signresult = node.signrawtransaction(rawtx, None, None, "ALL|FORKID") assert_equal(signresult["complete"], True) return signresult["hex"] # Create a spend of each passed-in utxo, splicing in "txouts" to each raw # transaction to make it large. See gen_return_txouts() above. def create_lots_of_big_transactions(node, txouts, utxos, num, fee): addr = node.getnewaddress() txids = [] for _ in range(num): t = utxos.pop() inputs = [{"txid": t["txid"], "vout": t["vout"]}] outputs = {} change = t['amount'] - fee outputs[addr] = satoshi_round(change) rawtx = node.createrawtransaction(inputs, outputs) newtx = rawtx[0:92] newtx = newtx + txouts newtx = newtx + rawtx[94:] signresult = node.signrawtransaction(newtx, None, None, "NONE|FORKID") txid = node.sendrawtransaction(signresult["hex"], True) txids.append(txid) return txids def mine_large_block(node, utxos=None): # generate a 66k transaction, # and 14 of them is close to the 1MB block limit num = 14 txouts = gen_return_txouts() utxos = utxos if utxos is not None else [] if len(utxos) < num: utxos.clear() utxos.extend(node.listunspent()) fee = 100 * node.getnetworkinfo()["relayfee"] create_lots_of_big_transactions(node, txouts, utxos, num, fee=fee) node.generate(1) def get_srcdir(calling_script=None): """ Try to find out the base folder containing the 'src' folder. If SRCDIR is set it does a sanity check and returns that. Otherwise it goes on a search and rescue mission. Returns None if it cannot find a suitable folder. TODO: This is only used for cdefs, consider moving that there. """ def contains_src(path_to_check): if not path_to_check: return False else: cand_path = os.path.join(path_to_check, 'src') return os.path.exists(cand_path) and os.path.isdir(cand_path) srcdir = os.environ.get('SRCDIR', '') if contains_src(srcdir): return srcdir # If we have a caller, try to guess from its location where the # top level might be. if calling_script: caller_basedir = os.path.dirname( os.path.dirname(os.path.dirname(calling_script))) if caller_basedir != '' and contains_src(os.path.abspath(caller_basedir)): return os.path.abspath(caller_basedir) # Try to work it based out on main module # We might expect the caller to be rpc-tests.py or a test script # itself. import sys mainmod = sys.modules['__main__'] mainmod_path = getattr(mainmod, '__file__', '') if mainmod_path and mainmod_path.endswith('.py'): maybe_top = os.path.dirname( os.path.dirname(os.path.dirname(mainmod_path))) if contains_src(os.path.abspath(maybe_top)): return os.path.abspath(maybe_top) # No luck, give up. return None