diff --git a/src/httpserver.cpp b/src/httpserver.cpp index fab59da3f0..835d33c01a 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -1,660 +1,669 @@ // Copyright (c) 2015-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "httpserver.h" #include "chainparamsbase.h" #include "compat.h" +#include "config.h" #include "netbase.h" #include "rpc/protocol.h" // For HTTP status codes #include "sync.h" #include "ui_interface.h" #include "util.h" #include "utilstrencodings.h" #include #include #include #include #include #include #include #include #include #ifdef EVENT__HAVE_NETINET_IN_H #include #ifdef _XOPEN_SOURCE_EXTENDED #include #endif #endif #include #include #include #include /** Maximum size of http request (request line + headers) */ static const size_t MAX_HEADERS_SIZE = 8192; +/** + * Maximum HTTP post body size. Twice the maximum block size is added to this + * value in practice. + */ +static const size_t MIN_SUPPORTED_BODY_SIZE = 0x02000000; + /** HTTP request work item */ class HTTPWorkItem final : public HTTPClosure { public: HTTPWorkItem(Config &_config, std::unique_ptr _req, const std::string &_path, const HTTPRequestHandler &_func) : req(std::move(_req)), path(_path), func(_func), config(&_config) {} void operator()() override { func(*config, req.get(), path); } std::unique_ptr req; private: std::string path; HTTPRequestHandler func; Config *config; }; -/** Simple work queue for distributing work over multiple threads. +/** + * Simple work queue for distributing work over multiple threads. * Work items are simply callable objects. */ template class WorkQueue { private: /** Mutex protects entire object */ std::mutex cs; std::condition_variable cond; std::deque> queue; bool running; size_t maxDepth; int numThreads; /** RAII object to keep track of number of running worker threads */ class ThreadCounter { public: WorkQueue &wq; ThreadCounter(WorkQueue &w) : wq(w) { std::lock_guard lock(wq.cs); wq.numThreads += 1; } ~ThreadCounter() { std::lock_guard lock(wq.cs); wq.numThreads -= 1; wq.cond.notify_all(); } }; public: WorkQueue(size_t _maxDepth) : running(true), maxDepth(_maxDepth), numThreads(0) {} /** Precondition: worker threads have all stopped * (call WaitExit) */ ~WorkQueue() {} /** Enqueue a work item */ bool Enqueue(WorkItem *item) { std::unique_lock lock(cs); if (queue.size() >= maxDepth) { return false; } queue.emplace_back(std::unique_ptr(item)); cond.notify_one(); return true; } /** Thread function */ void Run() { ThreadCounter count(*this); while (true) { std::unique_ptr i; { std::unique_lock lock(cs); while (running && queue.empty()) cond.wait(lock); if (!running) break; i = std::move(queue.front()); queue.pop_front(); } (*i)(); } } /** Interrupt and exit loops */ void Interrupt() { std::unique_lock lock(cs); running = false; cond.notify_all(); } /** Wait for worker threads to exit */ void WaitExit() { std::unique_lock lock(cs); while (numThreads > 0) cond.wait(lock); } /** Return current depth of queue */ size_t Depth() { std::unique_lock lock(cs); return queue.size(); } }; struct HTTPPathHandler { HTTPPathHandler() {} HTTPPathHandler(std::string _prefix, bool _exactMatch, HTTPRequestHandler _handler) : prefix(_prefix), exactMatch(_exactMatch), handler(_handler) {} std::string prefix; bool exactMatch; HTTPRequestHandler handler; }; /** HTTP module state */ //! libevent event loop static struct event_base *eventBase = 0; //! HTTP server struct evhttp *eventHTTP = 0; //! List of subnets to allow RPC connections from static std::vector rpc_allow_subnets; //! Work queue for handling longer requests off the event loop thread static WorkQueue *workQueue = 0; //! Handlers for (sub)paths std::vector pathHandlers; //! Bound listening sockets std::vector boundSockets; /** Check if a network address is allowed to access the HTTP server */ static bool ClientAllowed(const CNetAddr &netaddr) { if (!netaddr.IsValid()) return false; for (const CSubNet &subnet : rpc_allow_subnets) if (subnet.Match(netaddr)) return true; return false; } /** Initialize ACL list for HTTP server */ static bool InitHTTPAllowList() { rpc_allow_subnets.clear(); CNetAddr localv4; CNetAddr localv6; LookupHost("127.0.0.1", localv4, false); LookupHost("::1", localv6, false); // always allow IPv4 local subnet. rpc_allow_subnets.push_back(CSubNet(localv4, 8)); // always allow IPv6 localhost. rpc_allow_subnets.push_back(CSubNet(localv6)); if (gArgs.IsArgSet("-rpcallowip")) { for (const std::string &strAllow : gArgs.GetArgs("-rpcallowip")) { CSubNet subnet; LookupSubNet(strAllow.c_str(), subnet); if (!subnet.IsValid()) { uiInterface.ThreadSafeMessageBox( strprintf("Invalid -rpcallowip subnet specification: %s. " "Valid are a single IP (e.g. 1.2.3.4), a " "network/netmask (e.g. 1.2.3.4/255.255.255.0) or " "a network/CIDR (e.g. 1.2.3.4/24).", strAllow), "", CClientUIInterface::MSG_ERROR); return false; } rpc_allow_subnets.push_back(subnet); } } std::string strAllowed; for (const CSubNet &subnet : rpc_allow_subnets) strAllowed += subnet.ToString() + " "; LogPrint(BCLog::HTTP, "Allowing HTTP connections from: %s\n", strAllowed); return true; } /** HTTP request method as string - use for logging only */ static std::string RequestMethodString(HTTPRequest::RequestMethod m) { switch (m) { case HTTPRequest::GET: return "GET"; break; case HTTPRequest::POST: return "POST"; break; case HTTPRequest::HEAD: return "HEAD"; break; case HTTPRequest::PUT: return "PUT"; break; default: return "unknown"; } } /** HTTP request callback */ static void http_request_cb(struct evhttp_request *req, void *arg) { Config &config = *reinterpret_cast(arg); std::unique_ptr hreq(new HTTPRequest(req)); LogPrint(BCLog::HTTP, "Received a %s request for %s from %s\n", RequestMethodString(hreq->GetRequestMethod()), hreq->GetURI(), hreq->GetPeer().ToString()); // Early address-based allow check if (!ClientAllowed(hreq->GetPeer())) { hreq->WriteReply(HTTP_FORBIDDEN); return; } // Early reject unknown HTTP methods if (hreq->GetRequestMethod() == HTTPRequest::UNKNOWN) { hreq->WriteReply(HTTP_BADMETHOD); return; } // Find registered handler for prefix std::string strURI = hreq->GetURI(); std::string path; std::vector::const_iterator i = pathHandlers.begin(); std::vector::const_iterator iend = pathHandlers.end(); for (; i != iend; ++i) { bool match = false; if (i->exactMatch) { match = (strURI == i->prefix); } else { match = (strURI.substr(0, i->prefix.size()) == i->prefix); } if (match) { path = strURI.substr(i->prefix.size()); break; } } // Dispatch to worker thread. if (i != iend) { std::unique_ptr item( new HTTPWorkItem(config, std::move(hreq), path, i->handler)); assert(workQueue); if (workQueue->Enqueue(item.get())) { /* if true, queue took ownership */ item.release(); } else { LogPrintf("WARNING: request rejected because http work queue depth " "exceeded, it can be increased with the -rpcworkqueue= " "setting\n"); item->req->WriteReply(HTTP_INTERNAL, "Work queue depth exceeded"); } } else { hreq->WriteReply(HTTP_NOTFOUND); } } /** Callback to reject HTTP requests after shutdown. */ static void http_reject_request_cb(struct evhttp_request *req, void *) { LogPrint(BCLog::HTTP, "Rejecting request while shutting down\n"); evhttp_send_error(req, HTTP_SERVUNAVAIL, nullptr); } /** Event dispatcher thread */ static bool ThreadHTTP(struct event_base *base, struct evhttp *http) { RenameThread("bitcoin-http"); LogPrint(BCLog::HTTP, "Entering http event loop\n"); event_base_dispatch(base); // Event loop will be interrupted by InterruptHTTPServer() LogPrint(BCLog::HTTP, "Exited http event loop\n"); return event_base_got_break(base) == 0; } /** Bind HTTP server to specified addresses */ static bool HTTPBindAddresses(struct evhttp *http) { int defaultPort = gArgs.GetArg("-rpcport", BaseParams().RPCPort()); std::vector> endpoints; // Determine what addresses to bind to if (!gArgs.IsArgSet("-rpcallowip")) { // Default to loopback if not allowing external IPs. endpoints.push_back(std::make_pair("::1", defaultPort)); endpoints.push_back(std::make_pair("127.0.0.1", defaultPort)); if (gArgs.IsArgSet("-rpcbind")) { LogPrintf("WARNING: option -rpcbind was ignored because " "-rpcallowip was not specified, refusing to allow " "everyone to connect\n"); } } else if (gArgs.IsArgSet("-rpcbind")) { // Specific bind address. for (const std::string &strRPCBind : gArgs.GetArgs("-rpcbind")) { int port = defaultPort; std::string host; SplitHostPort(strRPCBind, port, host); endpoints.push_back(std::make_pair(host, port)); } } else { // No specific bind address specified, bind to any. endpoints.push_back(std::make_pair("::", defaultPort)); endpoints.push_back(std::make_pair("0.0.0.0", defaultPort)); } // Bind addresses for (std::vector>::iterator i = endpoints.begin(); i != endpoints.end(); ++i) { LogPrint(BCLog::HTTP, "Binding RPC on address %s port %i\n", i->first, i->second); evhttp_bound_socket *bind_handle = evhttp_bind_socket_with_handle( http, i->first.empty() ? nullptr : i->first.c_str(), i->second); if (bind_handle) { boundSockets.push_back(bind_handle); } else { LogPrintf("Binding RPC on address %s port %i failed.\n", i->first, i->second); } } return !boundSockets.empty(); } /** Simple wrapper to set thread name and run work queue */ static void HTTPWorkQueueRun(WorkQueue *queue) { RenameThread("bitcoin-httpworker"); queue->Run(); } /** libevent event log callback */ static void libevent_log_cb(int severity, const char *msg) { #ifndef EVENT_LOG_WARN // EVENT_LOG_WARN was added in 2.0.19; but before then _EVENT_LOG_WARN existed. #define EVENT_LOG_WARN _EVENT_LOG_WARN #endif // Log warn messages and higher without debug category. if (severity >= EVENT_LOG_WARN) { LogPrintf("libevent: %s\n", msg); } else { LogPrint(BCLog::LIBEVENT, "libevent: %s\n", msg); } } bool InitHTTPServer(Config &config) { struct evhttp *http = 0; struct event_base *base = 0; if (!InitHTTPAllowList()) return false; if (gArgs.GetBoolArg("-rpcssl", false)) { uiInterface.ThreadSafeMessageBox( "SSL mode for RPC (-rpcssl) is no longer supported.", "", CClientUIInterface::MSG_ERROR); return false; } // Redirect libevent's logging to our own log event_set_log_callback(&libevent_log_cb); #if LIBEVENT_VERSION_NUMBER >= 0x02010100 // If -debug=libevent, set full libevent debugging. // Otherwise, disable all libevent debugging. if (LogAcceptCategory(BCLog::LIBEVENT)) { event_enable_debug_logging(EVENT_DBG_ALL); } else { event_enable_debug_logging(EVENT_DBG_NONE); } #endif #ifdef WIN32 evthread_use_windows_threads(); #else evthread_use_pthreads(); #endif // XXX RAII base = event_base_new(); if (!base) { LogPrintf("Couldn't create an event_base: exiting\n"); return false; } /* Create a new evhttp object to handle requests. */ // XXX RAII http = evhttp_new(base); if (!http) { LogPrintf("couldn't create evhttp. Exiting.\n"); event_base_free(base); return false; } evhttp_set_timeout( http, gArgs.GetArg("-rpcservertimeout", DEFAULT_HTTP_SERVER_TIMEOUT)); evhttp_set_max_headers_size(http, MAX_HEADERS_SIZE); - evhttp_set_max_body_size(http, MAX_SIZE); + evhttp_set_max_body_size( + http, MIN_SUPPORTED_BODY_SIZE + 2 * config.GetMaxBlockSize()); evhttp_set_gencb(http, http_request_cb, &config); if (!HTTPBindAddresses(http)) { LogPrintf("Unable to bind any endpoint for RPC server\n"); evhttp_free(http); event_base_free(base); return false; } LogPrint(BCLog::HTTP, "Initialized HTTP server\n"); int workQueueDepth = std::max( (long)gArgs.GetArg("-rpcworkqueue", DEFAULT_HTTP_WORKQUEUE), 1L); LogPrintf("HTTP: creating work queue of depth %d\n", workQueueDepth); workQueue = new WorkQueue(workQueueDepth); eventBase = base; eventHTTP = http; return true; } std::thread threadHTTP; std::future threadResult; bool StartHTTPServer() { LogPrint(BCLog::HTTP, "Starting HTTP server\n"); int rpcThreads = std::max((long)gArgs.GetArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); LogPrintf("HTTP: starting %d worker threads\n", rpcThreads); std::packaged_task task(ThreadHTTP); threadResult = task.get_future(); threadHTTP = std::thread(std::move(task), eventBase, eventHTTP); for (int i = 0; i < rpcThreads; i++) { std::thread rpc_worker(HTTPWorkQueueRun, workQueue); rpc_worker.detach(); } return true; } void InterruptHTTPServer() { LogPrint(BCLog::HTTP, "Interrupting HTTP server\n"); if (eventHTTP) { // Unlisten sockets for (evhttp_bound_socket *socket : boundSockets) { evhttp_del_accept_socket(eventHTTP, socket); } // Reject requests on current connections evhttp_set_gencb(eventHTTP, http_reject_request_cb, nullptr); } if (workQueue) workQueue->Interrupt(); } void StopHTTPServer() { LogPrint(BCLog::HTTP, "Stopping HTTP server\n"); if (workQueue) { LogPrint(BCLog::HTTP, "Waiting for HTTP worker threads to exit\n"); workQueue->WaitExit(); delete workQueue; } if (eventBase) { LogPrint(BCLog::HTTP, "Waiting for HTTP event thread to exit\n"); // Give event loop a few seconds to exit (to send back last RPC // responses), then break it. Before this was solved with // event_base_loopexit, but that didn't work as expected in at least // libevent 2.0.21 and always introduced a delay. In libevent master // that appears to be solved, so in the future that solution could be // used again (if desirable). // (see discussion in https://github.com/bitcoin/bitcoin/pull/6990) if (threadResult.valid() && threadResult.wait_for(std::chrono::milliseconds(2000)) == std::future_status::timeout) { LogPrintf("HTTP event loop did not exit within allotted time, " "sending loopbreak\n"); event_base_loopbreak(eventBase); } threadHTTP.join(); } if (eventHTTP) { evhttp_free(eventHTTP); eventHTTP = 0; } if (eventBase) { event_base_free(eventBase); eventBase = 0; } LogPrint(BCLog::HTTP, "Stopped HTTP server\n"); } struct event_base *EventBase() { return eventBase; } static void httpevent_callback_fn(evutil_socket_t, short, void *data) { // Static handler: simply call inner handler HTTPEvent *self = ((HTTPEvent *)data); self->handler(); if (self->deleteWhenTriggered) delete self; } HTTPEvent::HTTPEvent(struct event_base *base, bool _deleteWhenTriggered, const std::function &_handler) : deleteWhenTriggered(_deleteWhenTriggered), handler(_handler) { ev = event_new(base, -1, 0, httpevent_callback_fn, this); assert(ev); } HTTPEvent::~HTTPEvent() { event_free(ev); } void HTTPEvent::trigger(struct timeval *tv) { if (tv == nullptr) { // Immediately trigger event in main thread. event_active(ev, 0, 0); } else { // Trigger after timeval passed. evtimer_add(ev, tv); } } HTTPRequest::HTTPRequest(struct evhttp_request *_req) : req(_req), replySent(false) {} HTTPRequest::~HTTPRequest() { if (!replySent) { // Keep track of whether reply was sent to avoid request leaks LogPrintf("%s: Unhandled request\n", __func__); WriteReply(HTTP_INTERNAL, "Unhandled request"); } // evhttpd cleans up the request, as long as a reply was sent. } std::pair HTTPRequest::GetHeader(const std::string &hdr) { const struct evkeyvalq *headers = evhttp_request_get_input_headers(req); assert(headers); const char *val = evhttp_find_header(headers, hdr.c_str()); if (val) return std::make_pair(true, val); else return std::make_pair(false, ""); } std::string HTTPRequest::ReadBody() { struct evbuffer *buf = evhttp_request_get_input_buffer(req); if (!buf) return ""; size_t size = evbuffer_get_length(buf); /** Trivial implementation: if this is ever a performance bottleneck, * internal copying can be avoided in multi-segment buffers by using * evbuffer_peek and an awkward loop. Though in that case, it'd be even * better to not copy into an intermediate string but use a stream * abstraction to consume the evbuffer on the fly in the parsing algorithm. */ const char *data = (const char *)evbuffer_pullup(buf, size); // returns nullptr in case of empty buffer. if (!data) { return ""; } std::string rv(data, size); evbuffer_drain(buf, size); return rv; } void HTTPRequest::WriteHeader(const std::string &hdr, const std::string &value) { struct evkeyvalq *headers = evhttp_request_get_output_headers(req); assert(headers); evhttp_add_header(headers, hdr.c_str(), value.c_str()); } /** Closure sent to main thread to request a reply to be sent to a HTTP request. * Replies must be sent in the main loop in the main http thread, this cannot be * done from worker threads. */ void HTTPRequest::WriteReply(int nStatus, const std::string &strReply) { assert(!replySent && req); // Send event to main http thread to send reply message struct evbuffer *evb = evhttp_request_get_output_buffer(req); assert(evb); evbuffer_add(evb, strReply.data(), strReply.size()); HTTPEvent *ev = new HTTPEvent(eventBase, true, std::bind(evhttp_send_reply, req, nStatus, (const char *)nullptr, (struct evbuffer *)nullptr)); ev->trigger(0); replySent = true; // transferred back to main thread. req = 0; } CService HTTPRequest::GetPeer() { evhttp_connection *con = evhttp_request_get_connection(req); CService peer; if (con) { // evhttp retains ownership over returned address string const char *address = ""; uint16_t port = 0; evhttp_connection_get_peer(con, (char **)&address, &port); peer = LookupNumeric(address, port); } return peer; } std::string HTTPRequest::GetURI() { return evhttp_request_get_uri(req); } HTTPRequest::RequestMethod HTTPRequest::GetRequestMethod() { switch (evhttp_request_get_command(req)) { case EVHTTP_REQ_GET: return GET; break; case EVHTTP_REQ_POST: return POST; break; case EVHTTP_REQ_HEAD: return HEAD; break; case EVHTTP_REQ_PUT: return PUT; break; default: return UNKNOWN; break; } } void RegisterHTTPHandler(const std::string &prefix, bool exactMatch, const HTTPRequestHandler &handler) { LogPrint(BCLog::HTTP, "Registering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); pathHandlers.push_back(HTTPPathHandler(prefix, exactMatch, handler)); } void UnregisterHTTPHandler(const std::string &prefix, bool exactMatch) { std::vector::iterator i = pathHandlers.begin(); std::vector::iterator iend = pathHandlers.end(); for (; i != iend; ++i) if (i->prefix == prefix && i->exactMatch == exactMatch) break; if (i != iend) { LogPrint(BCLog::HTTP, "Unregistering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); pathHandlers.erase(i); } } diff --git a/test/functional/abc-p2p-fullblocktest.py b/test/functional/abc-p2p-fullblocktest.py index bb1022bcd6..478cb77333 100755 --- a/test/functional/abc-p2p-fullblocktest.py +++ b/test/functional/abc-p2p-fullblocktest.py @@ -1,431 +1,436 @@ #!/usr/bin/env python3 # Copyright (c) 2015-2016 The Bitcoin Core developers # Copyright (c) 2017 The Bitcoin developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ This test checks simple acceptance of bigger blocks via p2p. It is derived from the much more complex p2p-fullblocktest. The intention is that small tests can be derived from this one, or this one can be extended, to cover the checks done for bigger blocks (e.g. sigops limits). """ from test_framework.test_framework import ComparisonTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error from test_framework.comptool import TestManager, TestInstance, RejectResult from test_framework.blocktools import * import time from test_framework.key import CECKey from test_framework.script import * from test_framework.cdefs import (ONE_MEGABYTE, LEGACY_MAX_BLOCK_SIZE, MAX_BLOCK_SIGOPS_PER_MB, MAX_TX_SIGOPS_COUNT) from collections import deque # far into the future MONOLITH_START_TIME = 2000000000 class PreviousSpendableOutput(): def __init__(self, tx=CTransaction(), n=-1): self.tx = tx self.n = n # the output we're spending class FullBlockTest(ComparisonTestFramework): # Can either run this test as 1 node with expected answers, or two and compare them. # Change the "outcome" variable from each TestInstance object to only do # the comparison. def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True self.block_heights = {} self.tip = None self.blocks = {} self.excessive_block_size = 100 * ONE_MEGABYTE self.extra_args = [['-whitelist=127.0.0.1', "-monolithactivationtime=%d" % MONOLITH_START_TIME, "-replayprotectionactivationtime=%d" % ( 2 * MONOLITH_START_TIME), "-excessiveblocksize=%d" % self.excessive_block_size]] def add_options(self, parser): super().add_options(parser) parser.add_option( "--runbarelyexpensive", dest="runbarelyexpensive", default=True) def run_test(self): self.test = TestManager(self, self.options.tmpdir) self.test.add_all_connections(self.nodes) # Start up network handling in another thread NetworkThread().start() # Set the blocksize to 2MB as initial condition self.nodes[0].setexcessiveblock(self.excessive_block_size) self.nodes[0].setmocktime(MONOLITH_START_TIME) self.test.run() def add_transactions_to_block(self, block, tx_list): [tx.rehash() for tx in tx_list] block.vtx.extend(tx_list) # this is a little handier to use than the version in blocktools.py def create_tx(self, spend, value, script=CScript([OP_TRUE])): tx = create_transaction(spend.tx, spend.n, b"", value, script) return tx def next_block(self, number, spend=None, script=CScript([OP_TRUE]), block_size=0, extra_sigops=0): if self.tip == None: base_block_hash = self.genesis_hash block_time = int(time.time()) + 1 else: base_block_hash = self.tip.sha256 block_time = self.tip.nTime + 1 # First create the coinbase height = self.block_heights[base_block_hash] + 1 coinbase = create_coinbase(height) coinbase.rehash() if spend == None: # We need to have something to spend to fill the block. assert_equal(block_size, 0) block = create_block(base_block_hash, coinbase, block_time) else: # all but one satoshi to fees coinbase.vout[0].nValue += spend.tx.vout[spend.n].nValue - 1 coinbase.rehash() block = create_block(base_block_hash, coinbase, block_time) # Make sure we have plenty engough to spend going forward. spendable_outputs = deque([spend]) def get_base_transaction(): # Create the new transaction tx = CTransaction() # Spend from one of the spendable outputs spend = spendable_outputs.popleft() tx.vin.append(CTxIn(COutPoint(spend.tx.sha256, spend.n))) # Add spendable outputs for i in range(4): tx.vout.append(CTxOut(0, CScript([OP_TRUE]))) spendable_outputs.append(PreviousSpendableOutput(tx, i)) return tx tx = get_base_transaction() # Make it the same format as transaction added for padding and save the size. # It's missing the padding output, so we add a constant to account for it. tx.rehash() base_tx_size = len(tx.serialize()) + 18 # If a specific script is required, add it. if script != None: tx.vout.append(CTxOut(1, script)) # Put some random data into the first transaction of the chain to randomize ids. tx.vout.append( CTxOut(0, CScript([random.randint(0, 256), OP_RETURN]))) # Add the transaction to the block self.add_transactions_to_block(block, [tx]) # If we have a block size requirement, just fill # the block until we get there current_block_size = len(block.serialize()) while current_block_size < block_size: # We will add a new transaction. That means the size of # the field enumerating how many transaction go in the block # may change. current_block_size -= len(ser_compact_size(len(block.vtx))) current_block_size += len(ser_compact_size(len(block.vtx) + 1)) # Create the new transaction tx = get_base_transaction() # Add padding to fill the block. script_length = block_size - current_block_size - base_tx_size if script_length > 510000: if script_length < 1000000: # Make sure we don't find ourselves in a position where we # need to generate a transaction smaller than what we expected. script_length = script_length // 2 else: script_length = 500000 tx_sigops = min(extra_sigops, script_length, MAX_TX_SIGOPS_COUNT) extra_sigops -= tx_sigops script_pad_len = script_length - tx_sigops script_output = CScript( [b'\x00' * script_pad_len] + [OP_CHECKSIG] * tx_sigops) tx.vout.append(CTxOut(0, script_output)) # Add the tx to the list of transactions to be included # in the block. self.add_transactions_to_block(block, [tx]) current_block_size += len(tx.serialize()) # Now that we added a bunch of transaction, we need to recompute # the merkle root. block.hashMerkleRoot = block.calc_merkle_root() # Check that the block size is what's expected if block_size > 0: assert_equal(len(block.serialize()), block_size) # Do PoW, which is cheap on regnet block.solve() self.tip = block self.block_heights[block.sha256] = height assert number not in self.blocks self.blocks[number] = block return block def get_tests(self): node = self.nodes[0] self.genesis_hash = int(node.getbestblockhash(), 16) self.block_heights[self.genesis_hash] = 0 spendable_outputs = [] # save the current tip so it can be spent by a later block def save_spendable_output(): spendable_outputs.append(self.tip) # get an output that we previously marked as spendable def get_spendable_output(): return PreviousSpendableOutput(spendable_outputs.pop(0).vtx[0], 0) # returns a test case that asserts that the current tip was accepted def accepted(): return TestInstance([[self.tip, True]]) # returns a test case that asserts that the current tip was rejected def rejected(reject=None): if reject is None: return TestInstance([[self.tip, False]]) else: return TestInstance([[self.tip, reject]]) # move the tip back to a previous block def tip(number): self.tip = self.blocks[number] # adds transactions to the block and updates state def update_block(block_number, new_transactions): block = self.blocks[block_number] self.add_transactions_to_block(block, new_transactions) old_sha256 = block.sha256 block.hashMerkleRoot = block.calc_merkle_root() block.solve() # Update the internal state just like in next_block self.tip = block if block.sha256 != old_sha256: self.block_heights[ block.sha256] = self.block_heights[old_sha256] del self.block_heights[old_sha256] self.blocks[block_number] = block return block # shorthand for functions block = self.next_block # Create a new block block(0) save_spendable_output() yield accepted() # Now we need that block to mature so we can spend the coinbase. test = TestInstance(sync_every_block=False) for i in range(99): block(5000 + i) test.blocks_and_transactions.append([self.tip, True]) save_spendable_output() yield test # collect spendable outputs now to avoid cluttering the code later on out = [] for i in range(100): out.append(get_spendable_output()) # Let's build some blocks and test them. for i in range(15): n = i + 1 block(n, spend=out[i], block_size=n * ONE_MEGABYTE // 2) yield accepted() # Start moving MTP forward bfork = block(5555, out[15], block_size=8 * ONE_MEGABYTE) bfork.nTime = MONOLITH_START_TIME - 1 update_block(5555, []) yield accepted() # Get to one block of the May 15, 2018 HF activation for i in range(5): block(5100 + i) test.blocks_and_transactions.append([self.tip, True]) yield test # Check that the MTP is just before the configured fork point. assert_equal(node.getblockheader(node.getbestblockhash())['mediantime'], MONOLITH_START_TIME - 1) # Before we acivate the May 15, 2018 HF, 8MB is the limit. block(4444, spend=out[16], block_size=8 * ONE_MEGABYTE + 1) yield rejected(RejectResult(16, b'bad-blk-length')) # Rewind bad block. tip(5104) # Actiavte the May 15, 2018 HF block(5556) yield accepted() # Now MTP is exactly the fork time. Bigger blocks are now accepted. assert_equal(node.getblockheader(node.getbestblockhash())['mediantime'], MONOLITH_START_TIME) # block of maximal size block(17, spend=out[16], block_size=self.excessive_block_size) yield accepted() # Reject oversized blocks with bad-blk-length error block(18, spend=out[17], block_size=self.excessive_block_size + 1) yield rejected(RejectResult(16, b'bad-blk-length')) # Rewind bad block. tip(17) # Accept many sigops lots_of_checksigs = CScript( [OP_CHECKSIG] * MAX_BLOCK_SIGOPS_PER_MB) block(19, spend=out[17], script=lots_of_checksigs, block_size=ONE_MEGABYTE) yield accepted() block(20, spend=out[18], script=lots_of_checksigs, block_size=ONE_MEGABYTE, extra_sigops=1) yield rejected(RejectResult(16, b'bad-blk-sigops')) # Rewind bad block tip(19) # Accept 40k sigops per block > 1MB and <= 2MB block(21, spend=out[18], script=lots_of_checksigs, extra_sigops=MAX_BLOCK_SIGOPS_PER_MB, block_size=ONE_MEGABYTE + 1) yield accepted() # Accept 40k sigops per block > 1MB and <= 2MB block(22, spend=out[19], script=lots_of_checksigs, extra_sigops=MAX_BLOCK_SIGOPS_PER_MB, block_size=2 * ONE_MEGABYTE) yield accepted() # Reject more than 40k sigops per block > 1MB and <= 2MB. block(23, spend=out[20], script=lots_of_checksigs, extra_sigops=MAX_BLOCK_SIGOPS_PER_MB + 1, block_size=ONE_MEGABYTE + 1) yield rejected(RejectResult(16, b'bad-blk-sigops')) # Rewind bad block tip(22) # Reject more than 40k sigops per block > 1MB and <= 2MB. block(24, spend=out[20], script=lots_of_checksigs, extra_sigops=MAX_BLOCK_SIGOPS_PER_MB + 1, block_size=2 * ONE_MEGABYTE) yield rejected(RejectResult(16, b'bad-blk-sigops')) # Rewind bad block tip(22) # Accept 60k sigops per block > 2MB and <= 3MB block(25, spend=out[20], script=lots_of_checksigs, extra_sigops=2 * MAX_BLOCK_SIGOPS_PER_MB, block_size=2 * ONE_MEGABYTE + 1) yield accepted() # Accept 60k sigops per block > 2MB and <= 3MB block(26, spend=out[21], script=lots_of_checksigs, extra_sigops=2 * MAX_BLOCK_SIGOPS_PER_MB, block_size=3 * ONE_MEGABYTE) yield accepted() # Reject more than 40k sigops per block > 1MB and <= 2MB. block(27, spend=out[22], script=lots_of_checksigs, extra_sigops=2 * MAX_BLOCK_SIGOPS_PER_MB + 1, block_size=2 * ONE_MEGABYTE + 1) yield rejected(RejectResult(16, b'bad-blk-sigops')) # Rewind bad block tip(26) # Reject more than 40k sigops per block > 1MB and <= 2MB. block(28, spend=out[22], script=lots_of_checksigs, extra_sigops=2 * MAX_BLOCK_SIGOPS_PER_MB + 1, block_size=3 * ONE_MEGABYTE) yield rejected(RejectResult(16, b'bad-blk-sigops')) # Rewind bad block tip(26) # Too many sigops in one txn too_many_tx_checksigs = CScript( [OP_CHECKSIG] * (MAX_BLOCK_SIGOPS_PER_MB + 1)) block( 29, spend=out[22], script=too_many_tx_checksigs, block_size=ONE_MEGABYTE + 1) yield rejected(RejectResult(16, b'bad-txn-sigops')) # Rewind bad block tip(26) # Generate a key pair to test P2SH sigops count private_key = CECKey() private_key.set_secretbytes(b"fatstacks") public_key = private_key.get_pubkey() # P2SH # Build the redeem script, hash it, use hash to create the p2sh script redeem_script = CScript( [public_key] + [OP_2DUP, OP_CHECKSIGVERIFY] * 5 + [OP_CHECKSIG]) redeem_script_hash = hash160(redeem_script) p2sh_script = CScript([OP_HASH160, redeem_script_hash, OP_EQUAL]) # Create a p2sh transaction p2sh_tx = self.create_tx(out[22], 1, p2sh_script) # Add the transaction to the block block(30) update_block(30, [p2sh_tx]) yield accepted() # Creates a new transaction using the p2sh transaction included in the # last block def spend_p2sh_tx(output_script=CScript([OP_TRUE])): # Create the transaction spent_p2sh_tx = CTransaction() spent_p2sh_tx.vin.append(CTxIn(COutPoint(p2sh_tx.sha256, 0), b'')) spent_p2sh_tx.vout.append(CTxOut(1, output_script)) # Sign the transaction using the redeem script sighash = SignatureHashForkId( redeem_script, spent_p2sh_tx, 0, SIGHASH_ALL | SIGHASH_FORKID, p2sh_tx.vout[0].nValue) sig = private_key.sign(sighash) + \ bytes(bytearray([SIGHASH_ALL | SIGHASH_FORKID])) spent_p2sh_tx.vin[0].scriptSig = CScript([sig, redeem_script]) spent_p2sh_tx.rehash() return spent_p2sh_tx # Sigops p2sh limit p2sh_sigops_limit = MAX_BLOCK_SIGOPS_PER_MB - \ redeem_script.GetSigOpCount(True) # Too many sigops in one p2sh txn too_many_p2sh_sigops = CScript([OP_CHECKSIG] * (p2sh_sigops_limit + 1)) block(31, spend=out[23], block_size=ONE_MEGABYTE + 1) update_block(31, [spend_p2sh_tx(too_many_p2sh_sigops)]) yield rejected(RejectResult(16, b'bad-txn-sigops')) # Rewind bad block tip(30) # Max sigops in one p2sh txn max_p2sh_sigops = CScript([OP_CHECKSIG] * (p2sh_sigops_limit)) block(32, spend=out[23], block_size=ONE_MEGABYTE + 1) update_block(32, [spend_p2sh_tx(max_p2sh_sigops)]) yield accepted() + # Submit a very large block via RPC + large_block = block( + 33, spend=out[24], block_size=self.excessive_block_size) + node.submitblock(ToHex(large_block)) + if __name__ == '__main__': FullBlockTest().main()