diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -159,6 +159,7 @@ 'abc-cmdline.py', 'abc-p2p-fullblocktest.py', 'abc-rpc.py', + 'abc-ec.py', 'mempool-accept-txn.py', ] if ENABLE_ZMQ: diff --git a/qa/rpc-tests/abc-ec.py b/qa/rpc-tests/abc-ec.py new file mode 100755 --- /dev/null +++ b/qa/rpc-tests/abc-ec.py @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2017 The Bitcoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +This test checks the new consensus behavior. +It is derived from the much more complex p2p-fullblocktest. +""" + +from test_framework.test_framework import ComparisonTestFramework +from test_framework.util import * +from test_framework.comptool import TestManager, TestInstance, RejectResult +from test_framework.blocktools import * +import time +from test_framework.key import CECKey +from test_framework.script import * +from test_framework.cdefs import (ONE_MEGABYTE, LEGACY_MAX_BLOCK_SIZE, + MAX_BLOCK_SIGOPS_PER_MB, MAX_TX_SIGOPS_COUNT) + +import unittest + +# far into the past +UAHF_START_TIME = 30000000 + + +class PreviousSpendableOutput(object): + + def __init__(self, tx=CTransaction(), n=-1): + self.tx = tx + self.n = n # the output we're spending + +# TestNode: A peer we use to send messages to bitcoind, and store responses. + + +class TestNode(SingleNodeConnCB): + + def __init__(self): + self.last_sendcmpct = None + self.last_cmpctblock = None + self.last_getheaders = None + self.last_headers = None + SingleNodeConnCB.__init__(self) + + def on_sendcmpct(self, conn, message): + self.last_sendcmpct = message + + def on_cmpctblock(self, conn, message): + self.last_cmpctblock = message + self.last_cmpctblock.header_and_shortids.header.calc_sha256() + + def on_getheaders(self, conn, message): + self.last_getheaders = message + + def on_headers(self, conn, message): + self.last_headers = message + for x in self.last_headers.headers: + x.calc_sha256() + + def clear_block_data(self): + with mininode_lock: + self.last_sendcmpct = None + self.last_cmpctblock = None + + +class FullBlockTest(ComparisonTestFramework): + + # Can either run this test as 1 node with expected answers, or two and compare them. + # Change the "outcome" variable from each TestInstance object to only do + # the comparison. + + def __init__(self): + super().__init__() + self.excessive_block_size_8 = 8 * ONE_MEGABYTE + self.excessive_block_size_16 = 16 * ONE_MEGABYTE + self.num_nodes = 4 + self.block_heights = {} + self.coinbase_key = CECKey() + self.coinbase_key.set_secretbytes(b"fatstacks") + self.coinbase_pubkey = self.coinbase_key.get_pubkey() + self.tip = None + self.blocks = {} + + def sync_all(self): + if self.is_network_split: + sync_blocks(self.nodes[:2], timeout=15) + sync_blocks(self.nodes[2:], timeout=15) + sync_mempools(self.nodes[:2]) + sync_mempools(self.nodes[2:]) + else: + sync_blocks(self.nodes) + sync_mempools(self.nodes) + + def setup_network(self): + self.is_network_split = True + self.array_opts = ['-debug', + '-norelaypriority', + '-whitelist=127.0.0.1', + '-limitancestorcount=9999', + '-limitancestorsize=9999', + '-limitdescendantcount=9999', + '-limitdescendantsize=9999', + '-maxmempool=999', + "-uahfstarttime=%d" % UAHF_START_TIME, + "-excessiveblocksize=%d" + % self.excessive_block_size_8] + + self.extra_args = [ + self.array_opts, self.array_opts, self.array_opts, self.array_opts] + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, + self.extra_args, + binary=[self.options.testbinary, self.options.testbinary, self.options.testbinary, self.options.testbinary]) + + connect_nodes_bi(self.nodes, 0, 1) + connect_nodes_bi(self.nodes, 2, 3) + self.sync_all() + + def add_options(self, parser): + super().add_options(parser) + parser.add_option( + "--runbarelyexpensive", dest="runbarelyexpensive", default=True) + + def run_test(self): + self.test = TestManager(self, self.options.tmpdir) + self.test.add_all_connections(self.nodes) + # Start up network handling in another thread + NetworkThread().start() + # Set the blocksize to 16MB as initial condition + self.nodes[0].setexcessiveblock(self.excessive_block_size_16) + self.test.run() + + def add_transactions_to_block(self, block, tx_list): + [tx.rehash() for tx in tx_list] + block.vtx.extend(tx_list) + + # this is a little handier to use than the version in blocktools.py + def create_tx(self, spend_tx, n, value, script=CScript([OP_TRUE])): + tx = create_transaction(spend_tx, n, b"", value, script) + return tx + + # sign a transaction, using the key we know about + # this signs input 0 in tx, which is assumed to be spending output n in + # spend_tx + def sign_tx(self, tx, spend_tx, n): + scriptPubKey = bytearray(spend_tx.vout[n].scriptPubKey) + if (scriptPubKey[0] == OP_TRUE): # an anyone-can-spend + tx.vin[0].scriptSig = CScript() + return + sighash = SignatureHashForkId( + spend_tx.vout[n].scriptPubKey, tx, 0, SIGHASH_ALL | SIGHASH_FORKID, spend_tx.vout[n].nValue) + tx.vin[0].scriptSig = CScript( + [self.coinbase_key.sign(sighash) + bytes(bytearray([SIGHASH_ALL | SIGHASH_FORKID]))]) + + def create_and_sign_transaction(self, spend_tx, n, value, script=CScript([OP_TRUE])): + tx = self.create_tx(spend_tx, n, value, script) + self.sign_tx(tx, spend_tx, n) + tx.rehash() + return tx + + def next_block(self, number, spend=None, additional_coinbase_value=0, script=None, extra_sigops=0, block_size=0, solve=True, submit=True, base_hash=None, base_time=None, base_height=None): + """ + Create a block on top of self.tip, and advance self.tip to point to the new block + if spend is specified, then 1 satoshi will be spent from that to an anyone-can-spend + output, and rest will go to fees. + """ + if self.tip == None: + base_block_hash = self.genesis_hash + block_time = int(time.time()) + 1 + else: + if base_hash == None and base_time == None: + base_block_hash = self.tip.sha256 + block_time = self.tip.nTime + 1 + else: + base_block_hash = base_hash + block_time = base_time + # First create the coinbase + if base_height == None: + height = self.block_heights[base_block_hash] + 1 + else: + height = base_height + coinbase = create_coinbase(height, self.coinbase_pubkey) + coinbase.vout[0].nValue += additional_coinbase_value + if (spend != None): + coinbase.vout[0].nValue += spend.tx.vout[ + spend.n].nValue - 1 # all but one satoshi to fees + coinbase.rehash() + block = create_block(base_block_hash, coinbase, block_time) + spendable_output = None + if (spend != None): + tx = CTransaction() + # no signature yet + tx.vin.append( + CTxIn(COutPoint(spend.tx.sha256, spend.n), b"", 0xffffffff)) + # We put some random data into the first transaction of the chain + # to randomize ids + tx.vout.append( + CTxOut(0, CScript([random.randint(0, 255), OP_DROP, OP_TRUE]))) + if script == None: + tx.vout.append(CTxOut(1, CScript([OP_TRUE]))) + else: + tx.vout.append(CTxOut(1, script)) + spendable_output = PreviousSpendableOutput(tx, 0) + + # Now sign it if necessary + scriptSig = b"" + scriptPubKey = bytearray(spend.tx.vout[spend.n].scriptPubKey) + if (scriptPubKey[0] == OP_TRUE): # looks like an anyone-can-spend + scriptSig = CScript([OP_TRUE]) + else: + # We have to actually sign it + sighash = SignatureHashForkId( + spend.tx.vout[spend.n].scriptPubKey, tx, 0, SIGHASH_ALL | SIGHASH_FORKID, spend.tx.vout[spend.n].nValue) + scriptSig = CScript( + [self.coinbase_key.sign(sighash) + bytes(bytearray([SIGHASH_ALL | SIGHASH_FORKID]))]) + tx.vin[0].scriptSig = scriptSig + # Now add the transaction to the block + self.add_transactions_to_block(block, [tx]) + block.hashMerkleRoot = block.calc_merkle_root() + if spendable_output != None and block_size > 0: + while len(block.serialize()) < block_size: + tx = CTransaction() + script_length = block_size - len(block.serialize()) - 79 + if script_length > 510000: + script_length = 500000 + tx_sigops = min( + extra_sigops, script_length, MAX_TX_SIGOPS_COUNT) + extra_sigops -= tx_sigops + script_pad_len = script_length - tx_sigops + script_output = CScript( + [b'\x00' * script_pad_len] + [OP_CHECKSIG] * tx_sigops) + tx.vout.append(CTxOut(0, CScript([OP_TRUE]))) + tx.vout.append(CTxOut(0, script_output)) + tx.vin.append( + CTxIn(COutPoint(spendable_output.tx.sha256, spendable_output.n))) + spendable_output = PreviousSpendableOutput(tx, 0) + self.add_transactions_to_block(block, [tx]) + block.hashMerkleRoot = block.calc_merkle_root() + # Make sure the math above worked out to produce the correct block size + # (the math will fail if there are too many transactions in the block) + assert_equal(len(block.serialize()), block_size) + # Make sure all the requested sigops have been included + assert_equal(extra_sigops, 0) + if solve: + block.solve() + if submit: + self.tip = block + self.block_heights[block.sha256] = height + assert number not in self.blocks + self.blocks[number] = block + return block + + def get_tests(self): + self.genesis_hash = int(self.nodes[0].getbestblockhash(), 16) + self.block_heights[self.genesis_hash] = 0 + spendable_outputs = [] + + # save the current tip so it can be spent by a later block + def save_spendable_output(): + spendable_outputs.append(self.tip) + + # get an output that we previously marked as spendable + def get_spendable_output(): + return PreviousSpendableOutput(spendable_outputs.pop(0).vtx[0], 0) + + # returns a test case that asserts that the current tip was accepted + def accepted(): + return TestInstance([[self.tip, True]]) + + # returns a test case that asserts that the current tip was rejected + def rejected(reject=None): + if reject is None: + return TestInstance([[self.tip, False]]) + else: + return TestInstance([[self.tip, reject]]) + + # move the tip back to a previous block + def tip(number): + self.tip = self.blocks[number] + + # adds transactions to the block and updates state + def update_block(block_number, new_transactions): + block = self.blocks[block_number] + self.add_transactions_to_block(block, new_transactions) + old_sha256 = block.sha256 + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + # Update the internal state just like in next_block + self.tip = block + if block.sha256 != old_sha256: + self.block_heights[ + block.sha256] = self.block_heights[old_sha256] + del self.block_heights[old_sha256] + self.blocks[block_number] = block + return block + + # shorthand for functions + block = self.next_block + + # Create a new block + block(0) + save_spendable_output() + yield accepted() + + # Now we need that block to mature so we can spend the coinbase. + test = TestInstance(sync_every_block=False) + for i in range(99): + block(5000 + i) + test.blocks_and_transactions.append([self.tip, True]) + save_spendable_output() + yield test + + # collect spendable outputs now to avoid cluttering the code later on + out = [] + for i in range(100): + out.append(get_spendable_output()) + + # block of maximal size + block(1, spend=out[0], block_size=self.excessive_block_size_8) + yield accepted() + + # Consensus code, the network is split: + # Node 0 and 1 are connected + # Node 2 and 3 are connected + + # Save current tip variables + base_block_hash = self.tip.sha256 + block_time = self.tip.nTime + 1 + block_height = self.block_heights[base_block_hash] + 1 + + # A block bigger than 16Mb should be rejected by all nodes + block(2, spend=out[1], block_size=self.excessive_block_size_16 + 1) + yield rejected(RejectResult(16, b'bad-blk-length')) + assert_equal(self.nodes[0].getblockcount(), 101) + assert_equal(self.nodes[1].getblockcount(), 101) + assert_equal(self.nodes[2].getblockcount(), 101) + assert_equal(self.nodes[3].getblockcount(), 101) + + # Create a block bigger than 8Mb. (Only node 0 can accept it because + # its excessive size is set to 16 Mb, the rest should mark it as + # invalid) + block_temp = block( + 3, spend=out[1], block_size=self.excessive_block_size_8 + 1, solve=False, submit=False, base_hash=base_block_hash, base_time=block_time, base_height=block_height) + block_temp.solve() + self.nodes[0].submitblock(ToHex(block_temp)) + # The block was submited + assert_equal(self.nodes[0].getblockcount(), 102) + + # Prepare to compare hashes + assert_equal(self.nodes[1].getblockcount(), 101) + big_block_hash = self.nodes[0].getbestblockhash() + node_1_last_hash = self.nodes[1].getbestblockhash() + try: + self.sync_all() + except AssertionError: + assert (True) + # Node 0 tip is a block bigger than 8mb and Node 1 tip is a not + # excessive block + assert_equal(self.nodes[0].getbestblockhash(), big_block_hash) + assert_equal(self.nodes[1].getbestblockhash(), node_1_last_hash) + + # Update the excessive size and resync + self.nodes[1].setexcessiveblock(self.excessive_block_size_16) + self.sync_all() + assert_equal(self.nodes[0].getbestblockhash(), big_block_hash) + assert_equal(self.nodes[1].getbestblockhash(), big_block_hash) + +if __name__ == '__main__': + FullBlockTest().main() diff --git a/src/chain.h b/src/chain.h --- a/src/chain.h +++ b/src/chain.h @@ -151,6 +151,12 @@ //!< descends from failed block BLOCK_FAILED_CHILD = 64, BLOCK_FAILED_MASK = BLOCK_FAILED_VALID | BLOCK_FAILED_CHILD, + + //!< excessive block + BLOCK_EXCESSIVE = 128, + + //!< mask for all the errors + BLOCK_NOT_VALID_MASK = BLOCK_EXCESSIVE | BLOCK_FAILED_MASK }; /** diff --git a/src/consensus/validation.h b/src/consensus/validation.h --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -22,9 +22,10 @@ class CValidationState { private: enum mode_state { - MODE_VALID, //!< everything ok - MODE_INVALID, //!< network rule violation (DoS value may be set) - MODE_ERROR, //!< run-time error + MODE_VALID, //!< everything ok + MODE_EXCESSIVE, //!< network soft rule violation + MODE_INVALID, //!< network rule violation (DoS value may be set) + MODE_ERROR, //!< run-time error } mode; int nDoS; std::string strRejectReason; @@ -53,6 +54,20 @@ return ret; } + // It works like the DoS function but setting the mode as EXCESSIVE + bool Excessive(int level, bool ret = false, unsigned int chRejectCodeIn = 0, + const std::string &strRejectReasonIn = "", + bool corruptionIn = false, + const std::string &strDebugMessageIn = "") { + ret = DoS(level, ret, chRejectCodeIn, strRejectReasonIn, corruptionIn, + strDebugMessageIn); + if (mode == MODE_ERROR) { + return ret; + } + mode = MODE_EXCESSIVE; + return ret; + } + bool Invalid(bool ret = false, unsigned int _chRejectCode = 0, const std::string &_strRejectReason = "", const std::string &_strDebugMessage = "") { @@ -69,6 +84,10 @@ } bool IsValid() const { return mode == MODE_VALID; } + bool IsNotValid() const { + return ((mode == MODE_EXCESSIVE) || (mode == MODE_INVALID)); + } + bool IsExcessive() const { return mode == MODE_EXCESSIVE; } bool IsInvalid() const { return mode == MODE_INVALID; } bool IsError() const { return mode == MODE_ERROR; } bool IsInvalid(int &nDoSOut) const { @@ -78,6 +97,13 @@ } return false; } + bool IsNotValid(int &nDoSOut) const { + if (IsNotValid()) { + nDoSOut = nDoS; + return true; + } + return false; + } bool CorruptionPossible() const { return corruptionPossible; } void SetCorruptionPossible() { corruptionPossible = true; } diff --git a/src/net_processing.cpp b/src/net_processing.cpp --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1015,7 +1015,7 @@ mapBlockSource.find(hash); int nDoS = 0; - if (state.IsInvalid(nDoS)) { + if (state.IsNotValid(nDoS)) { if (it != mapBlockSource.end() && State(it->second.first)) { // Blocks are never rejected with internal reject codes. assert(state.GetRejectCode() < REJECT_INTERNAL); diff --git a/src/test/blockcheck_tests.cpp b/src/test/blockcheck_tests.cpp --- a/src/test/blockcheck_tests.cpp +++ b/src/test/blockcheck_tests.cpp @@ -22,6 +22,7 @@ BOOST_CHECK_EQUAL(fValid, expected); BOOST_CHECK_EQUAL(fValid, state.IsValid()); + BOOST_CHECK_EQUAL(!fValid, state.IsNotValid()); } static void RunCheckOnBlock(const GlobalConfig &config, const CBlock &block) { @@ -38,6 +39,26 @@ BOOST_CHECK_EQUAL(state.GetRejectReason(), reason); } +// Excessive method tests, calling the `IsNotValid` function it's not enough +// because the blocks can be excessive or invalid +static void RunExessiveCheckOnBlock(const GlobalConfig &config, + const CBlock &block) { + CValidationState state; + RunCheckOnBlockImpl(config, block, state, true); + BOOST_CHECK_EQUAL(state.IsExcessive(), false); +} + +static void RunExessiveCheckOnBlock(const GlobalConfig &config, + const CBlock &block, + const std::string &reason) { + CValidationState state; + RunCheckOnBlockImpl(config, block, state, false); + + BOOST_CHECK_EQUAL(state.GetRejectCode(), REJECT_INVALID); + BOOST_CHECK_EQUAL(state.GetRejectReason(), reason); + BOOST_CHECK_EQUAL(state.IsExcessive(), true); +} + BOOST_AUTO_TEST_CASE(blockfail) { SelectParams(CBaseChainParams::MAIN); @@ -88,13 +109,13 @@ } // Check that at this point, we still accept the block. - RunCheckOnBlock(config, block); + RunExessiveCheckOnBlock(config, block); // But reject it with one more transaction as it goes over the maximum // allowed block size. tx.vin[0].prevout.hash = GetRandHash(); block.vtx.push_back(MakeTransactionRef(tx)); - RunCheckOnBlock(config, block, "bad-blk-length"); + RunExessiveCheckOnBlock(config, block, "bad-blk-length"); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/validation.cpp b/src/validation.cpp --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2506,7 +2506,7 @@ // which block files have been deleted. Remove those as candidates // for the most work chain if we come across them; we can't switch // to a chain unless we have all the non-active-chain parent blocks. - bool fFailedChain = pindexTest->nStatus & BLOCK_FAILED_MASK; + bool fFailedChain = pindexTest->nStatus & BLOCK_NOT_VALID_MASK; bool fMissingData = !(pindexTest->nStatus & BLOCK_HAVE_DATA); if (fFailedChain || fMissingData) { // Candidate chain is not usable (either invalid or missing @@ -3144,15 +3144,15 @@ // Bail early if there is no way this block is of reasonable size. if ((block.vtx.size() * MIN_TRANSACTION_SIZE) > nMaxBlockSize) { - return state.DoS(100, false, REJECT_INVALID, "bad-blk-length", false, - "size limits failed"); + return state.Excessive(10, false, REJECT_INVALID, "bad-blk-length", + false, "size limits failed"); } auto currentBlockSize = ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION); if (currentBlockSize > nMaxBlockSize) { - return state.DoS(100, false, REJECT_INVALID, "bad-blk-length", false, - "size limits failed"); + return state.Excessive(10, false, REJECT_INVALID, "bad-blk-length", + false, "size limits failed"); } // And a valid coinbase. @@ -3550,6 +3550,9 @@ if (state.IsInvalid() && !state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(pindex); + } else if (state.IsExcessive() && !state.CorruptionPossible()) { + pindex->nStatus |= BLOCK_EXCESSIVE; + setDirtyBlockIndex.insert(pindex); } return error("%s: %s (block %s)", __func__, FormatStateMessage(state), block.GetHash().ToString());