diff --git a/src/txmempool.h b/src/txmempool.h --- a/src/txmempool.h +++ b/src/txmempool.h @@ -610,6 +610,9 @@ void removeForBlock(const std::vector &vtx, unsigned int nBlockHeight); + // clear but then return a vector of all transactions + std::vector takeAll(); + void clear(); // lock free void _clear(); @@ -920,6 +923,8 @@ queuedTx.get().erase(entry); } + bool isEmpty() { return queuedTx.empty(); } + void clear() { cachedInnerUsage = 0; queuedTx.clear(); diff --git a/src/txmempool.cpp b/src/txmempool.cpp --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -930,6 +930,21 @@ return ret; } +std::vector CTxMemPool::takeAll() { + std::vector ret; + LOCK(cs); + + ret.reserve(mapTx.size()); + for (indexed_transaction_set::const_iterator it = mapTx.begin(); + it != mapTx.end(); it++) { + ret.push_back(it->GetSharedTx()); + } + + _clear(); + + return ret; +} + CTransactionRef CTxMemPool::get(const uint256 &txid) const { LOCK(cs); indexed_transaction_set::const_iterator i = mapTx.find(txid); diff --git a/src/validation.cpp b/src/validation.cpp --- a/src/validation.cpp +++ b/src/validation.cpp @@ -310,6 +310,11 @@ return IsMagneticAnomalyEnabled(config, chainActive.Tip()); } +static bool IsGreatWallEnabledForCurrentBlock(const Config &config) { + AssertLockHeld(cs_main); + return IsGreatWallEnabled(config, chainActive.Tip()); +} + // Command-line argument "-replayprotectionactivationtime=" will // cause the node to switch to replay protected SigHash ForkID value when the // median timestamp of the previous 11 blocks is greater than or equal to @@ -635,6 +640,10 @@ extraFlags |= SCRIPT_ENABLE_CHECKDATASIG; } + if (IsGreatWallEnabledForCurrentBlock(config)) { + extraFlags |= SCRIPT_ENABLE_SCHNORR; + } + // Check inputs based on the set of flags we activate. uint32_t scriptVerifyFlags = STANDARD_SCRIPT_VERIFY_FLAGS; if (!config.GetChainParams().RequireStandard()) { @@ -1204,6 +1213,24 @@ } } + // We also, regardless, need to check whether the transaction would + // be valid on the other side of the upgrade, so as to avoid + // splitting the network between upgraded and non-upgraded nodes. + // Note that this will create strange error messages like + // "non-mandatory-script-verify-flag (Non-canonical DER signature)" + // -- the tx was refused entry due to STRICTENC, a mandatory flag, + // but after the upgrade the signature would have been interpreted + // as valid Schnorr and thus STRICTENC would not happen. + CScriptCheck check3(scriptPubKey, amount, tx, i, + mandatoryFlags ^ SCRIPT_ENABLE_SCHNORR, + sigCacheStore, txdata); + if (check3()) { + return state.Invalid( + false, REJECT_NONSTANDARD, + strprintf("non-mandatory-script-verify-flag (%s)", + ScriptErrorString(check.GetScriptError()))); + } + // Failures of other flags indicate a transaction that is invalid in // new blocks, e.g. a invalid P2SH. We DoS ban such nodes as they // are not following the protocol. That said during an upgrade @@ -1520,6 +1547,10 @@ // Returns the script flags which should be checked for a given block static uint32_t GetBlockScriptFlags(const Config &config, const CBlockIndex *pChainTip) { + if (pChainTip == nullptr) { + return SCRIPT_VERIFY_NONE; + } + AssertLockHeld(cs_main); const Consensus::Params &consensusParams = config.GetChainParams().GetConsensus(); @@ -1571,6 +1602,14 @@ flags |= SCRIPT_VERIFY_CLEANSTACK; } + // When the great wall fork is enabled, we start accepting 65/64-byte + // Schnorr signatures in CHECKSIG and CHECKDATASIG respectively, and their + // verify variants. We also stop accepting 65 byte signatures in + // CHECKMULTISIG and its verify variant. + if (IsGreatWallEnabled(config, pChainTip)) { + flags |= SCRIPT_ENABLE_SCHNORR; + } + // We make sure this node will have replay protection during the next hard // fork. if (IsReplayProtectionEnabled(config, pChainTip)) { @@ -2201,6 +2240,21 @@ return false; } + if (GetBlockScriptFlags(config, pindexDelete) != + GetBlockScriptFlags(config, pindexDelete->pprev)) { + // We are de-activating a consensus upgrade. To be safe, we need to + // re-assess the entire mempool in case some transactions have now + // become invalid under the old rules. (Since this is a reorg, we will + // very likely land on another new post-upgrade tip that has the same + // rules. But this is not guaranteed.) + LogPrint(BCLog::MEMPOOL, + "Re-validating mempool for reorg that includes a possible " + "consensus rule downgrade"); + if (disconnectpool) { + disconnectpool->addForBlock(g_mempool.takeAll()); + } + } + // If this block was deactivating the replay protection, then we need to // remove transactions that are replay protected from the mempool. There is // no easy way to do this so we'll just discard the whole mempool and then @@ -2483,6 +2537,17 @@ // Remove conflicting transactions from the mempool.; g_mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); disconnectpool.removeForBlock(blockConnecting.vtx); + + if (GetBlockScriptFlags(config, pindexNew) != + GetBlockScriptFlags(config, pindexNew->pprev)) { + // We are activating a consensus upgrade. To be safe, we need to + // re-assess the entire mempool in case some transactions have now + // become invalid under the new rules. + LogPrint(BCLog::MEMPOOL, + "Re-validating mempool for consensus rule upgrade"); + disconnectpool.addForBlock(g_mempool.takeAll()); + } + // Update chainActive & related variables. UpdateTip(config, pindexNew); @@ -2751,9 +2816,10 @@ } } - if (fBlocksDisconnected) { - // If any blocks were disconnected, disconnectpool may be non empty. Add - // any disconnected transactions back to the mempool. + if (fBlocksDisconnected || !disconnectpool.isEmpty()) { + // If any blocks were disconnected, we need to update the mempool even + // if disconnectpool is empty. The disconnectpool may also be non-empty + // if there was a rule-restricting consensus upgrade. disconnectpool.updateMempoolForReorg(config, true); } diff --git a/test/functional/abc-schnorr-checksig-activation.py b/test/functional/abc-schnorr-checksig-activation.py new file mode 100755 --- /dev/null +++ b/test/functional/abc-schnorr-checksig-activation.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2017 The Bitcoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +This tests the activation of Schnorr transaction signatures: +- rejection prior to upgrade both in mempool and blocks. +- acceptance after upgrade both in mempool and blocks. +- advance and rewind mempool drop tests. (note: advance test requires + a temporary patch to bitcoind; see fakeDER comment below) + +Derived from abc-replay-protection.py +""" + +from test_framework.test_framework import ComparisonTestFramework +from test_framework.util import assert_equal, assert_raises_rpc_error +from test_framework.comptool import TestManager, TestInstance, RejectResult +from test_framework.blocktools import * +import time +from test_framework.key import CECKey +from test_framework.script import * + +# far into the future +GREAT_WALL_START_TIME = 2000000000 + +# If we don't do this, autoreplay protection will activate simultaneous with +# great_wall and schnorr sigs will mysteriously fail. +REPLAY_PROTECTION_START_TIME = GREAT_WALL_START_TIME*2 + +# Error due to passing a Schnorr signature to mempool before upgrade, but it will be valid after. +RPC_PREMATURE_SCHNORR_ERROR = '64: non-mandatory-script-verify-flag (Non-canonical DER signature)' +# Error due to passing a Schnorr signature to mempool before upgrade, but it will still be INVALID after. +RPC_BAD_SCHNORR_ERROR = '16: mandatory-script-verify-flag-failed (Non-canonical DER signature)' + +# Error due to passing Schnorr into multisig after upgrade -- it would have been invalid before as well. +RPC_BAD_MULTISIG_ERROR = '16: mandatory-script-verify-flag-failed (Signature cannot be 65 bytes in CHECKMULTISIG)' + +# Error due to passing a 64-byte DER CHECKSIG to mempool after upgrade, but it would have been valid before. +RPC_LATE_64ECDSA_CHECKSIG_ERROR = '64: non-mandatory-script-verify-flag (Signature must be zero for failed CHECK(MULTI)SIG operation)' +# Error due to passing a 64-byte DER CHECKMULTISIG to mempool after upgrade, but it would have been valid before. +RPC_LATE_64ECDSA_CHECKMULTISIG_ERROR = '64: non-mandatory-script-verify-flag (Signature cannot be 65 bytes in CHECKMULTISIG)' + +# For normal test running: +fakeDER = b'' + +# To properly test activation, we need to make txes with 64 byte ECDSA sigs. +# The easiest way to do this is to fake them, and then temporarily modify +# VerifySignature in src/script/interpreter.cpp to always `return true;` +# for ECDSA sigs, instead of `return pubkey.VerifyECDSA(sighash, vchSig);` +# Once that patch is done, you can uncomment the following and tests should +# pass. +# fakeDER = bytes.fromhex('303e021d44444444444444444444444444444444444444444' +# '44444444444444444021d4444444444444444444444444444' +# '444444444444444444444444444444') + +assert len(fakeDER) in [0, 64] + + +class PreviousSpendableOutput(object): + + def __init__(self, tx=CTransaction(), n=-1): + self.tx = tx + self.n = n # the output we're spending + + +class SchnorrActivationTest(ComparisonTestFramework): + + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + self.block_heights = {} + self.tip = None + self.blocks = {} + self.extra_args = [['-whitelist=127.0.0.1', + "-greatwallactivationtime=%d" % GREAT_WALL_START_TIME, + "-replayprotectionactivationtime=%d" % REPLAY_PROTECTION_START_TIME]] + + def run_test(self): + self.test = TestManager(self, self.options.tmpdir) + self.test.add_all_connections(self.nodes) + network_thread_start() + self.nodes[0].setmocktime(GREAT_WALL_START_TIME) + self.test.run() + + def next_block(self, number): + if self.tip == None: + base_block_hash = self.genesis_hash + block_time = int(time.time()) + 1 + else: + base_block_hash = self.tip.sha256 + block_time = self.tip.nTime + 1 + # First create the coinbase + height = self.block_heights[base_block_hash] + 1 + coinbase = create_coinbase(height) + coinbase.rehash() + block = create_block(base_block_hash, coinbase, block_time) + + # Do PoW, which is cheap on regnet + block.solve() + self.tip = block + self.block_heights[block.sha256] = height + assert number not in self.blocks + self.blocks[number] = block + return block + + def get_tests(self): + self.genesis_hash = int(self.nodes[0].getbestblockhash(), 16) + self.block_heights[self.genesis_hash] = 0 + spendable_outputs = [] + + # save the current tip so its coinbase can be spent by a later block + def save_spendable_output(): + spendable_outputs.append(self.tip) + + # get a coinbase that we previously marked as spendable + def get_spendable_output(): + return PreviousSpendableOutput(spendable_outputs.pop(0).vtx[0], 0) + + # returns a test case that asserts that the current tip was accepted + def accepted(): + return TestInstance([[self.tip, True]]) + + # returns a test case that asserts that the current tip was rejected + def rejected(reject=None): + if reject is None: + return TestInstance([[self.tip, False]]) + else: + return TestInstance([[self.tip, reject]]) + + # move the tip back to a previous block + def tip(number): + self.tip = self.blocks[number] + + # adds transactions to the block and updates state + def update_block(block_number, new_transactions): + [tx.rehash() for tx in new_transactions] + block = self.blocks[block_number] + block.vtx.extend(new_transactions) + old_sha256 = block.sha256 + make_conform_to_ctor(block) + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + # Update the internal state just like in next_block + self.tip = block + if block.sha256 != old_sha256: + self.block_heights[ + block.sha256] = self.block_heights[old_sha256] + del self.block_heights[old_sha256] + self.blocks[block_number] = block + return block + + # shorthand for functions + block = self.next_block + node = self.nodes[0] + + # Create a new block + block(0) + save_spendable_output() + yield accepted() + + # Now we need that block to mature so we can spend the coinbase. + test = TestInstance(sync_every_block=False) + for i in range(199): + block(5000 + i) + test.blocks_and_transactions.append([self.tip, True]) + save_spendable_output() + yield test + + # collect spendable outputs now to avoid cluttering the code later on + out = [] + for i in range(100): + out.append(get_spendable_output()) + + # Generate a key pair to test P2SH sigops count + private_key = CECKey() + private_key.set_secretbytes(b"Schnorr!"*4) + public_key = private_key.get_pubkey() # uncompressed + + def create_fund_and_spend_tx(spend, multi=False, sig='schnorr'): + if multi: + script = CScript([OP_1, public_key, OP_1, OP_CHECKMULTISIG]) + else: + script = CScript([public_key, OP_CHECKSIG]) + + # Fund transaction + txfund = create_transaction( + spend.tx, spend.n, b'', 50 * COIN, script) + txfund.rehash() + + # Spend transaction + schnorrchecksigtx = CTransaction() + schnorrchecksigtx.vout.append( + CTxOut(50 * COIN - 1000, CScript([OP_TRUE]))) + schnorrchecksigtx.vin.append( + CTxIn(COutPoint(txfund.sha256, 0), b'')) + + # Sign the transaction + sighashtype = SIGHASH_ALL | SIGHASH_FORKID + hashbyte = bytes([sighashtype & 0xff]) + sighash = SignatureHashForkId( + script, schnorrchecksigtx, 0, sighashtype, 50 * COIN) + if sig == 'schnorr': + txsig = private_key.sign_schnorr(sighash) + hashbyte + elif sig == 'ecdsa': + txsig = private_key.sign(sighash) + hashbyte + elif isinstance(sig, bytes): + txsig = sig + hashbyte + if multi: + schnorrchecksigtx.vin[0].scriptSig = CScript([b'', txsig]) + else: + schnorrchecksigtx.vin[0].scriptSig = CScript([txsig]) + schnorrchecksigtx.rehash() + + return txfund, schnorrchecksigtx + + def send_transaction_to_mempool(tx): + tx_id = node.sendrawtransaction(ToHex(tx)) + assert(tx_id in set(node.getrawmempool())) + return tx_id + + # Setup fundings + fundings = [] + _, schnorrchecksigtx = create_fund_and_spend_tx(out[0]) + fundings.append(_) + _, schnorrmultisigtx = create_fund_and_spend_tx(out[1], multi=True) + fundings.append(_) + _, ecdsachecksigtx = create_fund_and_spend_tx(out[2], sig='ecdsa') + fundings.append(_) + if fakeDER: + _, DER64checksigtx = create_fund_and_spend_tx(out[5], sig=fakeDER) + fundings.append(_) + _, DER64multisigtx = create_fund_and_spend_tx( + out[6], multi=True, sig=fakeDER) + fundings.append(_) + + for fund in fundings: + send_transaction_to_mempool(fund) + block(1) + update_block(1, fundings) + yield accepted() + + # We are before the upgrade, no Schnorrs get in the mempool. + assert_raises_rpc_error(-26, RPC_PREMATURE_SCHNORR_ERROR, + node.sendrawtransaction, ToHex(schnorrchecksigtx)) + assert_raises_rpc_error(-26, RPC_BAD_SCHNORR_ERROR, + node.sendrawtransaction, ToHex(schnorrmultisigtx)) + + # And blocks containing them are rejected as well. + block(2) + update_block(2, [schnorrchecksigtx]) + yield rejected(RejectResult(16, b'blk-bad-inputs')) + # Rewind bad block + tip(1) + + block(3) + update_block(3, [schnorrmultisigtx]) + yield rejected(RejectResult(16, b'blk-bad-inputs')) + # Rewind bad block + tip(1) + + # Create a block that would activate the schnorr + bfork = block(5555) + bfork.nTime = GREAT_WALL_START_TIME - 1 + update_block(5555, []) + yield accepted() + + for i in range(5): + block(5200 + i) + test.blocks_and_transactions.append([self.tip, True]) + yield test + + # Check we are just before the activation time + assert_equal(node.getblockheader(node.getbestblockhash())['mediantime'], + GREAT_WALL_START_TIME - 1) + + # We are just before the upgrade, still no Schnorrs get in the mempool, + assert_raises_rpc_error(-26, RPC_PREMATURE_SCHNORR_ERROR, + node.sendrawtransaction, ToHex(schnorrchecksigtx)) + assert_raises_rpc_error(-26, RPC_BAD_SCHNORR_ERROR, + node.sendrawtransaction, ToHex(schnorrmultisigtx)) + # ... nor in blocks. + block(10) + update_block(10, [schnorrchecksigtx]) + yield rejected(RejectResult(16, b'blk-bad-inputs')) + # Rewind bad block + tip(5204) + block(11) + update_block(11, [schnorrmultisigtx]) + yield rejected(RejectResult(16, b'blk-bad-inputs')) + # Rewind bad block + tip(5204) + + if fakeDER: + # Throw a couple of "valid" 65-byte ECDSA signatures into the + # mempool just prior to the activation. + faked_checksig_tx_id = send_transaction_to_mempool(DER64checksigtx) + faked_multisig_tx_id = send_transaction_to_mempool(DER64multisigtx) + + # Put a proper ECDSA transaction into the mempool but it won't + # be mined... + ecdsa_tx_id = send_transaction_to_mempool(ecdsachecksigtx) + + # Activate the Schnorr! + block(5556) + yield accepted() + + # Make sure ECDSA is still in -- we don't want to lose uninvolved txns + # when the upgrade happens. + assert ecdsa_tx_id in set(node.getrawmempool()) + + if fakeDER: + # The 64-byte DER sigs must be ejected. + assert faked_checksig_tx_id not in set(node.getrawmempool()) + assert faked_multisig_tx_id not in set(node.getrawmempool()) + + # If we try to re-add them, they fail with non-mandatory errors. + # In CHECKSIG it's invalid Schnorr and hence NULLFAIL. + assert_raises_rpc_error(-26, RPC_LATE_64ECDSA_CHECKSIG_ERROR, + node.sendrawtransaction, ToHex(DER64checksigtx)) + # In CHECKMULTISIG it's invalid length and hence BAD_LENGTH. + assert_raises_rpc_error(-26, RPC_LATE_64ECDSA_CHECKMULTISIG_ERROR, + node.sendrawtransaction, ToHex(DER64multisigtx)) + # And they can't be mined either... + block(14) + update_block(14, [DER64checksigtx]) + yield rejected(RejectResult(16, b'blk-bad-inputs')) + # Rewind bad block + tip(5556) + block(15) + update_block(15, [DER64multisigtx]) + yield rejected(RejectResult(16, b'blk-bad-inputs')) + # Rewind bad block + tip(5556) + + # The multisig throws a different error now + assert_raises_rpc_error(-26, RPC_BAD_MULTISIG_ERROR, + node.sendrawtransaction, ToHex(schnorrmultisigtx)) + # And it still can't be mined + block(16) + update_block(16, [schnorrmultisigtx]) + yield rejected(RejectResult(16, b'blk-bad-inputs')) + # Rewind bad block + tip(5556) + + # The Schnorr CHECKSIG is now valid + schnorr_tx_id = send_transaction_to_mempool(schnorrchecksigtx) + # It can also be mined + block(21) + update_block(21, [schnorrchecksigtx, ecdsachecksigtx]) + yield accepted() + # (we mined the ecdsa tx too) + assert schnorr_tx_id not in set(node.getrawmempool()) + assert ecdsa_tx_id not in set(node.getrawmempool()) + + # Ok, now we check if a reorg work properly accross the activation. + postforkblockid = node.getbestblockhash() + node.invalidateblock(postforkblockid) + # txes popped back into mempool + assert schnorr_tx_id in set(node.getrawmempool()) + assert ecdsa_tx_id in set(node.getrawmempool()) + + # Deactivating upgrade. + forkblockid = node.getbestblockhash() + node.invalidateblock(forkblockid) + # This should kick out the Schnorr sig, but not the valid ECDSA sig. + assert schnorr_tx_id not in set(node.getrawmempool()) + assert ecdsa_tx_id in set(node.getrawmempool()) + + # Check that we also do it properly on deeper reorg. + node.reconsiderblock(forkblockid) + node.reconsiderblock(postforkblockid) + node.invalidateblock(forkblockid) + assert schnorr_tx_id not in set(node.getrawmempool()) + assert ecdsa_tx_id in set(node.getrawmempool()) + + +if __name__ == '__main__': + SchnorrActivationTest().main()