diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index c92d38a397..a6fe7d2a24 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -1,339 +1,340 @@ #!/usr/bin/env python3 # # linearize-data.py: Construct a linear, no-fork version of the chain. # # Copyright (c) 2013-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # from __future__ import print_function, division import struct import re import os import os.path import sys import hashlib import datetime import time from collections import namedtuple from binascii import unhexlify settings = {} def hex_switchEndian(s): """ Switches the endianness of a hex string (in pairs of hex chars) """ pairList = [s[i:i + 2].encode() for i in range(0, len(s), 2)] return b''.join(pairList[::-1]).decode() def uint32(x): return x & 0xffffffff def bytereverse(x): return uint32((((x) << 24) | (((x) << 8) & 0x00ff0000) | (((x) >> 8) & 0x0000ff00) | ((x) >> 24))) def bufreverse(in_buf): out_words = [] for i in range(0, len(in_buf), 4): word = struct.unpack('@I', in_buf[i:i + 4])[0] out_words.append(struct.pack('@I', bytereverse(word))) return b''.join(out_words) def wordreverse(in_buf): out_words = [] for i in range(0, len(in_buf), 4): out_words.append(in_buf[i:i + 4]) out_words.reverse() return b''.join(out_words) def calc_hdr_hash(blk_hdr): hash1 = hashlib.sha256() hash1.update(blk_hdr) hash1_o = hash1.digest() hash2 = hashlib.sha256() hash2.update(hash1_o) hash2_o = hash2.digest() return hash2_o def calc_hash_str(blk_hdr): hash = calc_hdr_hash(blk_hdr) hash = bufreverse(hash) hash = wordreverse(hash) hash_str = hash.hex() return hash_str def get_blk_dt(blk_hdr): members = struct.unpack(" self.maxOutSz): self.outF.close() if self.setFileTime: os.utime(self.outFname, (int(time.time()), self.highTS)) self.outF = None self.outFname = None self.outFn = self.outFn + 1 self.outsz = 0 (blkDate, blkTS) = get_blk_dt(blk_hdr) if self.timestampSplit and (blkDate > self.lastDate): - print("New month " + blkDate.strftime("%Y-%m") + " @ " + self.hash_str) + print("New month " + blkDate.strftime("%Y-%m") + + " @ " + self.hash_str) self.lastDate = blkDate if self.outF: self.outF.close() if self.setFileTime: os.utime(self.outFname, (int(time.time()), self.highTS)) self.outF = None self.outFname = None self.outFn = self.outFn + 1 self.outsz = 0 if not self.outF: if self.fileOutput: self.outFname = self.settings['output_file'] else: self.outFname = os.path.join( self.settings['output'], "blk{:05d}.dat".format(self.outFn)) print("Output file " + self.outFname) self.outF = open(self.outFname, "wb") self.outF.write(inhdr) self.outF.write(blk_hdr) self.outF.write(rawblock) self.outsz = self.outsz + len(inhdr) + len(blk_hdr) + len(rawblock) self.blkCountOut = self.blkCountOut + 1 if blkTS > self.highTS: self.highTS = blkTS if (self.blkCountOut % 1000) == 0: print('{} blocks scanned, {} blocks written (of {}, {:.1f}% complete)'.format( self.blkCountIn, self.blkCountOut, len(self.blkindex), 100.0 * self.blkCountOut / len(self.blkindex))) def inFileName(self, fn): return os.path.join(self.settings['input'], "blk{:05d}.dat".format(fn)) def fetchBlock(self, extent): '''Fetch block contents from disk given extents''' with open(self.inFileName(extent.fn), "rb") as f: f.seek(extent.offset) return f.read(extent.size) def copyOneBlock(self): '''Find the next block to be written in the input, and copy it to the output.''' extent = self.blockExtents.pop(self.blkCountOut) if self.blkCountOut in self.outOfOrderData: # If the data is cached, use it from memory and remove from the cache rawblock = self.outOfOrderData.pop(self.blkCountOut) self.outOfOrderSize -= len(rawblock) else: # Otherwise look up data on disk rawblock = self.fetchBlock(extent) self.writeBlock(extent.inhdr, extent.blkhdr, rawblock) def run(self): while self.blkCountOut < len(self.blkindex): if not self.inF: fname = self.inFileName(self.inFn) print("Input file " + fname) try: self.inF = open(fname, "rb") except IOError: print("Premature end of block data") return inhdr = self.inF.read(8) if (not inhdr or (inhdr[0] == "\0")): self.inF.close() self.inF = None self.inFn = self.inFn + 1 continue inMagic = inhdr[:4] if (inMagic != self.settings['netmagic']): print("Invalid magic: " + inMagic.hex()) return inLenLE = inhdr[4:] su = struct.unpack(" 0: result = chars[value % 58] + result value //= 58 while (str[:2] == '00'): result = chars[0] + result str = str[2:] return result # TODO: def base58_decode def keyhash_to_p2pkh(hash, main=False): assert (len(hash) == 20) version = 0 if main else 111 return byte_to_base58(hash, version) def scripthash_to_p2sh(hash, main=False): assert (len(hash) == 20) version = 5 if main else 196 return byte_to_base58(hash, version) def key_to_p2pkh(key, main=False): key = check_key(key) return keyhash_to_p2pkh(hash160(key), main) def script_to_p2sh(script, main=False): script = check_script(script) return scripthash_to_p2sh(hash160(script), main) def check_key(key): - if (type(key) is str): + if (isinstance(key, str)): key = hex_str_to_bytes(key) # Assuming this is hex string - if (type(key) is bytes and (len(key) == 33 or len(key) == 65)): + if (isinstance(key, bytes) and (len(key) == 33 or len(key) == 65)): return key assert False def check_script(script): - if (type(script) is str): + if (isinstance(script, str)): script = hex_str_to_bytes(script) # Assuming this is hex string - if (type(script) is bytes or type(script) is CScript): + if (isinstance(script, bytes) or isinstance(script, CScript)): return script assert False diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index 33c43212bd..faef4d7b6b 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -1,104 +1,102 @@ #!/usr/bin/env python3 # Copyright (c) 2018 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test wallet group functionality.""" from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import CTransaction, FromHex, ToHex from test_framework.util import ( assert_equal, ) def assert_approx(v, vexp, vspan=0.00001): if v < vexp - vspan: raise AssertionError("{} < [{}..{}]".format( str(v), str(vexp - vspan), str(vexp + vspan))) if v > vexp + vspan: raise AssertionError("{} > [{}..{}]".format( str(v), str(vexp - vspan), str(vexp + vspan))) class WalletGroupTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 self.extra_args = [[], [], ['-avoidpartialspends']] self.rpc_timeout = 120 def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): # Mine some coins self.nodes[0].generate(110) # Get some addresses from the two nodes addr1 = [self.nodes[1].getnewaddress() for i in range(3)] addr2 = [self.nodes[2].getnewaddress() for i in range(3)] addrs = addr1 + addr2 # Send 1 + 0.5 coin to each address [self.nodes[0].sendtoaddress(addr, 1.0) for addr in addrs] [self.nodes[0].sendtoaddress(addr, 0.5) for addr in addrs] self.nodes[0].generate(1) self.sync_all() # For each node, send 0.2 coins back to 0; # - node[1] should pick one 0.5 UTXO and leave the rest # - node[2] should pick one (1.0 + 0.5) UTXO group corresponding to a # given address, and leave the rest txid1 = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 0.2) tx1 = self.nodes[1].getrawtransaction(txid1, True) # txid1 should have 1 input and 2 outputs assert_equal(1, len(tx1["vin"])) assert_equal(2, len(tx1["vout"])) # one output should be 0.2, the other should be ~0.3 - v = [vout["value"] for vout in tx1["vout"]] - v.sort() + v = sorted([vout["value"] for vout in tx1["vout"]]) assert_approx(v[0], 0.2) assert_approx(v[1], 0.3, 0.0001) txid2 = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 0.2) tx2 = self.nodes[2].getrawtransaction(txid2, True) # txid2 should have 2 inputs and 2 outputs assert_equal(2, len(tx2["vin"])) assert_equal(2, len(tx2["vout"])) # one output should be 0.2, the other should be ~1.3 - v = [vout["value"] for vout in tx2["vout"]] - v.sort() + v = sorted([vout["value"] for vout in tx2["vout"]]) assert_approx(v[0], 0.2) assert_approx(v[1], 1.3, 0.0001) # Empty out node2's wallet self.nodes[2].sendtoaddress(address=self.nodes[0].getnewaddress( ), amount=self.nodes[2].getbalance(), subtractfeefromamount=True) self.sync_all() self.nodes[0].generate(1) # Fill node2's wallet with 10000 outputs corresponding to the same # scriptPubKey for i in range(5): raw_tx = self.nodes[0].createrawtransaction( [{"txid": "0" * 64, "vout": 0}], [{addr2[0]: 0.05}]) tx = FromHex(CTransaction(), raw_tx) tx.vin = [] tx.vout = [tx.vout[0]] * 2000 funded_tx = self.nodes[0].fundrawtransaction(ToHex(tx)) signed_tx = self.nodes[0].signrawtransactionwithwallet( funded_tx['hex']) self.nodes[0].sendrawtransaction(signed_tx['hex']) self.nodes[0].generate(1) self.sync_all() # Check that we can create a transaction that only requires ~100 of our # utxos, without pulling in all outputs and creating a transaction that # is way too big. assert self.nodes[2].sendtoaddress(address=addr2[0], amount=5) if __name__ == '__main__': WalletGroupTest().main() diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index 46fc8dfaff..cfbc28003a 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -1,292 +1,292 @@ #!/usr/bin/env python3 # # Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # # Lint format strings: This program checks that the number of arguments passed # to a variadic format string function matches the number of format specifiers # in the format string. import argparse import doctest import re import sys FALSE_POSITIVES = [ ("src/dbwrapper.cpp", "vsnprintf(p, limit - p, format, backup_ap)"), ("src/index/base.cpp", "FatalError(const char *fmt, const Args &... args)"), ("src/netbase.cpp", "LogConnectFailure(bool manual_connection, const char *fmt, const Args &... args)"), ("src/util/system.cpp", "strprintf(_(COPYRIGHT_HOLDERS), _(COPYRIGHT_HOLDERS_SUBSTITUTION))"), ("src/seeder/main.cpp", "fprintf(stderr, help, argv[0])"), ("src/tinyformat.h", "printf(const char *fmt, const Args &... args)"), ("src/tinyformat.h", "printf(const char *fmt, TINYFORMAT_VARARGS(n))"), ("src/wallet/wallet.h", "LogPrintf((\"%s \" + fmt).c_str(), GetDisplayName(), parameters...)"), ("src/logging.h", "LogPrintf(const char *fmt, const Args &... args)"), ] FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS = [ ("FatalError", 0), ("fprintf", 1), ("LogConnectFailure", 1), ("LogPrint", 1), ("LogPrintf", 0), ("printf", 0), ("snprintf", 2), ("sprintf", 1), ("strprintf", 0), ("vfprintf", 1), ("vprintf", 1), ("vsnprintf", 1), ("vsprintf", 1), ] def parse_function_calls(function_name, source_code): """Return an array with all calls to function function_name in string source_code. Preprocessor directives and C++ style comments ("//") in source_code are removed. >>> len(parse_function_calls("foo", "foo();bar();foo();bar();")) 2 >>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[0].startswith("foo(1);") True >>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[1].startswith("foo(2);") True >>> len(parse_function_calls("foo", "foo();bar();// foo();bar();")) 1 >>> len(parse_function_calls("foo", "#define FOO foo();")) 0 """ - assert type(function_name) is str and type( - source_code) is str and function_name + assert isinstance(function_name, str) and isinstance( + source_code, str) and function_name lines = [re.sub("// .*", " ", line).strip() for line in source_code.split("\n") if not line.strip().startswith("#")] return re.findall(r"[^a-zA-Z_](?=({}\(.*).*)".format(function_name), " " + " ".join(lines)) def normalize(s): """Return a normalized version of string s with newlines, tabs and C style comments ("/* ... */") replaced with spaces. Multiple spaces are replaced with a single space. >>> normalize(" /* nothing */ foo\tfoo /* bar */ foo ") 'foo foo foo' """ - assert type(s) is str + assert isinstance(s, str) s = s.replace("\n", " ") s = s.replace("\t", " ") s = re.sub(r"/\*.*?\*/", " ", s) s = re.sub(" {2,}", " ", s) return s.strip() ESCAPE_MAP = { r"\n": "[escaped-newline]", r"\t": "[escaped-tab]", r'\"': "[escaped-quote]", } def escape(s): """Return the escaped version of string s with "\\\"", "\\n" and "\\t" escaped as "[escaped-backslash]", "[escaped-newline]" and "[escaped-tab]". >>> unescape(escape("foo")) == "foo" True >>> escape(r'foo \\t foo \\n foo \\\\ foo \\ foo \\"bar\\"') 'foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]' """ - assert type(s) is str + assert isinstance(s, str) for raw_value, escaped_value in ESCAPE_MAP.items(): s = s.replace(raw_value, escaped_value) return s def unescape(s): """Return the unescaped version of escaped string s. Reverses the replacements made in function escape(s). >>> unescape(escape("bar")) 'bar' >>> unescape("foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]") 'foo \\\\t foo \\\\n foo \\\\\\\\ foo \\\\ foo \\\\"bar\\\\"' """ - assert type(s) is str + assert isinstance(s, str) for raw_value, escaped_value in ESCAPE_MAP.items(): s = s.replace(escaped_value, raw_value) return s def parse_function_call_and_arguments(function_name, function_call): """Split string function_call into an array of strings consisting of: * the string function_call followed by "(" * the function call argument #1 * ... * the function call argument #n * a trailing ");" The strings returned are in escaped form. See escape(...). >>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");') ['foo(', '"%s",', ' "foo"', ')'] >>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");') ['foo(', '"%s",', ' "foo"', ')'] >>> parse_function_call_and_arguments("foo", 'foo("%s %s", "foo", "bar");') ['foo(', '"%s %s",', ' "foo",', ' "bar"', ')'] >>> parse_function_call_and_arguments("fooprintf", 'fooprintf("%050d", i);') ['fooprintf(', '"%050d",', ' i', ')'] >>> parse_function_call_and_arguments("foo", 'foo(bar(foobar(barfoo("foo"))), foobar); barfoo') ['foo(', 'bar(foobar(barfoo("foo"))),', ' foobar', ')'] >>> parse_function_call_and_arguments("foo", "foo()") ['foo(', '', ')'] >>> parse_function_call_and_arguments("foo", "foo(123)") ['foo(', '123', ')'] >>> parse_function_call_and_arguments("foo", 'foo("foo")') ['foo(', '"foo"', ')'] """ - assert type(function_name) is str and type( - function_call) is str and function_name + assert isinstance(function_name, str) and isinstance( + function_call, str) and function_name remaining = normalize(escape(function_call)) expected_function_call = "{}(".format(function_name) assert remaining.startswith(expected_function_call) parts = [expected_function_call] remaining = remaining[len(expected_function_call):] open_parentheses = 1 in_string = False parts.append("") for char in remaining: parts.append(parts.pop() + char) if char == "\"": in_string = not in_string continue if in_string: continue if char == "(": open_parentheses += 1 continue if char == ")": open_parentheses -= 1 if open_parentheses > 1: continue if open_parentheses == 0: parts.append(parts.pop()[:-1]) parts.append(char) break if char == ",": parts.append("") return parts def parse_string_content(argument): """Return the text within quotes in string argument. >>> parse_string_content('1 "foo %d bar" 2') 'foo %d bar' >>> parse_string_content('1 foobar 2') '' >>> parse_string_content('1 "bar" 2') 'bar' >>> parse_string_content('1 "foo" 2 "bar" 3') 'foobar' >>> parse_string_content('1 "foo" 2 " " "bar" 3') 'foo bar' >>> parse_string_content('""') '' >>> parse_string_content('') '' >>> parse_string_content('1 2 3') '' """ - assert type(argument) is str + assert isinstance(argument, str) string_content = "" in_string = False for char in normalize(escape(argument)): if char == "\"": in_string = not in_string elif in_string: string_content += char return string_content def count_format_specifiers(format_string): """Return the number of format specifiers in string format_string. >>> count_format_specifiers("foo bar foo") 0 >>> count_format_specifiers("foo %d bar foo") 1 >>> count_format_specifiers("foo %d bar %i foo") 2 >>> count_format_specifiers("foo %d bar %i foo %% foo") 2 >>> count_format_specifiers("foo %d bar %i foo %% foo %d foo") 3 >>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo") 4 """ - assert type(format_string) is str + assert isinstance(format_string, str) n = 0 in_specifier = False for i, char in enumerate(format_string): if format_string[i - 1:i + 1] == "%%" or format_string[i:i + 2] == "%%": pass elif char == "%": in_specifier = True n += 1 elif char in "aAcdeEfFgGinopsuxX": in_specifier = False elif in_specifier and char == "*": n += 1 return n def main(args_in): """ Return a string output with information on string format errors >>> main(["test/lint/lint-format-strings-tests.txt"]) test/lint/lint-format-strings-tests.txt: Expected 1 argument(s) after format string but found 2 argument(s): printf("%d", 1, 2) test/lint/lint-format-strings-tests.txt: Expected 2 argument(s) after format string but found 3 argument(s): printf("%a %b", 1, 2, "anything") test/lint/lint-format-strings-tests.txt: Expected 1 argument(s) after format string but found 0 argument(s): printf("%d") test/lint/lint-format-strings-tests.txt: Expected 3 argument(s) after format string but found 2 argument(s): printf("%a%b%z", 1, "anything") >>> main(["test/lint/lint-format-strings-tests-skip-arguments.txt"]) test/lint/lint-format-strings-tests-skip-arguments.txt: Expected 1 argument(s) after format string but found 2 argument(s): fprintf(skipped, "%d", 1, 2) test/lint/lint-format-strings-tests-skip-arguments.txt: Expected 1 argument(s) after format string but found 0 argument(s): fprintf(skipped, "%d") test/lint/lint-format-strings-tests-skip-arguments.txt: Expected 1 argument(s) after format string but found 2 argument(s): snprintf(skip1, skip2, "%d", 1, 2) test/lint/lint-format-strings-tests-skip-arguments.txt: Expected 1 argument(s) after format string but found 0 argument(s): snprintf(skip1, skip2, "%d") test/lint/lint-format-strings-tests-skip-arguments.txt: Could not parse function call string "snprintf(...)": snprintf(skip1, "%d") """ parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed " "to a variadic format string function matches the number of format " "specifiers in the format string.") parser.add_argument("file", type=argparse.FileType( "r", encoding="utf-8"), nargs="*", help="C++ source code file (e.g. foo.cpp)") args = parser.parse_args(args_in) for f in args.file: file_content = f.read() for (function_name, skip_arguments) in FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS: for function_call_str in parse_function_calls(function_name, file_content): parts = parse_function_call_and_arguments( function_name, function_call_str) relevant_function_call_str = unescape("".join(parts))[:512] if (f.name, relevant_function_call_str) in FALSE_POSITIVES: continue if len(parts) < 3 + skip_arguments: print("{}: Could not parse function call string \"{}(...)\": {}".format( f.name, function_name, relevant_function_call_str)) continue argument_count = len(parts) - 3 - skip_arguments format_str = parse_string_content(parts[1 + skip_arguments]) format_specifier_count = count_format_specifiers(format_str) if format_specifier_count != argument_count: print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format( f.name, format_specifier_count, argument_count, relevant_function_call_str)) continue if __name__ == "__main__": doctest.testmod() main(sys.argv[1:])