diff --git a/contrib/devtools/chainparams/generate_chainparams_constants.py b/contrib/devtools/chainparams/generate_chainparams_constants.py index 205241ca9..40f426598 100755 --- a/contrib/devtools/chainparams/generate_chainparams_constants.py +++ b/contrib/devtools/chainparams/generate_chainparams_constants.py @@ -1,73 +1,74 @@ #!/usr/bin/env python3 # Copyright (c) 2019 The Bitcoin developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Script to generate list of chainparams constants (ie. assumevalid and minimum chainwork). This script expects a text file for each chain in the directory that is passed as an argument: chainparams_main.txt chainparams_test.txt These files must consist of lines in the format The outputted constants should be pasted into `src/chainparamsconstants.h`. ''' import sys import os def process_constants(indir, file_name): with open(os.path.join(indir, file_name), 'r', encoding="utf8") as f: constants = f.readlines() # Ensure only two lines are read from the file. assert(len(constants) == 2) return [line.rstrip() for line in constants] def main(): if len(sys.argv) != 2: - print('Usage: {} '.format(sys.argv[0]), file=sys.stderr) + print('Usage: {} '.format( + sys.argv[0]), file=sys.stderr) sys.exit(1) indir = sys.argv[1] print('''\ #ifndef BITCOIN_CHAINPARAMSCONSTANTS_H #define BITCOIN_CHAINPARAMSCONSTANTS_H /** * Chain params constants for each tracked chain. * @{} by contrib/devtools/chainparams/generate_chainparams_constants.py */ #include #include namespace ChainParamsConstants {{ const BlockHash MAINNET_DEFAULT_ASSUME_VALID = BlockHash::fromHex("{}"); const uint256 MAINNET_MINIMUM_CHAIN_WORK = uint256S("{}"); const BlockHash TESTNET_DEFAULT_ASSUME_VALID = BlockHash::fromHex("{}"); const uint256 TESTNET_MINIMUM_CHAIN_WORK = uint256S("{}"); }} // namespace ChainParamsConstants #endif // BITCOIN_CHAINPARAMSCONSTANTS_H\ '''.format( # 'generated' is split out so this file is not identified as generated. "generated", *process_constants(indir, 'chainparams_main.txt'), *process_constants(indir, 'chainparams_test.txt')) ) if __name__ == '__main__': main() diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index a082dd1a4..230ea96e4 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -1,163 +1,164 @@ #!/usr/bin/env python3 # # linearize-hashes.py: List blocks in a linear, no-fork version of the chain. # # Copyright (c) 2013-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # from __future__ import print_function try: # Python 3 import http.client as httplib except ImportError: # Python 2 import httplib import json import re import base64 import sys import os import os.path settings = {} def hex_switchEndian(s): """ Switches the endianness of a hex string (in pairs of hex chars) """ pairList = [s[i:i + 2].encode() for i in range(0, len(s), 2)] return b''.join(pairList[::-1]).decode() class BitcoinRPC: def __init__(self, host, port, username, password): authpair = "{}:{}".format(username, password) authpair = authpair.encode('utf-8') self.authhdr = b"Basic " + base64.b64encode(authpair) self.conn = httplib.HTTPConnection(host, port=port, timeout=30) def execute(self, obj): try: self.conn.request('POST', '/', json.dumps(obj), {'Authorization': self.authhdr, 'Content-type': 'application/json'}) except ConnectionRefusedError: print('RPC connection refused. Check RPC settings and the server status.', file=sys.stderr) return None resp = self.conn.getresponse() if resp is None: print("JSON-RPC: no response", file=sys.stderr) return None body = resp.read().decode('utf-8') resp_obj = json.loads(body) return resp_obj @staticmethod def build_request(idx, method, params): obj = {'version': '1.1', 'method': method, 'id': idx} if params is None: obj['params'] = [] else: obj['params'] = params return obj @staticmethod def response_is_error(resp_obj): return 'error' in resp_obj and resp_obj['error'] is not None def get_block_hashes(settings, max_blocks_per_call=10000): rpc = BitcoinRPC(settings['host'], settings['port'], settings['rpcuser'], settings['rpcpassword']) height = settings['min_height'] while height < settings['max_height'] + 1: - num_blocks = min(settings['max_height'] + - 1 - height, max_blocks_per_call) + num_blocks = min(settings['max_height'] + + 1 - height, max_blocks_per_call) batch = [] for x in range(num_blocks): batch.append(rpc.build_request(x, 'getblockhash', [height + x])) reply = rpc.execute(batch) if reply is None: print('Cannot continue. Program will halt.') return None for x, resp_obj in enumerate(reply): if rpc.response_is_error(resp_obj): print('JSON-RPC: error at height', height + x, ': ', resp_obj['error'], file=sys.stderr) sys.exit(1) assert(resp_obj['id'] == x) # assume replies are in-sequence if settings['rev_hash_bytes'] == 'true': resp_obj['result'] = hex_switchEndian(resp_obj['result']) print(resp_obj['result']) height += num_blocks def get_rpc_cookie(): # Open the cookie file with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r', encoding="ascii") as f: combined = f.readline() combined_split = combined.split(":") settings['rpcuser'] = combined_split[0] settings['rpcpassword'] = combined_split[1] if __name__ == '__main__': if len(sys.argv) != 2: print("Usage: linearize-hashes.py CONFIG-FILE") sys.exit(1) f = open(sys.argv[1], encoding="utf8") for line in f: # skip comment lines m = re.search(r'^\s*#', line) if m: continue # parse key=value lines m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line) if m is None: continue settings[m.group(1)] = m.group(2) f.close() if 'host' not in settings: settings['host'] = '127.0.0.1' if 'port' not in settings: settings['port'] = 8332 if 'min_height' not in settings: settings['min_height'] = 0 if 'max_height' not in settings: settings['max_height'] = 313000 if 'rev_hash_bytes' not in settings: settings['rev_hash_bytes'] = 'false' use_userpass = True use_datadir = False if 'rpcuser' not in settings or 'rpcpassword' not in settings: use_userpass = False if 'datadir' in settings and not use_userpass: use_datadir = True if not use_userpass and not use_datadir: - print("Missing datadir or username and/or password in cfg file", file=sys.stderr) + print("Missing datadir or username and/or password in cfg file", + file=sys.stderr) sys.exit(1) settings['port'] = int(settings['port']) settings['min_height'] = int(settings['min_height']) settings['max_height'] = int(settings['max_height']) # Force hash byte format setting to be lowercase to make comparisons easier. settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower() # Get the rpc user and pass from the cookie if the datadir is set if use_datadir: get_rpc_cookie() get_block_hashes(settings) diff --git a/test/functional/combine_logs.py b/test/functional/combine_logs.py index 98de861b5..63e0acfc2 100755 --- a/test/functional/combine_logs.py +++ b/test/functional/combine_logs.py @@ -1,123 +1,124 @@ #!/usr/bin/env python3 """Combine logs from multiple bitcoin nodes as well as the test_framework log. This streams the combined log output to stdout. Use combine_logs.py > outputfile to write to an outputfile.""" import argparse from collections import defaultdict, namedtuple import heapq import itertools import os import re import sys # Matches on the date format at the start of the log event TIMESTAMP_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z") LogEvent = namedtuple('LogEvent', ['timestamp', 'source', 'event']) def main(): """Main function. Parses args, reads the log files and renders them as text or html.""" parser = argparse.ArgumentParser( usage='%(prog)s [options] ', description=__doc__) parser.add_argument('-c', '--color', dest='color', action='store_true', help='outputs the combined log with events colored by source (requires posix terminal colors. Use less -r for viewing)') parser.add_argument('--html', dest='html', action='store_true', help='outputs the combined log as html. Requires jinja2. pip install jinja2') args, unknown_args = parser.parse_known_args() if args.color and os.name != 'posix': print("Color output requires posix terminal colors.") sys.exit(1) if args.html and args.color: print("Only one out of --color or --html should be specified") sys.exit(1) # There should only be one unknown argument - the path of the temporary test directory if len(unknown_args) != 1: print("Unexpected arguments" + str(unknown_args)) sys.exit(1) log_events = read_logs(unknown_args[0]) print_logs(log_events, color=args.color, html=args.html) def read_logs(tmp_dir): """Reads log files. Delegates to generator function get_log_events() to provide individual log events for each of the input log files.""" files = [("test", "{}/test_framework.log".format(tmp_dir))] for i in itertools.count(): logfile = "{}/node{}/regtest/debug.log".format(tmp_dir, i) if not os.path.isfile(logfile): break files.append(("node{}".format(i), logfile)) return heapq.merge(*[get_log_events(source, f) for source, f in files]) def get_log_events(source, logfile): """Generator function that returns individual log events. Log events may be split over multiple lines. We use the timestamp regex match as the marker for a new log event.""" try: with open(logfile, 'r', encoding='utf-8') as infile: event = '' timestamp = '' for line in infile: # skip blank lines if line == '\n': continue # if this line has a timestamp, it's the start of a new log event. time_match = TIMESTAMP_PATTERN.match(line) if time_match: if event: yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip()) event = line timestamp = time_match.group() # if it doesn't have a timestamp, it's a continuation line of the previous log. else: event += "\n" + line # Flush the final event yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip()) except FileNotFoundError: - print("File {} could not be opened. Continuing without it.".format(logfile), file=sys.stderr) + print("File {} could not be opened. Continuing without it.".format( + logfile), file=sys.stderr) def print_logs(log_events, color=False, html=False): """Renders the iterator of log events into text or html.""" if not html: colors = defaultdict(lambda: '') if color: colors["test"] = "\033[0;36m" # CYAN colors["node0"] = "\033[0;34m" # BLUE colors["node1"] = "\033[0;32m" # GREEN colors["node2"] = "\033[0;31m" # RED colors["node3"] = "\033[0;33m" # YELLOW colors["reset"] = "\033[0m" # Reset font color for event in log_events: print("{0} {1: <5} {2} {3}".format( colors[event.source.rstrip()], event.source, event.event, colors["reset"])) else: try: import jinja2 except ImportError: print("jinja2 not found. Try `pip install jinja2`") sys.exit(1) print(jinja2.Environment(loader=jinja2.FileSystemLoader('./')) .get_template('combined_log_template.html') .render(title="Combined Logs from testcase", log_events=[event._asdict() for event in log_events])) if __name__ == '__main__': main() diff --git a/test/lint/check-rpc-mappings.py b/test/lint/check-rpc-mappings.py index 57c64e922..8acdb45b6 100755 --- a/test/lint/check-rpc-mappings.py +++ b/test/lint/check-rpc-mappings.py @@ -1,177 +1,178 @@ #!/usr/bin/env python3 # Copyright (c) 2017-2018 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Check RPC argument consistency.""" from collections import defaultdict import glob import os import re import sys # Source files (relative to root) to scan for dispatch tables SOURCE_PATTERNS = [ "src/rpc/*.cpp", "src/wallet/rpc*.cpp", "src/zmq/zmqrpc.cpp", ] # Source file (relative to root) containing conversion mapping SOURCE_CLIENT = 'src/rpc/client.cpp' # Argument names that should be ignored in consistency checks IGNORE_DUMMY_ARGS = {'dummy', 'arg0', 'arg1', 'arg2', 'arg3', 'arg4', 'arg5', 'arg6', 'arg7', 'arg8', 'arg9'} class RPCCommand: def __init__(self, name, args): self.name = name self.args = args class RPCArgument: def __init__(self, names, idx): self.names = names self.idx = idx self.convert = False def parse_string(s): assert s[0] == '"' assert s[-1] == '"' return s[1:-1] def process_commands(fname): """Find and parse dispatch table in implementation file `fname`.""" cmds = [] in_rpcs = False with open(fname, "r", encoding="utf8") as f: for line in f: line = line.rstrip() if not in_rpcs: if re.match(r"static const ContextFreeRPCCommand .*\[\] =", line): in_rpcs = True else: if line.startswith('};'): in_rpcs = False elif '{' in line and '"' in line: m = re.search( '{ *("[^"]*"), *("[^"]*"), *([^,]*), *{([^}]*)} *},', line) assert m, 'No match to table expression: {}'.format(line) name = parse_string(m.group(2)) args_str = m.group(4).strip() if args_str: args = [RPCArgument(parse_string(x.strip()).split( '|'), idx) for idx, x in enumerate(args_str.split(','))] else: args = [] cmds.append(RPCCommand(name, args)) assert not in_rpcs, "Something went wrong with parsing the C++ file: update the regexps" return cmds def process_mapping(fname): """Find and parse conversion table in implementation file `fname`.""" cmds = [] in_rpcs = False with open(fname, "r", encoding="utf8") as f: for line in f: line = line.rstrip() if not in_rpcs: if line == 'static const CRPCConvertParam vRPCConvertParams[] = {': in_rpcs = True else: if line.startswith('};'): in_rpcs = False elif '{' in line and '"' in line: m = re.search( '{ *("[^"]*"), *([0-9]+) *, *("[^"]*") *},', line) assert m, 'No match to table expression: {}'.format(line) name = parse_string(m.group(1)) idx = int(m.group(2)) argname = parse_string(m.group(3)) cmds.append((name, idx, argname)) assert not in_rpcs and cmds return cmds def main(): if len(sys.argv) != 2: print('Usage: {} ROOT-DIR'.format(sys.argv[0]), file=sys.stderr) sys.exit(1) root = sys.argv[1] # Find the sources files sources = [] for glob_regex in SOURCE_PATTERNS: sources.extend(glob.glob(os.path.join(root, glob_regex))) # Get all commands from dispatch tables cmds = [] for fname in set(sources): cmds += process_commands(fname) cmds_by_name = {} for cmd in cmds: cmds_by_name[cmd.name] = cmd # Get current convert mapping for client client = SOURCE_CLIENT mapping = set(process_mapping(os.path.join(root, client))) print('* Checking consistency between dispatch tables and vRPCConvertParams') # Check mapping consistency errors = 0 for (cmdname, argidx, argname) in mapping: try: rargnames = cmds_by_name[cmdname].args[argidx].names except IndexError: print('ERROR: {} argument {} (named {} in vRPCConvertParams) is not defined in dispatch table'.format( cmdname, argidx, argname)) errors += 1 continue if argname not in rargnames: - print('ERROR: {} argument {} is named {} in vRPCConvertParams but {} in dispatch table'.format(cmdname, argidx, argname, rargnames), file=sys.stderr) + print('ERROR: {} argument {} is named {} in vRPCConvertParams but {} in dispatch table'.format( + cmdname, argidx, argname, rargnames), file=sys.stderr) errors += 1 # Check for conflicts in vRPCConvertParams conversion # All aliases for an argument must either be present in the # conversion table, or not. Anything in between means an oversight # and some aliases won't work. for cmd in cmds: for arg in cmd.args: convert = [((cmd.name, arg.idx, argname) in mapping) for argname in arg.names] if any(convert) != all(convert): print('ERROR: {} argument {} has conflicts in vRPCConvertParams conversion specifier {}'.format( cmd.name, arg.names, convert)) errors += 1 arg.convert = all(convert) # Check for conversion difference by argument name. # It is preferable for API consistency that arguments with the same name # have the same conversion, so bin by argument name. all_methods_by_argname = defaultdict(list) converts_by_argname = defaultdict(list) for cmd in cmds: for arg in cmd.args: for argname in arg.names: all_methods_by_argname[argname].append(cmd.name) converts_by_argname[argname].append(arg.convert) for argname, convert in converts_by_argname.items(): if all(convert) != any(convert): if argname in IGNORE_DUMMY_ARGS: # these are testing or dummy, don't warn for them continue print('WARNING: conversion mismatch for argument named {} ({})'.format( argname, list(zip(all_methods_by_argname[argname], converts_by_argname[argname])))) sys.exit(0) if __name__ == '__main__': main()