diff --git a/contrib/devtools/check-doc.py b/contrib/devtools/check-doc.py index 885d2e1245..7123076f44 100755 --- a/contrib/devtools/check-doc.py +++ b/contrib/devtools/check-doc.py @@ -1,56 +1,57 @@ #!/usr/bin/env python # Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' This checks if all command line args are documented. Return value is 0 to indicate no error. Author: @MarcoFalke ''' # FIXME: the script makes assumptions on how to find documented # options - these have been broken by clang-format # reformatting of the sources. # The script likely needs major rework. from subprocess import check_output import re +import sys FOLDER_GREP = 'src' FOLDER_TEST = 'src/test/' CMD_ROOT_DIR = '`git rev-parse --show-toplevel`/%s' % FOLDER_GREP CMD_GREP_ARGS = r"egrep -r -I '(map(Multi)?Args(\.count\(|\[)|Get(Bool)?Arg\()\"\-[^\"]+?\"' %s | grep -v '%s'" % ( CMD_ROOT_DIR, FOLDER_TEST) CMD_GREP_DOCS = r"egrep -r -I 'HelpMessageOpt\(\"\-[^\"=]+?(=|\")' %s" % ( CMD_ROOT_DIR) REGEX_ARG = re.compile( r'(?:map(?:Multi)?Args(?:\.count\(|\[)|Get(?:Bool)?Arg\()\"(\-[^\"]+?)\"') REGEX_DOC = re.compile(r'HelpMessageOpt\(\"(\-[^\"=]+?)(?:=|\")') # list unsupported, deprecated and duplicate args as they need no documentation SET_DOC_OPTIONAL = set(['-rpcssl', '-benchmark', '-h', '-help', '-socks', '-tor', '-debugnet', '-whitelistalwaysrelay', '-prematurewitness', '-walletprematurewitness', '-promiscuousmempoolflags', '-blockminsize', '-forcecompactdb', '-dbcrashratio']) def main(): used = check_output(CMD_GREP_ARGS, shell=True) docd = check_output(CMD_GREP_DOCS, shell=True) args_used = set(re.findall(REGEX_ARG, used)) args_docd = set(re.findall(REGEX_DOC, docd)).union(SET_DOC_OPTIONAL) args_need_doc = args_used.difference(args_docd) args_unknown = args_docd.difference(args_used) print "Args used : %s" % len(args_used) print "Args documented : %s" % len(args_docd) print "Args undocumented: %s" % len(args_need_doc) print args_need_doc print "Args unknown : %s" % len(args_unknown) print args_unknown - exit(len(args_need_doc)) + sys.exit(len(args_need_doc)) if __name__ == "__main__": main() diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 27a9f944c6..fd53f4aaf5 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -1,235 +1,235 @@ #!/usr/bin/env python # Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Perform basic ELF security checks on a series of executables. Exit status will be 0 if successful, and the program will be silent. Otherwise the exit status will be 1 and it will log which executables failed which checks. Needs `readelf` (for ELF) and `objdump` (for PE). ''' from __future__ import division, print_function, unicode_literals import subprocess import sys import os READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') # checks which are non-fatal for now but only generate a warning NONFATAL = {'HIGH_ENTROPY_VA'} def check_ELF_PIE(executable): ''' Check for position independent executable (PIE), allowing for address space randomization. ''' p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') ok = False for line in stdout.splitlines(): line = line.split() if len(line) >= 2 and line[0] == 'Type:' and line[1] == 'DYN': ok = True return ok def get_ELF_program_headers(executable): '''Return type and flags for ELF program headers''' p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') in_headers = False count = 0 headers = [] for line in stdout.splitlines(): if line.startswith('Program Headers:'): in_headers = True if line == '': in_headers = False if in_headers: if count == 1: # header line ofs_typ = line.find('Type') ofs_offset = line.find('Offset') ofs_flags = line.find('Flg') ofs_align = line.find('Align') if ofs_typ == -1 or ofs_offset == -1 or ofs_flags == -1 or ofs_align == -1: raise ValueError('Cannot parse elfread -lW output') elif count > 1: typ = line[ofs_typ:ofs_offset].rstrip() flags = line[ofs_flags:ofs_align].rstrip() headers.append((typ, flags)) count += 1 return headers def check_ELF_NX(executable): ''' Check that no sections are writable and executable (including the stack) ''' have_wx = False have_gnu_stack = False for (typ, flags) in get_ELF_program_headers(executable): if typ == 'GNU_STACK': have_gnu_stack = True if 'W' in flags and 'E' in flags: # section is both writable and executable have_wx = True return have_gnu_stack and not have_wx def check_ELF_RELRO(executable): ''' Check for read-only relocations. GNU_RELRO program header must exist Dynamic section must have BIND_NOW flag ''' have_gnu_relro = False for (typ, flags) in get_ELF_program_headers(executable): # Note: not checking flags == 'R': here as linkers set the permission differently # This does not affect security: the permission flags of the GNU_RELRO program header are ignored, the PT_LOAD header determines the effective permissions. # However, the dynamic linker need to write to this area so these are RW. # Glibc itself takes care of mprotecting this area R after relocations are finished. # See also http://permalink.gmane.org/gmane.comp.gnu.binutils/71347 if typ == 'GNU_RELRO': have_gnu_relro = True have_bindnow = False p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') for line in stdout.splitlines(): tokens = line.split() if len(tokens) > 1 and tokens[1] == '(BIND_NOW)' or (len(tokens) > 2 and tokens[1] == '(FLAGS)' and 'BIND_NOW' in tokens[2]): have_bindnow = True return have_gnu_relro and have_bindnow def check_ELF_Canary(executable): ''' Check for use of stack canary ''' p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') ok = False for line in stdout.splitlines(): if '__stack_chk_fail' in line: ok = True return ok def get_PE_dll_characteristics(executable): ''' Get PE DllCharacteristics bits. Returns a tuple (arch,bits) where arch is 'i386:x86-64' or 'i386' and bits is the DllCharacteristics value. ''' p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') arch = '' bits = 0 for line in stdout.splitlines(): tokens = line.split() if len(tokens) >= 2 and tokens[0] == 'architecture:': arch = tokens[1].rstrip(',') if len(tokens) >= 2 and tokens[0] == 'DllCharacteristics': bits = int(tokens[1], 16) return (arch, bits) IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA = 0x0020 IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE = 0x0040 IMAGE_DLL_CHARACTERISTICS_NX_COMPAT = 0x0100 def check_PE_DYNAMIC_BASE(executable): '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' (arch, bits) = get_PE_dll_characteristics(executable) reqbits = IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE return (bits & reqbits) == reqbits # On 64 bit, must support high-entropy 64-bit address space layout randomization in addition to DYNAMIC_BASE # to have secure ASLR. def check_PE_HIGH_ENTROPY_VA(executable): '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' (arch, bits) = get_PE_dll_characteristics(executable) if arch == 'i386:x86-64': reqbits = IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA else: # Unnecessary on 32-bit assert(arch == 'i386') reqbits = 0 return (bits & reqbits) == reqbits def check_PE_NX(executable): '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)''' (arch, bits) = get_PE_dll_characteristics(executable) return (bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT CHECKS = { 'ELF': [ ('PIE', check_ELF_PIE), ('NX', check_ELF_NX), ('RELRO', check_ELF_RELRO), ('Canary', check_ELF_Canary) ], 'PE': [ ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), ('NX', check_PE_NX) ] } def identify_executable(executable): with open(filename, 'rb') as f: magic = f.read(4) if magic.startswith(b'MZ'): return 'PE' elif magic.startswith(b'\x7fELF'): return 'ELF' return None if __name__ == '__main__': retval = 0 for filename in sys.argv[1:]: try: etype = identify_executable(filename) if etype is None: print('%s: unknown format' % filename) retval = 1 continue failed = [] warning = [] for (name, func) in CHECKS[etype]: if not func(filename): if name in NONFATAL: warning.append(name) else: failed.append(name) if failed: print('%s: failed %s' % (filename, ' '.join(failed))) retval = 1 if warning: print('%s: warning %s' % (filename, ' '.join(warning))) except IOError: print('%s: cannot open' % filename) retval = 1 - exit(retval) + sys.exit(retval) diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 3bd3c945b4..3d81db042f 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -1,179 +1,179 @@ #!/usr/bin/env python # Copyright (c) 2014 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' A script to check that the (Linux) executables produced by gitian only contain allowed gcc, glibc and libstdc++ version symbols. This makes sure they are still compatible with the minimum supported Linux distribution versions. Example usage: find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py ''' from __future__ import division, print_function, unicode_literals import subprocess import re import sys import os # Debian 6.0.9 (Squeeze) has: # # - g++ version 4.4.5 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=g%2B%2B) # - libc version 2.11.3 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libc6) # - libstdc++ version 4.4.5 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libstdc%2B%2B6) # # Ubuntu 10.04.4 (Lucid Lynx) has: # # - g++ version 4.4.3 (http://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=lucid§ion=all) # - libc version 2.11.1 (http://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=lucid§ion=all) # - libstdc++ version 4.4.3 (http://packages.ubuntu.com/search?suite=lucid§ion=all&arch=any&keywords=libstdc%2B%2B&searchon=names) # # Taking the minimum of these as our target. # # According to GNU ABI document (http://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: # GCC 4.4.0: GCC_4.4.0 # GCC 4.4.2: GLIBCXX_3.4.13, CXXABI_1.3.3 # (glibc) GLIBC_2_11 # MAX_VERSIONS = { 'GCC': (4, 4, 0), 'CXXABI': (1, 3, 3), 'GLIBCXX': (3, 4, 13), 'GLIBC': (2, 11) } # See here for a description of _IO_stdin_used: # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { '_edata', '_end', '_init', '__bss_start', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', # Figure out why we get these symbols exported on xenial. '_ZNKSt5ctypeIcE8do_widenEc', 'in6addr_any', 'optarg', '_ZNSt16_Sp_counted_baseILN9__gnu_cxx12_Lock_policyE2EE10_M_destroyEv' } READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') # Allowed NEEDED libraries ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt 'libgcc_s.so.1', # GCC base support 'libc.so.6', # C library 'libpthread.so.0', # threading 'libanl.so.1', # DNS resolve 'libm.so.6', # math library 'librt.so.1', # real-time (clock) 'ld-linux-x86-64.so.2', # 64-bit dynamic linker 'ld-linux.so.2', # 32-bit dynamic linker # bitcoin-qt only 'libX11-xcb.so.1', # part of X11 'libX11.so.6', # part of X11 'libxcb.so.1', # part of X11 'libfontconfig.so.1', # font support 'libfreetype.so.6', # font parsing 'libdl.so.2' # programming interface to dynamic linker } class CPPFilt(object): ''' Demangle C++ symbol names. Use a pipe to the 'c++filt' command. ''' def __init__(self): self.proc = subprocess.Popen( CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) def __call__(self, mangled): self.proc.stdin.write(mangled + '\n') self.proc.stdin.flush() return self.proc.stdout.readline().rstrip() def close(self): self.proc.stdin.close() self.proc.stdout.close() self.proc.wait() def read_symbols(executable, imports=True): ''' Parse an ELF executable and return a list of (symbol,version) tuples for dynamic, imported symbols. ''' p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Could not read symbols for %s: %s' % (executable, stderr.strip())) syms = [] for line in stdout.splitlines(): line = line.split() if len(line) > 7 and re.match('[0-9]+:$', line[0]): (sym, _, version) = line[7].partition('@') is_import = line[6] == 'UND' if version.startswith('@'): version = version[1:] if is_import == imports: syms.append((sym, version)) return syms def check_version(max_versions, version): if '_' in version: (lib, _, ver) = version.rpartition('_') else: lib = version ver = '0' ver = tuple([int(x) for x in ver.split('.')]) if not lib in max_versions: return False return ver <= max_versions[lib] def read_libraries(filename): p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: raise IOError('Error opening file') libraries = [] for line in stdout.splitlines(): tokens = line.split() if len(tokens) > 2 and tokens[1] == '(NEEDED)': match = re.match( '^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) if match: libraries.append(match.group(1)) else: raise ValueError('Unparseable (NEEDED) specification') return libraries if __name__ == '__main__': cppfilt = CPPFilt() retval = 0 for filename in sys.argv[1:]: # Check imported symbols for sym, version in read_symbols(filename, True): if version and not check_version(MAX_VERSIONS, version): print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym), version)) retval = 1 # Check exported symbols for sym, version in read_symbols(filename, False): if sym in IGNORE_EXPORTS: continue print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym))) retval = 1 # Check dependency libraries for library_name in read_libraries(filename): if library_name not in ALLOWED_LIBRARIES: print('%s: NEEDED library %s is not allowed' % (filename, library_name)) retval = 1 - exit(retval) + sys.exit(retval) diff --git a/contrib/devtools/update-translations.py b/contrib/devtools/update-translations.py index 338cf2dbd3..435af49b13 100755 --- a/contrib/devtools/update-translations.py +++ b/contrib/devtools/update-translations.py @@ -1,229 +1,229 @@ #!/usr/bin/env python # Copyright (c) 2014 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Run this script from the root of the repository to update all translations from transifex. It will do the following automatically: - fetch all translations using the tx tool - post-process them into valid and committable format - remove invalid control characters - remove location tags (makes diffs less noisy) TODO: - auto-add new translations to the build system according to the translation process ''' from __future__ import division, print_function import subprocess import re import sys import os import io import xml.etree.ElementTree as ET # Name of transifex tool TX = 'tx' # Name of source language file SOURCE_LANG = 'bitcoin_en.ts' # Directory with locale files LOCALE_DIR = 'src/qt/locale' # Minimum number of messages for translation to be considered at all MIN_NUM_MESSAGES = 10 def check_at_repository_root(): if not os.path.exists('.git'): print('No .git directory found') print('Execute this script at the root of the repository', file=sys.stderr) - exit(1) + sys.exit(1) def fetch_all_translations(): if subprocess.call([TX, 'pull', '-f', '-a']): print('Error while fetching translations', file=sys.stderr) - exit(1) + sys.exit(1) def find_format_specifiers(s): '''Find all format specifiers in a string.''' pos = 0 specifiers = [] while True: percent = s.find('%', pos) if percent < 0: break specifiers.append(s[percent + 1]) pos = percent + 2 return specifiers def split_format_specifiers(specifiers): '''Split format specifiers between numeric (Qt) and others (strprintf)''' numeric = [] other = [] for s in specifiers: if s in {'1', '2', '3', '4', '5', '6', '7', '8', '9'}: numeric.append(s) else: other.append(s) # If both numeric format specifiers and "others" are used, assume we're dealing # with a Qt-formatted message. In the case of Qt formatting (see https://doc.qt.io/qt-5/qstring.html#arg) # only numeric formats are replaced at all. This means "(percentage: %1%)" is valid, without needing # any kind of escaping that would be necessary for strprintf. Without this, this function # would wrongly detect '%)' as a printf format specifier. if numeric: other = [] # numeric (Qt) can be present in any order, others (strprintf) must be in specified order return set(numeric), other def sanitize_string(s): '''Sanitize string for printing''' return s.replace('\n', ' ') def check_format_specifiers(source, translation, errors, numerus): source_f = split_format_specifiers(find_format_specifiers(source)) # assert that no source messages contain both Qt and strprintf format specifiers # if this fails, go change the source as this is hacky and confusing! assert(not(source_f[0] and source_f[1])) try: translation_f = split_format_specifiers( find_format_specifiers(translation)) except IndexError: errors.append("Parse error in translation for '%s': '%s'" % ( sanitize_string(source), sanitize_string(translation))) return False else: if source_f != translation_f: if numerus and source_f == (set(), ['n']) and translation_f == (set(), []) and translation.find('%') == -1: # Allow numerus translations to omit %n specifier (usually when it only has one possible value) return True errors.append("Mismatch between '%s' and '%s'" % ( sanitize_string(source), sanitize_string(translation))) return False return True def all_ts_files(suffix=''): for filename in os.listdir(LOCALE_DIR): # process only language files, and do not process source language if not filename.endswith('.ts' + suffix) or filename == SOURCE_LANG + suffix: continue if suffix: # remove provided suffix filename = filename[0:-len(suffix)] filepath = os.path.join(LOCALE_DIR, filename) yield(filename, filepath) FIX_RE = re.compile(b'[\x00-\x09\x0b\x0c\x0e-\x1f]') def remove_invalid_characters(s): '''Remove invalid characters from translation string''' return FIX_RE.sub(b'', s) # Override cdata escape function to make our output match Qt's (optional, just for cleaner diffs for # comparison, disable by default) _orig_escape_cdata = None def escape_cdata(text): text = _orig_escape_cdata(text) text = text.replace("'", ''') text = text.replace('"', '"') return text def postprocess_translations(reduce_diff_hacks=False): print('Checking and postprocessing...') if reduce_diff_hacks: global _orig_escape_cdata _orig_escape_cdata = ET._escape_cdata ET._escape_cdata = escape_cdata for (filename, filepath) in all_ts_files(): os.rename(filepath, filepath + '.orig') have_errors = False for (filename, filepath) in all_ts_files('.orig'): # pre-fixups to cope with transifex output # need to override encoding because 'utf8' is not understood only 'utf-8' parser = ET.XMLParser(encoding='utf-8') with open(filepath + '.orig', 'rb') as f: data = f.read() # remove control characters; this must be done over the entire file otherwise the XML parser will fail data = remove_invalid_characters(data) tree = ET.parse(io.BytesIO(data), parser=parser) # iterate over all messages in file root = tree.getroot() for context in root.findall('context'): for message in context.findall('message'): numerus = message.get('numerus') == 'yes' source = message.find('source').text translation_node = message.find('translation') # pick all numerusforms if numerus: translations = [ i.text for i in translation_node.findall('numerusform')] else: translations = [translation_node.text] for translation in translations: if translation is None: continue errors = [] valid = check_format_specifiers( source, translation, errors, numerus) for error in errors: print('%s: %s' % (filename, error)) if not valid: # set type to unfinished and clear string if invalid translation_node.clear() translation_node.set('type', 'unfinished') have_errors = True # Remove location tags for location in message.findall('location'): message.remove(location) # Remove entire message if it is an unfinished translation if translation_node.get('type') == 'unfinished': context.remove(message) # check if document is (virtually) empty, and remove it if so num_messages = 0 for context in root.findall('context'): for message in context.findall('message'): num_messages += 1 if num_messages < MIN_NUM_MESSAGES: print('Removing %s, as it contains only %i messages' % (filepath, num_messages)) continue # write fixed-up tree # if diff reduction requested, replace some XML to 'sanitize' to qt formatting if reduce_diff_hacks: out = io.BytesIO() tree.write(out, encoding='utf-8') out = out.getvalue() out = out.replace(b' />', b'/>') with open(filepath, 'wb') as f: f.write(out) else: tree.write(filepath, encoding='utf-8') return have_errors if __name__ == '__main__': check_at_repository_root() fetch_all_translations() postprocess_translations() diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index facfea9a11..d016b996b1 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -1,165 +1,165 @@ #!/usr/bin/env python3 # # linearize-hashes.py: List blocks in a linear, no-fork version of the chain. # # Copyright (c) 2013-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # from __future__ import print_function try: # Python 3 import http.client as httplib except ImportError: # Python 2 import httplib import json import re import base64 import sys import os import os.path settings = {} ##### Switch endian-ness ##### def hex_switchEndian(s): """ Switches the endianness of a hex string (in pairs of hex chars) """ pairList = [s[i:i + 2].encode() for i in range(0, len(s), 2)] return b''.join(pairList[::-1]).decode() class BitcoinRPC: def __init__(self, host, port, username, password): authpair = "%s:%s" % (username, password) authpair = authpair.encode('utf-8') self.authhdr = b"Basic " + base64.b64encode(authpair) self.conn = httplib.HTTPConnection(host, port=port, timeout=30) def execute(self, obj): try: self.conn.request('POST', '/', json.dumps(obj), {'Authorization': self.authhdr, 'Content-type': 'application/json'}) except ConnectionRefusedError: print('RPC connection refused. Check RPC settings and the server status.', file=sys.stderr) return None resp = self.conn.getresponse() if resp is None: print("JSON-RPC: no response", file=sys.stderr) return None body = resp.read().decode('utf-8') resp_obj = json.loads(body) return resp_obj @staticmethod def build_request(idx, method, params): obj = {'version': '1.1', 'method': method, 'id': idx} if params is None: obj['params'] = [] else: obj['params'] = params return obj @staticmethod def response_is_error(resp_obj): return 'error' in resp_obj and resp_obj['error'] is not None def get_block_hashes(settings, max_blocks_per_call=10000): rpc = BitcoinRPC(settings['host'], settings['port'], settings['rpcuser'], settings['rpcpassword']) height = settings['min_height'] while height < settings['max_height'] + 1: num_blocks = min(settings['max_height'] + 1 - height, max_blocks_per_call) batch = [] for x in range(num_blocks): batch.append(rpc.build_request(x, 'getblockhash', [height + x])) reply = rpc.execute(batch) if reply is None: print('Cannot continue. Program will halt.') return None for x, resp_obj in enumerate(reply): if rpc.response_is_error(resp_obj): print('JSON-RPC: error at height', height + x, ': ', resp_obj['error'], file=sys.stderr) - exit(1) + sys.exit(1) assert(resp_obj['id'] == x) # assume replies are in-sequence if settings['rev_hash_bytes'] == 'true': resp_obj['result'] = hex_switchEndian(resp_obj['result']) print(resp_obj['result']) height += num_blocks def get_rpc_cookie(): # Open the cookie file with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r') as f: combined = f.readline() combined_split = combined.split(":") settings['rpcuser'] = combined_split[0] settings['rpcpassword'] = combined_split[1] if __name__ == '__main__': if len(sys.argv) != 2: print("Usage: linearize-hashes.py CONFIG-FILE") sys.exit(1) f = open(sys.argv[1]) for line in f: # skip comment lines m = re.search('^\s*#', line) if m: continue # parse key=value lines m = re.search('^(\w+)\s*=\s*(\S.*)$', line) if m is None: continue settings[m.group(1)] = m.group(2) f.close() if 'host' not in settings: settings['host'] = '127.0.0.1' if 'port' not in settings: settings['port'] = 8332 if 'min_height' not in settings: settings['min_height'] = 0 if 'max_height' not in settings: settings['max_height'] = 313000 if 'rev_hash_bytes' not in settings: settings['rev_hash_bytes'] = 'false' use_userpass = True use_datadir = False if 'rpcuser' not in settings or 'rpcpassword' not in settings: use_userpass = False if 'datadir' in settings and not use_userpass: use_datadir = True if not use_userpass and not use_datadir: print("Missing datadir or username and/or password in cfg file", file=sys.stderr) sys.exit(1) settings['port'] = int(settings['port']) settings['min_height'] = int(settings['min_height']) settings['max_height'] = int(settings['max_height']) # Force hash byte format setting to be lowercase to make comparisons easier. settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower() # Get the rpc user and pass from the cookie if the datadir is set if use_datadir: get_rpc_cookie() get_block_hashes(settings) diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index a31a1b72b7..9f15911e56 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -1,145 +1,145 @@ #!/usr/bin/env python3 # Copyright (c) 2014-2017 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Script to generate list of seed nodes for chainparams.cpp. This script expects two text files in the directory that is passed as an argument: nodes_main.txt nodes_test.txt These files must consist of lines in the format : [] []: .onion 0xDDBBCCAA (IPv4 little-endian old pnSeeds format) The output will be two data structures with the peers in binary format: static SeedSpec6 pnSeed6_main[]={ ... } static SeedSpec6 pnSeed6_test[]={ ... } These should be pasted into `src/chainparamsseeds.h`. ''' from base64 import b32decode from binascii import a2b_hex import sys import os import re # ipv4 in ipv6 prefix pchIPv4 = bytearray([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff]) # tor-specific ipv6 prefix pchOnionCat = bytearray([0xFD, 0x87, 0xD8, 0x7E, 0xEB, 0x43]) def name_to_ipv6(addr): if len(addr) > 6 and addr.endswith('.onion'): vchAddr = b32decode(addr[0:-6], True) if len(vchAddr) != 16 - len(pchOnionCat): raise ValueError('Invalid onion %s' % s) return pchOnionCat + vchAddr elif '.' in addr: # IPv4 return pchIPv4 + bytearray((int(x) for x in addr.split('.'))) elif ':' in addr: # IPv6 sub = [[], []] # prefix, suffix x = 0 addr = addr.split(':') for i, comp in enumerate(addr): if comp == '': # skip empty component at beginning or end if i == 0 or i == (len(addr) - 1): continue x += 1 # :: skips to suffix assert(x < 2) else: # two bytes per component val = int(comp, 16) sub[x].append(val >> 8) sub[x].append(val & 0xff) nullbytes = 16 - len(sub[0]) - len(sub[1]) assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0)) return bytearray(sub[0] + ([0] * nullbytes) + sub[1]) elif addr.startswith('0x'): # IPv4-in-little-endian return pchIPv4 + bytearray(reversed(a2b_hex(addr[2:]))) else: raise ValueError('Could not parse address %s' % addr) def parse_spec(s, defaultport): match = re.match('\[([0-9a-fA-F:]+)\](?::([0-9]+))?$', s) if match: # ipv6 host = match.group(1) port = match.group(2) elif s.count(':') > 1: # ipv6, no port host = s port = '' else: (host, _, port) = s.partition(':') if not port: port = defaultport else: port = int(port) host = name_to_ipv6(host) return (host, port) def process_nodes(g, f, structname, defaultport): g.write('static SeedSpec6 %s[] = {\n' % structname) first = True for line in f: comment = line.find('#') if comment != -1: line = line[0:comment] line = line.strip() if not line: continue if not first: g.write(',\n') first = False (host, port) = parse_spec(line, defaultport) hoststr = ','.join(('0x%02x' % b) for b in host) g.write(' {{%s}, %i}' % (hoststr, port)) g.write('\n};\n') def main(): if len(sys.argv) < 2: print(('Usage: %s ' % sys.argv[0]), file=sys.stderr) - exit(1) + sys.exit(1) g = sys.stdout indir = sys.argv[1] g.write('#ifndef BITCOIN_CHAINPARAMSSEEDS_H\n') g.write('#define BITCOIN_CHAINPARAMSSEEDS_H\n') g.write('/**\n') g.write(' * List of fixed seed nodes for the bitcoin network\n') g.write(' * @generated by contrib/seeds/generate-seeds.py\n') g.write(' *\n') g.write(' * Each line contains a 16-byte IPv6 address and a port.\n') g.write( ' * IPv4 as well as onion addresses are wrapped inside a IPv6 address accordingly.\n') g.write(' */\n') with open(os.path.join(indir, 'nodes_main.txt'), 'r') as f: process_nodes(g, f, 'pnSeed6_main', 8333) g.write('\n') with open(os.path.join(indir, 'nodes_test.txt'), 'r') as f: process_nodes(g, f, 'pnSeed6_test', 18333) g.write('#endif // BITCOIN_CHAINPARAMSSEEDS_H\n') if __name__ == '__main__': main() diff --git a/share/qt/extract_strings_qt.py b/share/qt/extract_strings_qt.py index fc32a8622d..8dee47a64c 100755 --- a/share/qt/extract_strings_qt.py +++ b/share/qt/extract_strings_qt.py @@ -1,97 +1,97 @@ #!/usr/bin/env python # Copyright (c) 2012-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Extract _("...") strings for translation and convert to Qt stringdefs so that they can be picked up by Qt linguist. ''' from __future__ import division, print_function, unicode_literals from subprocess import Popen, PIPE import operator import os import sys OUT_CPP = "qt/bitcoinstrings.cpp" EMPTY = ['""'] def parse_po(text): """ Parse 'po' format produced by xgettext. Return a list of (msgid,msgstr) tuples. """ messages = [] msgid = [] msgstr = [] in_msgid = False in_msgstr = False for line in text.split('\n'): line = line.rstrip('\r') if line.startswith('msgid '): if in_msgstr: messages.append((msgid, msgstr)) in_msgstr = False # message start in_msgid = True msgid = [line[6:]] elif line.startswith('msgstr '): in_msgid = False in_msgstr = True msgstr = [line[7:]] elif line.startswith('"'): if in_msgid: msgid.append(line) if in_msgstr: msgstr.append(line) if in_msgstr: messages.append((msgid, msgstr)) return messages files = sys.argv[1:] # xgettext -n --keyword=_ $FILES XGETTEXT = os.getenv('XGETTEXT', 'xgettext') if not XGETTEXT: print('Cannot extract strings: xgettext utility is not installed or not configured.', file=sys.stderr) print('Please install package "gettext" and re-run \'./configure\'.', file=sys.stderr) - exit(1) + sys.exit(1) child = Popen([XGETTEXT, '--output=-', '-n', '--keyword=_'] + files, stdout=PIPE) (out, err) = child.communicate() messages = parse_po(out.decode('utf-8')) f = open(OUT_CPP, 'w') f.write(""" #include // Automatically generated by extract_strings_qt.py #ifdef __GNUC__ #define UNUSED __attribute__((unused)) #else #define UNUSED #endif """) f.write('static const char UNUSED *bitcoin_strings[] = {\n') f.write('QT_TRANSLATE_NOOP("bitcoin-core", "%s"),\n' % (os.getenv('PACKAGE_NAME'),)) f.write('QT_TRANSLATE_NOOP("bitcoin-core", "%s"),\n' % (os.getenv('COPYRIGHT_HOLDERS'),)) if os.getenv('COPYRIGHT_HOLDERS_SUBSTITUTION') != os.getenv('PACKAGE_NAME'): f.write('QT_TRANSLATE_NOOP("bitcoin-core", "%s"),\n' % (os.getenv('COPYRIGHT_HOLDERS_SUBSTITUTION'),)) messages.sort(key=operator.itemgetter(0)) for (msgid, msgstr) in messages: if msgid != EMPTY: f.write('QT_TRANSLATE_NOOP("bitcoin-core", %s),\n' % ('\n'.join(msgid))) f.write('};\n') f.close()