diff --git a/.arclint b/.arclint --- a/.arclint +++ b/.arclint @@ -19,7 +19,7 @@ "autopep8": { "type": "autopep8", "version": ">=1.3.4", - "include": "(^contrib/(buildbot|devtools)/.*\\.py$)", + "include": "(^contrib/(buildbot)/.*\\.py$)", "flags": [ "--aggressive", "--ignore=W503,W504", @@ -33,7 +33,6 @@ "exclude": [ "(^contrib/gitian-builder/)", "(^contrib/apple-sdk-tools/)", - "(^contrib/devtools/)", "(^contrib/buildbot/)" ], "flags": [ diff --git a/contrib/devtools/chainparams/generate_chainparams_constants.py b/contrib/devtools/chainparams/generate_chainparams_constants.py --- a/contrib/devtools/chainparams/generate_chainparams_constants.py +++ b/contrib/devtools/chainparams/generate_chainparams_constants.py @@ -2,7 +2,7 @@ # Copyright (c) 2019 The Bitcoin developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' +""" Script to generate list of chainparams constants (ie. assumevalid and minimum chainwork). @@ -20,14 +20,14 @@ The outputted constants should be pasted into `src/chainparamsconstants.h`. -''' +""" import os import sys def process_constants(indir, file_name): - with open(os.path.join(indir, file_name), 'r', encoding="utf8") as f: + with open(os.path.join(indir, file_name), "r", encoding="utf8") as f: constants = f.readlines() # Ensure only the expected number of lines are read from the file @@ -38,12 +38,13 @@ def main(): if len(sys.argv) != 2: - print(f'Usage: {sys.argv[0]} ', file=sys.stderr) + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) sys.exit(1) indir = sys.argv[1] - print('''\ + print( + """\ /** * @{} by contrib/devtools/chainparams/generate_chainparams_constants.py */ @@ -61,13 +62,14 @@ const uint64_t TESTNET_ASSUMED_BLOCKCHAIN_SIZE = {}; const uint64_t TESTNET_ASSUMED_CHAINSTATE_SIZE = {}; }} // namespace ChainParamsConstants -'''.format( - # 'generated' is split out so this file is not identified as generated. - "generated", - *process_constants(indir, 'chainparams_main.txt'), - *process_constants(indir, 'chainparams_test.txt')) +""".format( + # 'generated' is split out so this file is not identified as generated. + "generated", + *process_constants(indir, "chainparams_main.txt"), + *process_constants(indir, "chainparams_test.txt"), + ) ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/contrib/devtools/chainparams/make_chainparams.py b/contrib/devtools/chainparams/make_chainparams.py --- a/contrib/devtools/chainparams/make_chainparams.py +++ b/contrib/devtools/chainparams/make_chainparams.py @@ -10,7 +10,7 @@ import sys from enum import Enum -sys.path.append('../../../test/functional/test_framework') +sys.path.append("../../../test/functional/test_framework") from authproxy import AuthServiceProxy # noqa: E402 @@ -25,28 +25,28 @@ def get_chainparams(rpc_caller, block): # Fetch initial chain info chaininfo = rpc_caller.getblockchaininfo() - if chaininfo['chain'] == 'main': + if chaininfo["chain"] == "main": chain = Chain.MainNet else: chain = Chain.TestNet # Use highest valid chainwork. This doesn't need to match the block hash # used by assume valid. - chainwork = chaininfo['chainwork'] - if not re.match('^[0-9a-z]{64}$', chainwork): + chainwork = chaininfo["chainwork"] + if not re.match("^[0-9a-z]{64}$", chainwork): raise Exception("Chain work is not a valid uint256 hex value.") # Default to N blocks from the chain tip, depending on which chain we're on if not block: - block = chaininfo['blocks'] + block = chaininfo["blocks"] if chain == Chain.MainNet: block -= 10 else: block -= 2000 block = str(block) - if not re.match('^[0-9a-z]{64}$', block): - if re.match('^[0-9]*$', block): + if not re.match("^[0-9a-z]{64}$", block): + if re.match("^[0-9]*$", block): # Fetch block hash using block height block = rpc_caller.getblockhash(int(block)) else: @@ -57,37 +57,54 @@ rpc_caller.getblockheader(block) # Block size on disk (in GB) with some margin for growth - diskSizeBlocks = str( - int(math.ceil(chaininfo['size_on_disk'] / GIGABYTE * 1.3))) + diskSizeBlocks = str(int(math.ceil(chaininfo["size_on_disk"] / GIGABYTE * 1.3))) # Chainstate size on disk (in GB) with some margin for growth utxos = rpc_caller.gettxoutsetinfo() - diskSizeChainstate = str( - int(math.ceil(utxos['disk_size'] / GIGABYTE * 1.3))) + diskSizeChainstate = str(int(math.ceil(utxos["disk_size"] / GIGABYTE * 1.3))) return (block, chainwork, diskSizeBlocks, diskSizeChainstate) def main(args): - return "\n".join(get_chainparams(args['rpc'], args['block'])) + return "\n".join(get_chainparams(args["rpc"], args["block"])) if __name__ == "__main__": - parser = argparse.ArgumentParser(description=( - "Make chainparams file.\n" - "Prerequisites: RPC access to a bitcoind node.\n\n"), - formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument('--address', '-a', default="127.0.0.1:8332", - help="Node address for making RPC calls.\n" - "The chain (MainNet or TestNet) will be automatically detected.\n" - "Default: '127.0.0.1:8332'") - parser.add_argument('--block', '-b', - help="The block hash or height to use for fetching chainparams.\n" - "MainNet default: 10 blocks from the chain tip." - "TestNet default: 2000 blocks from the chain tip.") - parser.add_argument('--config', '-c', default="~/.bitcoin/bitcoin.conf", - help="Path to bitcoin.conf for RPC authentication arguments (rpcuser & rpcpassword).\n" - "Default: ~/.bitcoin/bitcoin.conf") + parser = argparse.ArgumentParser( + description=( + "Make chainparams file.\nPrerequisites: RPC access to a bitcoind node.\n\n" + ), + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument( + "--address", + "-a", + default="127.0.0.1:8332", + help=( + "Node address for making RPC calls.\n" + "The chain (MainNet or TestNet) will be automatically detected.\n" + "Default: '127.0.0.1:8332'" + ), + ) + parser.add_argument( + "--block", + "-b", + help=( + "The block hash or height to use for fetching chainparams.\n" + "MainNet default: 10 blocks from the chain tip." + "TestNet default: 2000 blocks from the chain tip." + ), + ) + parser.add_argument( + "--config", + "-c", + default="~/.bitcoin/bitcoin.conf", + help=( + "Path to bitcoin.conf for RPC authentication arguments (rpcuser &" + " rpcpassword).\nDefault: ~/.bitcoin/bitcoin.conf" + ), + ) args = parser.parse_args() args.config = os.path.expanduser(args.config) @@ -95,7 +112,7 @@ user = None password = None if os.path.isfile(args.config): - with open(args.config, 'r', encoding='utf8') as f: + with open(args.config, "r", encoding="utf8") as f: for line in f: if line.startswith("rpcuser="): # Ensure that there is only one rpcuser line @@ -113,7 +130,8 @@ raise ValueError("Config is missing rpcpassword") args.rpc = AuthServiceProxy( - service_url=f'http://{user}:{password}@{args.address}', timeout=1200) + service_url=f"http://{user}:{password}@{args.address}", timeout=1200 + ) output = main(vars(args)) if output: print(output) diff --git a/contrib/devtools/chainparams/test_make_chainparams.py b/contrib/devtools/chainparams/test_make_chainparams.py --- a/contrib/devtools/chainparams/test_make_chainparams.py +++ b/contrib/devtools/chainparams/test_make_chainparams.py @@ -9,8 +9,17 @@ class MockRPC: - def __init__(self, test, chain, numBlocks, expectedBlock, - blockHash, chainWork, blockchainSize, chainstateSize): + def __init__( + self, + test, + chain, + numBlocks, + expectedBlock, + blockHash, + chainWork, + blockchainSize, + chainstateSize, + ): self.test = test self.chain = chain self.numBlocks = numBlocks @@ -25,7 +34,9 @@ "chain": self.chain, "blocks": self.numBlocks, "headers": self.numBlocks, - "bestblockhash": "0000000000000000039c96605d7fca74a5e185ea5634198346e9e07fd235b666", + "bestblockhash": ( + "0000000000000000039c96605d7fca74a5e185ea5634198346e9e07fd235b666" + ), "difficulty": 274412074285.6605, "mediantime": 1562168718, "verificationprogress": 0.9999958005632363, @@ -42,40 +53,60 @@ # Tests should always request the right block height. Even though a # real node will rarely raise an exception for this call, we are # more strict during testing. - self.test.assertEqual(block, self.expectedBlock, "Called 'getblockhash {}' when expected was 'getblockhash {}'".format( - block, self.expectedBlock)) + self.test.assertEqual( + block, + self.expectedBlock, + "Called 'getblockhash {}' when expected was 'getblockhash {}'".format( + block, self.expectedBlock + ), + ) return self.blockHash def getblockheader(self, blockHash): # Make sure to raise an exception in the same way a real node would # when calling 'getblockheader' on a block hash that is not part of # the chain. - self.test.assertEqual(blockHash, self.blockHash, "Called 'getblockheader {}' when expected was 'getblockheader {}'".format( - blockHash, self.blockHash)) + self.test.assertEqual( + blockHash, + self.blockHash, + "Called 'getblockheader {}' when expected was 'getblockheader {}'".format( + blockHash, self.blockHash + ), + ) return { "hash": blockHash, "confirmations": 1, "height": 591463, "version": 536870912, "versionHex": "20000000", - "merkleroot": "51c898f034b6c5a5513a7c35912e86d009188311e550bb3096e04afb11f40aba", + "merkleroot": ( + "51c898f034b6c5a5513a7c35912e86d009188311e550bb3096e04afb11f40aba" + ), "time": 1563212034, "mediantime": 1563208994, "nonce": 3501699724, "bits": "18040cd6", "difficulty": 271470800310.0635, - "chainwork": "000000000000000000000000000000000000000000f4c5e639fa012518a48a57", - "previousblockhash": "00000000000000000307b45e4a6cf8d49e70b9012ea1d72a5ce334a4213f66bd", + "chainwork": ( + "000000000000000000000000000000000000000000f4c5e639fa012518a48a57" + ), + "previousblockhash": ( + "00000000000000000307b45e4a6cf8d49e70b9012ea1d72a5ce334a4213f66bd" + ), } def gettxoutsetinfo(self): return { "height": 636013, - "bestblock": "00000000000000000250a6ab6c6c4778086807f5b39910a8c108efa511282280", + "bestblock": ( + "00000000000000000250a6ab6c6c4778086807f5b39910a8c108efa511282280" + ), "transactions": 19360831, "txouts": 42145889, "bogosize": 3187119531, - "hash_serialized": "1b1cc457771e8b6f849ac21c4da43ebe5c614df9e61a943252978437ad774ce5", + "hash_serialized": ( + "1b1cc457771e8b6f849ac21c4da43ebe5c614df9e61a943252978437ad774ce5" + ), "disk_size": self.chainstateSize, "total_amount": 18412423.42452419, } @@ -84,8 +115,18 @@ class MockFailRPC(MockRPC): # Provides a fail counter to fail after the Nth RPC command - def __init__(self, test, chain, numBlocks, expectedBlock, blockHash, - chainWork, blockchainSize, chainstateSize, failCounter): + def __init__( + self, + test, + chain, + numBlocks, + expectedBlock, + blockHash, + chainWork, + blockchainSize, + chainstateSize, + failCounter, + ): super().__init__( test, chain, @@ -94,7 +135,8 @@ blockHash, chainWork, blockchainSize, - chainstateSize) + chainstateSize, + ) self.failCounter = failCounter def checkFailCounter(self): @@ -121,7 +163,7 @@ return super().gettxoutsetinfo() -def CheckMockFailure(test, args, errorMessage='error code: -99'): +def CheckMockFailure(test, args, errorMessage="error code: -99"): with test.assertRaises(Exception) as context: GenerateChainParams(args) test.assertIn(errorMessage, str(context.exception)) @@ -131,89 +173,128 @@ maxDiff = None def setUp(self): - self.blockHash1 = '0000000000000000003ef673ae12bc6017481830d37b9c52ce1e79c080e812b8' - self.chainWork1 = '000000000000000000000000000000000000000000f2537ccf2e07bbe15e70e1' - self.blockHash2 = '0000000000000298a9fa227f0ec32f2b7585f3e64c8b3369e7f8b4fd8ea3d836' - self.chainWork2 = '00000000000000000000000000000000000000000000004fdb4795a837f19671' + self.blockHash1 = ( + "0000000000000000003ef673ae12bc6017481830d37b9c52ce1e79c080e812b8" + ) + self.chainWork1 = ( + "000000000000000000000000000000000000000000f2537ccf2e07bbe15e70e1" + ) + self.blockHash2 = ( + "0000000000000298a9fa227f0ec32f2b7585f3e64c8b3369e7f8b4fd8ea3d836" + ) + self.chainWork2 = ( + "00000000000000000000000000000000000000000000004fdb4795a837f19671" + ) def test_happy_path_mainnet(self): mockRPC = MockRPC( test=self, - chain='main', + chain="main", numBlocks=123000, expectedBlock=122990, blockHash=self.blockHash1, chainWork=self.chainWork1, blockchainSize=160111222333, - chainstateSize=2000111222) + chainstateSize=2000111222, + ) args = { - 'rpc': mockRPC, - 'block': None, + "rpc": mockRPC, + "block": None, } - self.assertEqual(GenerateChainParams(args), "\n".join([ - "0000000000000000003ef673ae12bc6017481830d37b9c52ce1e79c080e812b8", - "000000000000000000000000000000000000000000f2537ccf2e07bbe15e70e1", - "194", "3"])) + self.assertEqual( + GenerateChainParams(args), + "\n".join( + [ + "0000000000000000003ef673ae12bc6017481830d37b9c52ce1e79c080e812b8", + "000000000000000000000000000000000000000000f2537ccf2e07bbe15e70e1", + "194", + "3", + ] + ), + ) def test_happy_path_testnet(self): mockRPC = MockRPC( test=self, - chain='test', + chain="test", numBlocks=234000, expectedBlock=232000, blockHash=self.blockHash1, chainWork=self.chainWork1, blockchainSize=50111222333, - chainstateSize=1000111222) + chainstateSize=1000111222, + ) args = { - 'rpc': mockRPC, - 'block': None, + "rpc": mockRPC, + "block": None, } - self.assertEqual(GenerateChainParams(args), "\n".join([ - "0000000000000000003ef673ae12bc6017481830d37b9c52ce1e79c080e812b8", - "000000000000000000000000000000000000000000f2537ccf2e07bbe15e70e1", - "61", "2"])) + self.assertEqual( + GenerateChainParams(args), + "\n".join( + [ + "0000000000000000003ef673ae12bc6017481830d37b9c52ce1e79c080e812b8", + "000000000000000000000000000000000000000000f2537ccf2e07bbe15e70e1", + "61", + "2", + ] + ), + ) def test_specific_block(self): mockRPC = MockRPC( test=self, - chain='main', + chain="main", numBlocks=123000, expectedBlock=122990, blockHash=self.blockHash1, chainWork=self.chainWork1, blockchainSize=160111222333, - chainstateSize=2000111222) + chainstateSize=2000111222, + ) args = { - 'rpc': mockRPC, - 'block': self.blockHash1, + "rpc": mockRPC, + "block": self.blockHash1, } - self.assertEqual(GenerateChainParams(args), "\n".join([ - "0000000000000000003ef673ae12bc6017481830d37b9c52ce1e79c080e812b8", - "000000000000000000000000000000000000000000f2537ccf2e07bbe15e70e1", - "194", "3"])) + self.assertEqual( + GenerateChainParams(args), + "\n".join( + [ + "0000000000000000003ef673ae12bc6017481830d37b9c52ce1e79c080e812b8", + "000000000000000000000000000000000000000000f2537ccf2e07bbe15e70e1", + "194", + "3", + ] + ), + ) def test_wrong_chain(self): mockRPC = MockRPC( test=self, - chain='main', + chain="main", numBlocks=123000, expectedBlock=122990, blockHash=self.blockHash1, chainWork=self.chainWork1, blockchainSize=160111222333, - chainstateSize=2000111222) + chainstateSize=2000111222, + ) args = { - 'rpc': mockRPC, - 'block': self.blockHash2, + "rpc": mockRPC, + "block": self.blockHash2, } CheckMockFailure( - self, args, "expected was 'getblockheader 0000000000000000003ef673ae12bc6017481830d37b9c52ce1e79c080e812b8'") + self, + args, + ( + "expected was 'getblockheader" + " 0000000000000000003ef673ae12bc6017481830d37b9c52ce1e79c080e812b8'" + ), + ) def test_bitcoin_cli_failures_testnet(self): - for chain in ['main', 'test']: + for chain in ["main", "test"]: expectedBlock = 133990 - if chain == 'test': + if chain == "test": expectedBlock = 132000 for failCounter in range(4): @@ -226,10 +307,11 @@ chainWork=self.chainWork1, failCounter=failCounter, blockchainSize=160111222333, - chainstateSize=2000111222) + chainstateSize=2000111222, + ) argsFail = { - 'rpc': mockFailRPC, - 'block': None, + "rpc": mockFailRPC, + "block": None, } CheckMockFailure(self, argsFail) diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -8,16 +8,14 @@ from typing import Dict, List, Set MAPPING = { - 'core_read.cpp': 'core_io.cpp', - 'core_write.cpp': 'core_io.cpp', + "core_read.cpp": "core_io.cpp", + "core_write.cpp": "core_io.cpp", } # Directories with header-based modules, where the assumption that .cpp files # define functions and variables declared in corresponding .h files is # incorrect. -HEADER_MODULE_PATHS = [ - 'interfaces/' -] +HEADER_MODULE_PATHS = ["interfaces/"] def module_name(path): @@ -52,13 +50,17 @@ # TODO: implement support for multiple include directories for arg in sorted(files.keys()): module = files[arg] - with open(arg, 'r', encoding="utf8") as f: + with open(arg, "r", encoding="utf8") as f: for line in f: match = RE.match(line) if match: include = match.group(1) included_module = module_name(include) - if included_module is not None and included_module in deps and included_module != module: + if ( + included_module is not None + and included_module in deps + and included_module != module + ): deps[module].add(included_module) # Loop to find the shortest (remaining) circular dependency @@ -79,8 +81,9 @@ break # If module is in its own transitive closure, it's a circular # dependency; check if it is the shortest - if module in closure and (shortest_cycle is None or len( - closure[module]) + 1 < len(shortest_cycle)): + if module in closure and ( + shortest_cycle is None or len(closure[module]) + 1 < len(shortest_cycle) + ): shortest_cycle = [module] + closure[module] if shortest_cycle is None: break diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -17,19 +17,18 @@ EXCLUDE = [ # auto generated: - 'src/qt/bitcoinstrings.cpp', - 'src/chainparamsseeds.h', + "src/qt/bitcoinstrings.cpp", + "src/chainparamsseeds.h", # other external copyrights: - 'src/reverse_iterator.h', - 'src/test/fuzz/FuzzedDataProvider.h', - 'src/tinyformat.h', - 'src/bench/nanobench.h', - 'test/functional/test_framework/bignum.py', + "src/reverse_iterator.h", + "src/test/fuzz/FuzzedDataProvider.h", + "src/tinyformat.h", + "src/bench/nanobench.h", + "test/functional/test_framework/bignum.py", # python init: - '*__init__.py', + "*__init__.py", ] -EXCLUDE_COMPILED = re.compile( - '|'.join([fnmatch.translate(m) for m in EXCLUDE])) +EXCLUDE_COMPILED = re.compile("|".join([fnmatch.translate(m) for m in EXCLUDE])) EXCLUDE_DIRS = [ # git subtrees @@ -39,31 +38,31 @@ "src/univalue/", ] -INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.mm', '*.py', '*.sh', - '*.bash-completion'] -INCLUDE_COMPILED = re.compile( - '|'.join([fnmatch.translate(m) for m in INCLUDE])) +INCLUDE = ["*.h", "*.cpp", "*.cc", "*.c", "*.mm", "*.py", "*.sh", "*.bash-completion"] +INCLUDE_COMPILED = re.compile("|".join([fnmatch.translate(m) for m in INCLUDE])) def applies_to_file(filename): for excluded_dir in EXCLUDE_DIRS: if filename.startswith(excluded_dir): return False - return ((EXCLUDE_COMPILED.match(filename) is None) - and (INCLUDE_COMPILED.match(filename) is not None)) + return (EXCLUDE_COMPILED.match(filename) is None) and ( + INCLUDE_COMPILED.match(filename) is not None + ) + ########################################################################## # obtain list of files in repo according to INCLUDE and EXCLUDE ########################################################################## -GIT_LS_CMD = 'git ls-files --full-name'.split(' ') -GIT_TOPLEVEL_CMD = 'git rev-parse --show-toplevel'.split(' ') +GIT_LS_CMD = "git ls-files --full-name".split(" ") +GIT_TOPLEVEL_CMD = "git rev-parse --show-toplevel".split(" ") def call_git_ls(base_directory): out = subprocess.check_output([*GIT_LS_CMD, base_directory]) - return [f for f in out.decode("utf-8").split('\n') if f != ''] + return [f for f in out.decode("utf-8").split("\n") if f != ""] def call_git_toplevel(): @@ -75,30 +74,35 @@ "Returns an array of absolute paths to any project files in the base_directory that pass the include/exclude filters" root = call_git_toplevel() filenames = call_git_ls(base_directory) - return sorted([os.path.join(root, filename) for filename in filenames if - applies_to_file(filename)]) + return sorted( + [ + os.path.join(root, filename) + for filename in filenames + if applies_to_file(filename) + ] + ) + ########################################################################## # define and compile regexes for the patterns we are looking for ########################################################################## -COPYRIGHT_WITH_C = r'Copyright \(c\)' -COPYRIGHT_WITHOUT_C = 'Copyright' -ANY_COPYRIGHT_STYLE = f'({COPYRIGHT_WITH_C}|{COPYRIGHT_WITHOUT_C})' +COPYRIGHT_WITH_C = r"Copyright \(c\)" +COPYRIGHT_WITHOUT_C = "Copyright" +ANY_COPYRIGHT_STYLE = f"({COPYRIGHT_WITH_C}|{COPYRIGHT_WITHOUT_C})" YEAR = "20[0-9][0-9]" -YEAR_RANGE = f'({YEAR})(-{YEAR})?' -YEAR_LIST = f'({YEAR})(, {YEAR})+' -ANY_YEAR_STYLE = f'({YEAR_RANGE}|{YEAR_LIST})' -ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE = ( - f"{ANY_COPYRIGHT_STYLE} {ANY_YEAR_STYLE}") +YEAR_RANGE = f"({YEAR})(-{YEAR})?" +YEAR_LIST = f"({YEAR})(, {YEAR})+" +ANY_YEAR_STYLE = f"({YEAR_RANGE}|{YEAR_LIST})" +ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE = f"{ANY_COPYRIGHT_STYLE} {ANY_YEAR_STYLE}" ANY_COPYRIGHT_COMPILED = re.compile(ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE) def compile_copyright_regex(copyright_style, year_style, name): - return re.compile(f'{copyright_style} {year_style},? {name}') + return re.compile(f"{copyright_style} {year_style},? {name}") EXPECTED_HOLDER_NAMES = [ @@ -125,13 +129,15 @@ WITHOUT_C_STYLE_COMPILED = {} for holder_name in EXPECTED_HOLDER_NAMES: - DOMINANT_STYLE_COMPILED[holder_name] = ( - compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_RANGE, holder_name)) - YEAR_LIST_STYLE_COMPILED[holder_name] = ( - compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_LIST, holder_name)) - WITHOUT_C_STYLE_COMPILED[holder_name] = ( - compile_copyright_regex(COPYRIGHT_WITHOUT_C, ANY_YEAR_STYLE, - holder_name)) + DOMINANT_STYLE_COMPILED[holder_name] = compile_copyright_regex( + COPYRIGHT_WITH_C, YEAR_RANGE, holder_name + ) + YEAR_LIST_STYLE_COMPILED[holder_name] = compile_copyright_regex( + COPYRIGHT_WITH_C, YEAR_LIST, holder_name + ) + WITHOUT_C_STYLE_COMPILED[holder_name] = compile_copyright_regex( + COPYRIGHT_WITHOUT_C, ANY_YEAR_STYLE, holder_name + ) ########################################################################## # search file contents for copyright message of particular category @@ -156,47 +162,52 @@ match = WITHOUT_C_STYLE_COMPILED[holder_name].search(contents) return match is not None + ########################################################################## # get file info ########################################################################## def read_file(filename): - return open(filename, 'r', encoding="utf8").read() + return open(filename, "r", encoding="utf8").read() def gather_file_info(filename): info = {} - info['filename'] = filename + info["filename"] = filename c = read_file(filename) - info['contents'] = c + info["contents"] = c - info['all_copyrights'] = get_count_of_copyrights_of_any_style_any_holder(c) + info["all_copyrights"] = get_count_of_copyrights_of_any_style_any_holder(c) - info['classified_copyrights'] = 0 - info['dominant_style'] = {} - info['year_list_style'] = {} - info['without_c_style'] = {} + info["classified_copyrights"] = 0 + info["dominant_style"] = {} + info["year_list_style"] = {} + info["without_c_style"] = {} for holder_name in EXPECTED_HOLDER_NAMES: - has_dominant_style = ( - file_has_dominant_style_copyright_for_holder(c, holder_name)) - has_year_list_style = ( - file_has_year_list_style_copyright_for_holder(c, holder_name)) - has_without_c_style = ( - file_has_without_c_style_copyright_for_holder(c, holder_name)) - info['dominant_style'][holder_name] = has_dominant_style - info['year_list_style'][holder_name] = has_year_list_style - info['without_c_style'][holder_name] = has_without_c_style + has_dominant_style = file_has_dominant_style_copyright_for_holder( + c, holder_name + ) + has_year_list_style = file_has_year_list_style_copyright_for_holder( + c, holder_name + ) + has_without_c_style = file_has_without_c_style_copyright_for_holder( + c, holder_name + ) + info["dominant_style"][holder_name] = has_dominant_style + info["year_list_style"][holder_name] = has_year_list_style + info["without_c_style"][holder_name] = has_without_c_style if has_dominant_style or has_year_list_style or has_without_c_style: - info['classified_copyrights'] = info['classified_copyrights'] + 1 + info["classified_copyrights"] = info["classified_copyrights"] + 1 return info + ########################################################################## # report execution ########################################################################## -SEPARATOR = '-'.join(['' for _ in range(80)]) +SEPARATOR = "-".join(["" for _ in range(80)]) def print_filenames(filenames, verbose): @@ -208,72 +219,91 @@ def print_report(file_infos, verbose): print(SEPARATOR) - examined = [i['filename'] for i in file_infos] - print("{} files examined according to INCLUDE and EXCLUDE fnmatch rules".format( - len(examined))) + examined = [i["filename"] for i in file_infos] + print( + f"{len(examined)} files examined according to INCLUDE and EXCLUDE fnmatch rules" + ) print_filenames(examined, verbose) print(SEPARATOR) - print('') - zero_copyrights = [i['filename'] for i in file_infos if - i['all_copyrights'] == 0] + print("") + zero_copyrights = [i["filename"] for i in file_infos if i["all_copyrights"] == 0] print(f"{len(zero_copyrights):4d} with zero copyrights") print_filenames(zero_copyrights, verbose) - one_copyright = [i['filename'] for i in file_infos if - i['all_copyrights'] == 1] + one_copyright = [i["filename"] for i in file_infos if i["all_copyrights"] == 1] print(f"{len(one_copyright):4d} with one copyright") print_filenames(one_copyright, verbose) - two_copyrights = [i['filename'] for i in file_infos if - i['all_copyrights'] == 2] + two_copyrights = [i["filename"] for i in file_infos if i["all_copyrights"] == 2] print(f"{len(two_copyrights):4d} with two copyrights") print_filenames(two_copyrights, verbose) - three_copyrights = [i['filename'] for i in file_infos if - i['all_copyrights'] == 3] + three_copyrights = [i["filename"] for i in file_infos if i["all_copyrights"] == 3] print(f"{len(three_copyrights):4d} with three copyrights") print_filenames(three_copyrights, verbose) - four_or_more_copyrights = [i['filename'] for i in file_infos if - i['all_copyrights'] >= 4] + four_or_more_copyrights = [ + i["filename"] for i in file_infos if i["all_copyrights"] >= 4 + ] print(f"{len(four_or_more_copyrights):4d} with four or more copyrights") print_filenames(four_or_more_copyrights, verbose) - print('') + print("") print(SEPARATOR) - print('Copyrights with dominant style:\ne.g. "Copyright (c)" and ' - '"" or "-":\n') + print( + 'Copyrights with dominant style:\ne.g. "Copyright (c)" and ' + '"" or "-":\n' + ) for holder_name in EXPECTED_HOLDER_NAMES: - dominant_style = [i['filename'] for i in file_infos if - i['dominant_style'][holder_name]] + dominant_style = [ + i["filename"] for i in file_infos if i["dominant_style"][holder_name] + ] if len(dominant_style) > 0: - print("{:4d} with '{}'".format( - len(dominant_style), holder_name.replace('\n', '\\n'))) + print( + "{:4d} with '{}'".format( + len(dominant_style), holder_name.replace("\n", "\\n") + ) + ) print_filenames(dominant_style, verbose) - print('') + print("") print(SEPARATOR) - print('Copyrights with year list style:\ne.g. "Copyright (c)" and ' - '", , ...":\n') + print( + 'Copyrights with year list style:\ne.g. "Copyright (c)" and ' + '", , ...":\n' + ) for holder_name in EXPECTED_HOLDER_NAMES: - year_list_style = [i['filename'] for i in file_infos if - i['year_list_style'][holder_name]] + year_list_style = [ + i["filename"] for i in file_infos if i["year_list_style"][holder_name] + ] if len(year_list_style) > 0: - print("{:4d} with '{}'".format( - len(year_list_style), holder_name.replace('\n', '\\n'))) + print( + "{:4d} with '{}'".format( + len(year_list_style), holder_name.replace("\n", "\\n") + ) + ) print_filenames(year_list_style, verbose) - print('') + print("") print(SEPARATOR) - print('Copyrights with no "(c)" style:\ne.g. "Copyright" and "" or ' - '"-":\n') + print( + 'Copyrights with no "(c)" style:\ne.g. "Copyright" and "" or ' + '"-":\n' + ) for holder_name in EXPECTED_HOLDER_NAMES: - without_c_style = [i['filename'] for i in file_infos if - i['without_c_style'][holder_name]] + without_c_style = [ + i["filename"] for i in file_infos if i["without_c_style"][holder_name] + ] if len(without_c_style) > 0: - print("{:4d} with '{}'".format( - len(without_c_style), holder_name.replace('\n', '\\n'))) + print( + "{:4d} with '{}'".format( + len(without_c_style), holder_name.replace("\n", "\\n") + ) + ) print_filenames(without_c_style, verbose) - print('') + print("") print(SEPARATOR) - unclassified_copyrights = [i['filename'] for i in file_infos if - i['classified_copyrights'] < i['all_copyrights']] + unclassified_copyrights = [ + i["filename"] + for i in file_infos + if i["classified_copyrights"] < i["all_copyrights"] + ] print(f"{len(unclassified_copyrights)} with unexpected copyright holder names") print_filenames(unclassified_copyrights, verbose) print(SEPARATOR) @@ -284,6 +314,7 @@ file_infos = [gather_file_info(f) for f in filenames] print_report(file_infos, verbose) + ########################################################################## # report cmd ########################################################################## @@ -312,13 +343,14 @@ if len(argv) == 3: verbose = False - elif argv[3] == 'verbose': + elif argv[3] == "verbose": verbose = True else: sys.exit(f"*** unknown argument: {argv[2]}") exec_report(base_directory, verbose) + ########################################################################## # query git for year of last change ########################################################################## @@ -328,8 +360,8 @@ def call_git_log(filename): - out = subprocess.check_output((GIT_LOG_CMD.format(filename)).split(' ')) - return out.decode("utf-8").split('\n') + out = subprocess.check_output((GIT_LOG_CMD.format(filename)).split(" ")) + return out.decode("utf-8").split("\n") def get_git_change_years(filename): @@ -337,43 +369,45 @@ if len(git_log_lines) == 0: return [datetime.date.today().year] # timestamp is in ISO 8601 format. e.g. "2016-09-05 14:25:32 -0600" - return [line.split(' ')[0].split('-')[0] for line in git_log_lines] + return [line.split(" ")[0].split("-")[0] for line in git_log_lines] def get_most_recent_git_change_year(filename): return max(get_git_change_years(filename)) + ########################################################################## # read and write to file ########################################################################## def read_file_lines(filename): - f = open(filename, 'r', encoding="utf8") + f = open(filename, "r", encoding="utf8") file_lines = f.readlines() f.close() return file_lines def write_file_lines(filename, file_lines): - f = open(filename, 'w', encoding="utf8") - f.write(''.join(file_lines)) + f = open(filename, "w", encoding="utf8") + f.write("".join(file_lines)) f.close() + ########################################################################## # update header years execution ########################################################################## -COPYRIGHT = r'Copyright \(c\)' +COPYRIGHT = r"Copyright \(c\)" YEAR = "20[0-9][0-9]" -YEAR_RANGE = f'({YEAR})(-{YEAR})?' -HOLDER = 'The Bitcoin developers' -UPDATEABLE_LINE_COMPILED = re.compile( - ' '.join([COPYRIGHT, YEAR_RANGE, HOLDER])) +YEAR_RANGE = f"({YEAR})(-{YEAR})?" +HOLDER = "The Bitcoin developers" +UPDATEABLE_LINE_COMPILED = re.compile(" ".join([COPYRIGHT, YEAR_RANGE, HOLDER])) DISTRIBUTION_LINE = re.compile( - r"Distributed under the MIT software license, see the accompanying") + r"Distributed under the MIT software license, see the accompanying" +) def get_updatable_copyright_line(file_lines): @@ -395,7 +429,7 @@ def parse_year_range(year_range): - year_split = year_range.split('-') + year_split = year_range.split("-") start_year = year_split[0] if len(year_split) == 1: return start_year, start_year @@ -409,21 +443,25 @@ def create_updated_copyright_line(line, last_git_change_year): - copyright_splitter = 'Copyright (c) ' + copyright_splitter = "Copyright (c) " copyright_split = line.split(copyright_splitter) # Preserve characters on line that are ahead of the start of the copyright # notice - they are part of the comment block and vary from file-to-file. before_copyright = copyright_split[0] after_copyright = copyright_split[1] - space_split = after_copyright.split(' ') + space_split = after_copyright.split(" ") year_range = space_split[0] start_year, end_year = parse_year_range(year_range) if end_year == last_git_change_year: return line - return (before_copyright + copyright_splitter - + year_range_to_str(start_year, last_git_change_year) + ' ' - + ' '.join(space_split[1:])) + return ( + before_copyright + + copyright_splitter + + year_range_to_str(start_year, last_git_change_year) + + " " + + " ".join(space_split[1:]) + ) def update_updatable_copyright(filename): @@ -439,14 +477,14 @@ return file_lines[index] = new_line write_file_lines(filename, file_lines) - print_file_action_message(filename, - f"Copyright updated! -> {last_git_change_year}") + print_file_action_message(filename, f"Copyright updated! -> {last_git_change_year}") def exec_update_header_year(base_directory): for filename in get_filenames_to_examine(base_directory): update_updatable_copyright(filename) + ########################################################################## # update cmd ########################################################################## @@ -495,38 +533,40 @@ sys.exit(f"*** bad base_directory: {base_directory}") exec_update_header_year(base_directory) + ########################################################################## # inserted copyright header format ########################################################################## def get_header_lines(header, start_year, end_year): - lines = header.split('\n')[1:-1] + lines = header.split("\n")[1:-1] lines[0] = lines[0].format(year_range_to_str(start_year, end_year)) - return [line + '\n' for line in lines] + return [line + "\n" for line in lines] -CPP_HEADER = ''' +CPP_HEADER = """ // Copyright (c) {} The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' +""" def get_cpp_header_lines_to_insert(start_year, end_year): return reversed(get_header_lines(CPP_HEADER, start_year, end_year)) -SCRIPT_HEADER = ''' +SCRIPT_HEADER = """ # Copyright (c) {} The Bitcoin developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' +""" def get_script_header_lines_to_insert(start_year, end_year): return reversed(get_header_lines(SCRIPT_HEADER, start_year, end_year)) + ########################################################################## # query git for year of last change ########################################################################## @@ -536,6 +576,7 @@ years = get_git_change_years(filename) return min(years), max(years) + ########################################################################## # check for existing ABC copyright ########################################################################## @@ -545,6 +586,7 @@ index, _ = get_updatable_copyright_line(file_lines) return index is not None + ########################################################################## # insert header execution ########################################################################## @@ -555,7 +597,7 @@ return False if len(file_lines[0]) <= 2: return False - return file_lines[0][:2] == '#!' + return file_lines[0][:2] == "#!" def insert_script_header(filename, file_lines, start_year, end_year): @@ -574,7 +616,7 @@ def insert_cpp_header(filename, file_lines, start_year, end_year): - file_lines.insert(0, '\n') + file_lines.insert(0, "\n") header_lines = get_cpp_header_lines_to_insert(start_year, end_year) insert_idx = find_distribution_line_index(file_lines) if insert_idx is not None: @@ -588,13 +630,14 @@ def exec_insert_header(filename, style): file_lines = read_file_lines(filename) if file_already_has_bitcoin_copyright(file_lines): - sys.exit(f'*** {filename} already has a copyright by The Bitcoin developers') + sys.exit(f"*** {filename} already has a copyright by The Bitcoin developers") start_year, end_year = get_git_change_year_range(filename) - if style in ['python', 'shell']: + if style in ["python", "shell"]: insert_script_header(filename, file_lines, start_year, end_year) else: insert_cpp_header(filename, file_lines, start_year, end_year) + ########################################################################## # insert cmd ########################################################################## @@ -634,17 +677,18 @@ if not os.path.isfile(filename): sys.exit(f"*** bad filename: {filename}") _, extension = os.path.splitext(filename) - if extension not in ['.h', '.cpp', '.cc', '.c', '.py', '.sh']: + if extension not in [".h", ".cpp", ".cc", ".c", ".py", ".sh"]: sys.exit(f"*** cannot insert for file extension {extension}") - if extension == '.py': - style = 'python' - elif extension == '.sh': - style = 'shell' + if extension == ".py": + style = "python" + elif extension == ".sh": + style = "shell" else: - style = 'cpp' + style = "cpp" exec_insert_header(filename, style) + ########################################################################## # UI ########################################################################## @@ -665,7 +709,7 @@ To see subcommand usage, run them without arguments. """ -SUBCOMMANDS = ['report', 'update', 'insert'] +SUBCOMMANDS = ["report", "update", "insert"] if __name__ == "__main__": if len(sys.argv) == 1: @@ -673,9 +717,9 @@ subcommand = sys.argv[1] if subcommand not in SUBCOMMANDS: sys.exit(USAGE) - if subcommand == 'report': + if subcommand == "report": report_cmd(sys.argv) - elif subcommand == 'update': + elif subcommand == "update": update_cmd(sys.argv) - elif subcommand == 'insert': + elif subcommand == "insert": insert_cmd(sys.argv) diff --git a/contrib/devtools/optimize-pngs.py b/contrib/devtools/optimize-pngs.py --- a/contrib/devtools/optimize-pngs.py +++ b/contrib/devtools/optimize-pngs.py @@ -2,10 +2,10 @@ # Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' +""" Run this script every time you change one of the png files. Using pngcrush, it will optimize the png files, remove various color profiles, remove ancillary chunks (alla) and text chunks (text). #pngcrush -brute -ow -rem gAMA -rem cHRM -rem iCCP -rem sRGB -rem alla -rem text -''' +""" import hashlib import os import subprocess @@ -16,24 +16,25 @@ def file_hash(filename): - '''Return hash of raw file contents''' - with open(filename, 'rb') as f: + """Return hash of raw file contents""" + with open(filename, "rb") as f: return hashlib.sha256(f.read()).hexdigest() def content_hash(filename): - '''Return hash of RGBA contents of image''' + """Return hash of RGBA contents of image""" i = Image.open(filename) - i = i.convert('RGBA') + i = i.convert("RGBA") data = i.tobytes() return hashlib.sha256(data).hexdigest() -pngcrush = 'pngcrush' -git = 'git' +pngcrush = "pngcrush" +git = "git" folders = ["src/qt/res/animation", "src/qt/res/icons", "share/pixmaps"] basePath = subprocess.check_output( - [git, 'rev-parse', '--show-toplevel'], universal_newlines=True, encoding='utf8').rstrip('\n') + [git, "rev-parse", "--show-toplevel"], universal_newlines=True, encoding="utf8" +).rstrip("\n") totalSaveBytes = 0 noHashChange = True @@ -42,47 +43,95 @@ absFolder = os.path.join(basePath, folder) for file in os.listdir(absFolder): extension = os.path.splitext(file)[1] - if extension.lower() == '.png': - print(f"optimizing {file}...", end=' ') + if extension.lower() == ".png": + print(f"optimizing {file}...", end=" ") file_path = os.path.join(absFolder, file) - fileMetaMap = {'file': file, 'osize': os.path.getsize( - file_path), 'sha256Old': file_hash(file_path)} - fileMetaMap['contentHashPre'] = content_hash(file_path) + fileMetaMap = { + "file": file, + "osize": os.path.getsize(file_path), + "sha256Old": file_hash(file_path), + } + fileMetaMap["contentHashPre"] = content_hash(file_path) try: - subprocess.call([pngcrush, "-brute", "-ow", "-rem", "gAMA", "-rem", "cHRM", "-rem", "iCCP", "-rem", "sRGB", - "-rem", "alla", "-rem", "text", file_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.call( + [ + pngcrush, + "-brute", + "-ow", + "-rem", + "gAMA", + "-rem", + "cHRM", + "-rem", + "iCCP", + "-rem", + "sRGB", + "-rem", + "alla", + "-rem", + "text", + file_path, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) except OSError: print("pngcrush is not installed, aborting...") sys.exit(0) # verify if "Not a PNG file" in subprocess.check_output( - [pngcrush, "-n", "-v", file_path], stderr=subprocess.STDOUT, universal_newlines=True, encoding='utf8'): - print("PNG file " + file + - " is corrupted after crushing, check out pngcursh version") + [pngcrush, "-n", "-v", file_path], + stderr=subprocess.STDOUT, + universal_newlines=True, + encoding="utf8", + ): + print( + "PNG file " + + file + + " is corrupted after crushing, check out pngcursh version" + ) sys.exit(1) - fileMetaMap['sha256New'] = file_hash(file_path) - fileMetaMap['contentHashPost'] = content_hash(file_path) + fileMetaMap["sha256New"] = file_hash(file_path) + fileMetaMap["contentHashPost"] = content_hash(file_path) - if fileMetaMap['contentHashPre'] != fileMetaMap['contentHashPost']: - print("Image contents of PNG file " + file + - " before and after crushing don't match") + if fileMetaMap["contentHashPre"] != fileMetaMap["contentHashPost"]: + print( + "Image contents of PNG file " + + file + + " before and after crushing don't match" + ) sys.exit(1) - fileMetaMap['psize'] = os.path.getsize(file_path) + fileMetaMap["psize"] = os.path.getsize(file_path) outputArray.append(fileMetaMap) print("done") print("summary:\n+++++++++++++++++") for fileDict in outputArray: - oldHash = fileDict['sha256Old'] - newHash = fileDict['sha256New'] - totalSaveBytes += fileDict['osize'] - fileDict['psize'] + oldHash = fileDict["sha256Old"] + newHash = fileDict["sha256New"] + totalSaveBytes += fileDict["osize"] - fileDict["psize"] noHashChange = noHashChange and (oldHash == newHash) - print(fileDict['file'] + "\n size diff from: " + str(fileDict['osize']) + " to: " + - str(fileDict['psize']) + "\n old sha256: " + oldHash + "\n new sha256: " + newHash + "\n") + print( + fileDict["file"] + + "\n size diff from: " + + str(fileDict["osize"]) + + " to: " + + str(fileDict["psize"]) + + "\n old sha256: " + + oldHash + + "\n new sha256: " + + newHash + + "\n" + ) -print("completed. Checksum stable: " + str(noHashChange) + - ". Total reduction: " + str(totalSaveBytes) + " bytes") +print( + "completed. Checksum stable: " + + str(noHashChange) + + ". Total reduction: " + + str(totalSaveBytes) + + " bytes" +) diff --git a/contrib/devtools/pixie.py b/contrib/devtools/pixie.py --- a/contrib/devtools/pixie.py +++ b/contrib/devtools/pixie.py @@ -2,9 +2,9 @@ # Copyright (c) 2020 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' +""" Compact, self-contained ELF implementation for bitcoin-abc security checks. -''' +""" import struct import types from typing import Dict, List, Optional, Tuple, Union @@ -38,18 +38,18 @@ SHT_STRTAB = 3 SHT_DYNAMIC = 6 SHT_DYNSYM = 11 -SHT_GNU_verneed = 0x6ffffffe -SHT_GNU_versym = 0x6fffffff +SHT_GNU_verneed = 0x6FFFFFFE +SHT_GNU_versym = 0x6FFFFFFF # relevant values for p_type PT_LOAD = 1 -PT_GNU_STACK = 0x6474e551 -PT_GNU_RELRO = 0x6474e552 +PT_GNU_STACK = 0x6474E551 +PT_GNU_RELRO = 0x6474E552 # relevant values for p_flags -PF_X = (1 << 0) -PF_W = (1 << 1) -PF_R = (1 << 2) +PF_X = 1 << 0 +PF_W = 1 << 1 +PF_R = 1 << 2 # relevant values for d_tag DT_NEEDED = 1 @@ -66,76 +66,91 @@ class ELFRecord(types.SimpleNamespace): - '''Unified parsing for ELF records.''' + """Unified parsing for ELF records.""" - def __init__(self, data: bytes, offset: int, eh: 'ELFHeader', - total_size: Optional[int]) -> None: + def __init__( + self, data: bytes, offset: int, eh: "ELFHeader", total_size: Optional[int] + ) -> None: hdr_struct = self.STRUCT[eh.ei_class][0][eh.ei_data] if total_size is not None and hdr_struct.size > total_size: raise ValueError( - f'{self.__class__.__name__} header size too small ({total_size} < {hdr_struct.size})') - for field, value in zip(self.STRUCT[eh.ei_class][1], hdr_struct.unpack( - data[offset:offset + hdr_struct.size])): + f"{self.__class__.__name__} header size too small ({total_size} <" + f" {hdr_struct.size})" + ) + for field, value in zip( + self.STRUCT[eh.ei_class][1], + hdr_struct.unpack(data[offset : offset + hdr_struct.size]), + ): setattr(self, field, value) def BiStruct(chars: str) -> Dict[int, struct.Struct]: - '''Compile a struct parser for both endians.''' + """Compile a struct parser for both endians.""" return { - ELFDATA2LSB: struct.Struct('<' + chars), - ELFDATA2MSB: struct.Struct('>' + chars), + ELFDATA2LSB: struct.Struct("<" + chars), + ELFDATA2MSB: struct.Struct(">" + chars), } class ELFHeader(ELFRecord): FIELDS = [ - 'e_type', - 'e_machine', - 'e_version', - 'e_entry', - 'e_phoff', - 'e_shoff', - 'e_flags', - 'e_ehsize', - 'e_phentsize', - 'e_phnum', - 'e_shentsize', - 'e_shnum', - 'e_shstrndx'] + "e_type", + "e_machine", + "e_version", + "e_entry", + "e_phoff", + "e_shoff", + "e_flags", + "e_ehsize", + "e_phentsize", + "e_phnum", + "e_shentsize", + "e_shnum", + "e_shstrndx", + ] STRUCT = { - ELFCLASS32: (BiStruct('HHIIIIIHHHHHH'), FIELDS), - ELFCLASS64: (BiStruct('HHIQQQIHHHHHH'), FIELDS), + ELFCLASS32: (BiStruct("HHIIIIIHHHHHH"), FIELDS), + ELFCLASS64: (BiStruct("HHIQQQIHHHHHH"), FIELDS), } def __init__(self, data: bytes, offset: int) -> None: - self.e_ident = data[offset:offset + EI_NIDENT] - if self.e_ident[0:4] != b'\x7fELF': - raise ValueError('invalid ELF magic') + self.e_ident = data[offset : offset + EI_NIDENT] + if self.e_ident[0:4] != b"\x7fELF": + raise ValueError("invalid ELF magic") self.ei_class = self.e_ident[EI_CLASS] self.ei_data = self.e_ident[EI_DATA] super().__init__(data, offset + EI_NIDENT, self, None) def __repr__(self) -> str: - return f'Header(e_ident={self.e_ident!r}, e_type={self.e_type}, e_machine={self.e_machine}, e_version={self.e_version}, e_entry={self.e_entry}, e_phoff={self.e_phoff}, e_shoff={self.e_shoff}, e_flags={self.e_flags}, e_ehsize={self.e_ehsize}, e_phentsize={self.e_phentsize}, e_phnum={self.e_phnum}, e_shentsize={self.e_shentsize}, e_shnum={self.e_shnum}, e_shstrndx={self.e_shstrndx})' + return ( + f"Header(e_ident={self.e_ident!r}, e_type={self.e_type}," + f" e_machine={self.e_machine}, e_version={self.e_version}," + f" e_entry={self.e_entry}, e_phoff={self.e_phoff}, e_shoff={self.e_shoff}," + f" e_flags={self.e_flags}, e_ehsize={self.e_ehsize}," + f" e_phentsize={self.e_phentsize}, e_phnum={self.e_phnum}," + f" e_shentsize={self.e_shentsize}, e_shnum={self.e_shnum}," + f" e_shstrndx={self.e_shstrndx})" + ) class Section(ELFRecord): name: Optional[bytes] = None FIELDS = [ - 'sh_name', - 'sh_type', - 'sh_flags', - 'sh_addr', - 'sh_offset', - 'sh_size', - 'sh_link', - 'sh_info', - 'sh_addralign', - 'sh_entsize'] + "sh_name", + "sh_type", + "sh_flags", + "sh_addr", + "sh_offset", + "sh_size", + "sh_link", + "sh_info", + "sh_addralign", + "sh_entsize", + ] STRUCT = { - ELFCLASS32: (BiStruct('IIIIIIIIII'), FIELDS), - ELFCLASS64: (BiStruct('IIQQQQIIQQ'), FIELDS), + ELFCLASS32: (BiStruct("IIIIIIIIII"), FIELDS), + ELFCLASS64: (BiStruct("IIQQQQIIQQ"), FIELDS), } def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: @@ -143,75 +158,133 @@ self._data = data def __repr__(self) -> str: - return f'Section(sh_name={self.sh_name}({self.name!r}), sh_type=0x{self.sh_type:x}, sh_flags={self.sh_flags}, sh_addr=0x{self.sh_addr:x}, sh_offset=0x{self.sh_offset:x}, sh_size={self.sh_size}, sh_link={self.sh_link}, sh_info={self.sh_info}, sh_addralign={self.sh_addralign}, sh_entsize={self.sh_entsize})' + return ( + f"Section(sh_name={self.sh_name}({self.name!r})," + f" sh_type=0x{self.sh_type:x}, sh_flags={self.sh_flags}," + f" sh_addr=0x{self.sh_addr:x}, sh_offset=0x{self.sh_offset:x}," + f" sh_size={self.sh_size}, sh_link={self.sh_link}, sh_info={self.sh_info}," + f" sh_addralign={self.sh_addralign}, sh_entsize={self.sh_entsize})" + ) def contents(self) -> bytes: - '''Return section contents.''' - return self._data[self.sh_offset:self.sh_offset + self.sh_size] + """Return section contents.""" + return self._data[self.sh_offset : self.sh_offset + self.sh_size] class ProgramHeader(ELFRecord): STRUCT = { # different ELF classes have the same fields, but in a different order to # optimize space versus alignment - ELFCLASS32: (BiStruct('IIIIIIII'), ['p_type', 'p_offset', 'p_vaddr', 'p_paddr', 'p_filesz', 'p_memsz', 'p_flags', 'p_align']), - ELFCLASS64: (BiStruct('IIQQQQQQ'), ['p_type', 'p_flags', 'p_offset', 'p_vaddr', 'p_paddr', 'p_filesz', 'p_memsz', 'p_align']), + ELFCLASS32: ( + BiStruct("IIIIIIII"), + [ + "p_type", + "p_offset", + "p_vaddr", + "p_paddr", + "p_filesz", + "p_memsz", + "p_flags", + "p_align", + ], + ), + ELFCLASS64: ( + BiStruct("IIQQQQQQ"), + [ + "p_type", + "p_flags", + "p_offset", + "p_vaddr", + "p_paddr", + "p_filesz", + "p_memsz", + "p_align", + ], + ), } def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: super().__init__(data, offset, eh, eh.e_phentsize) def __repr__(self) -> str: - return f'ProgramHeader(p_type={self.p_type}, p_offset={self.p_offset}, p_vaddr={self.p_vaddr}, p_paddr={self.p_paddr}, p_filesz={self.p_filesz}, p_memsz={self.p_memsz}, p_flags={self.p_flags}, p_align={self.p_align})' + return ( + f"ProgramHeader(p_type={self.p_type}, p_offset={self.p_offset}," + f" p_vaddr={self.p_vaddr}, p_paddr={self.p_paddr}," + f" p_filesz={self.p_filesz}, p_memsz={self.p_memsz}," + f" p_flags={self.p_flags}, p_align={self.p_align})" + ) class Symbol(ELFRecord): STRUCT = { # different ELF classes have the same fields, but in a different order to # optimize space versus alignment - ELFCLASS32: (BiStruct('IIIBBH'), ['st_name', 'st_value', 'st_size', 'st_info', 'st_other', 'st_shndx']), - ELFCLASS64: (BiStruct('IBBHQQ'), ['st_name', 'st_info', 'st_other', 'st_shndx', 'st_value', 'st_size']), + ELFCLASS32: ( + BiStruct("IIIBBH"), + ["st_name", "st_value", "st_size", "st_info", "st_other", "st_shndx"], + ), + ELFCLASS64: ( + BiStruct("IBBHQQ"), + ["st_name", "st_info", "st_other", "st_shndx", "st_value", "st_size"], + ), } - def __init__(self, data: bytes, offset: int, eh: ELFHeader, - symtab: Section, strings: bytes, version: Optional[bytes]) -> None: + def __init__( + self, + data: bytes, + offset: int, + eh: ELFHeader, + symtab: Section, + strings: bytes, + version: Optional[bytes], + ) -> None: super().__init__(data, offset, eh, symtab.sh_entsize) self.name = _lookup_string(strings, self.st_name) self.version = version def __repr__(self) -> str: - return f'Symbol(st_name={self.st_name}({self.name!r}), st_value={self.st_value}, st_size={self.st_size}, st_info={self.st_info}, st_other={self.st_other}, st_shndx={self.st_shndx}, version={self.version!r})' + return ( + f"Symbol(st_name={self.st_name}({self.name!r}), st_value={self.st_value}," + f" st_size={self.st_size}, st_info={self.st_info}," + f" st_other={self.st_other}, st_shndx={self.st_shndx}," + f" version={self.version!r})" + ) @property def is_import(self) -> bool: - '''Returns whether the symbol is an imported symbol.''' + """Returns whether the symbol is an imported symbol.""" return self.st_bind != STB_LOCAL and self.st_shndx == 0 @property def is_export(self) -> bool: - '''Returns whether the symbol is an exported symbol.''' + """Returns whether the symbol is an exported symbol.""" return self.st_bind != STB_LOCAL and self.st_shndx != 0 @property def st_bind(self) -> int: - '''Returns STB_*.''' + """Returns STB_*.""" return self.st_info >> 4 class Verneed(ELFRecord): - DEF = (BiStruct('HHIII'), ['vn_version', 'vn_cnt', 'vn_file', 'vn_aux', 'vn_next']) + DEF = (BiStruct("HHIII"), ["vn_version", "vn_cnt", "vn_file", "vn_aux", "vn_next"]) STRUCT = {ELFCLASS32: DEF, ELFCLASS64: DEF} def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: super().__init__(data, offset, eh, None) def __repr__(self) -> str: - return f'Verneed(vn_version={self.vn_version}, vn_cnt={self.vn_cnt}, vn_file={self.vn_file}, vn_aux={self.vn_aux}, vn_next={self.vn_next})' + return ( + f"Verneed(vn_version={self.vn_version}, vn_cnt={self.vn_cnt}," + f" vn_file={self.vn_file}, vn_aux={self.vn_aux}, vn_next={self.vn_next})" + ) class Vernaux(ELFRecord): - DEF = (BiStruct('IHHII'), ['vna_hash', 'vna_flags', - 'vna_other', 'vna_name', 'vna_next']) + DEF = ( + BiStruct("IHHII"), + ["vna_hash", "vna_flags", "vna_other", "vna_name", "vna_next"], + ) STRUCT = {ELFCLASS32: DEF, ELFCLASS64: DEF} def __init__(self, data: bytes, offset: int, eh: ELFHeader, strings: bytes) -> None: @@ -219,37 +292,47 @@ self.name = _lookup_string(strings, self.vna_name) def __repr__(self) -> str: - return f'Veraux(vna_hash={self.vna_hash}, vna_flags={self.vna_flags}, vna_other={self.vna_other}, vna_name={self.vna_name}({self.name!r}), vna_next={self.vna_next})' + return ( + f"Veraux(vna_hash={self.vna_hash}, vna_flags={self.vna_flags}," + f" vna_other={self.vna_other}, vna_name={self.vna_name}({self.name!r})," + f" vna_next={self.vna_next})" + ) class DynTag(ELFRecord): STRUCT = { - ELFCLASS32: (BiStruct('II'), ['d_tag', 'd_val']), - ELFCLASS64: (BiStruct('QQ'), ['d_tag', 'd_val']), + ELFCLASS32: (BiStruct("II"), ["d_tag", "d_val"]), + ELFCLASS64: (BiStruct("QQ"), ["d_tag", "d_val"]), } - def __init__(self, data: bytes, offset: int, - eh: ELFHeader, section: Section) -> None: + def __init__( + self, data: bytes, offset: int, eh: ELFHeader, section: Section + ) -> None: super().__init__(data, offset, eh, section.sh_entsize) def __repr__(self) -> str: - return f'DynTag(d_tag={self.d_tag}, d_val={self.d_val})' + return f"DynTag(d_tag={self.d_tag}, d_val={self.d_val})" def _lookup_string(data: bytes, index: int) -> bytes: - '''Look up string by offset in ELF string table.''' - endx = data.find(b'\x00', index) + """Look up string by offset in ELF string table.""" + endx = data.find(b"\x00", index) assert endx != -1 return data[index:endx] # .gnu_version section has a single 16-bit integer per symbol in the linked section -VERSYM_S = BiStruct('H') +VERSYM_S = BiStruct("H") -def _parse_symbol_table(section: Section, strings: bytes, eh: ELFHeader, - versym: bytes, verneed: Dict[int, bytes]) -> List[Symbol]: - '''Parse symbol table, return a list of symbols.''' +def _parse_symbol_table( + section: Section, + strings: bytes, + eh: ELFHeader, + versym: bytes, + verneed: Dict[int, bytes], +) -> List[Symbol]: + """Parse symbol table, return a list of symbols.""" data = section.contents() symbols = [] versym_iter = (verneed.get(v[0]) for v in VERSYM_S[eh.ei_data].iter_unpack(versym)) @@ -259,7 +342,7 @@ def _parse_verneed(section: Section, strings: bytes, eh: ELFHeader) -> Dict[int, bytes]: - '''Parse .gnu.version_r section, return a dictionary of {versym: 'GLIBC_...'}.''' + """Parse .gnu.version_r section, return a dictionary of {versym: 'GLIBC_...'}.""" data = section.contents() ofs = 0 result = {} @@ -280,16 +363,20 @@ return result -def _parse_dyn_tags(section: Section, strings: bytes, - eh: ELFHeader) -> List[Tuple[int, Union[bytes, int]]]: - '''Parse dynamic tags. Return array of tuples.''' +def _parse_dyn_tags( + section: Section, strings: bytes, eh: ELFHeader +) -> List[Tuple[int, Union[bytes, int]]]: + """Parse dynamic tags. Return array of tuples.""" data = section.contents() ofs = 0 result = [] for ofs in range(0, len(data), section.sh_entsize): tag = DynTag(data, ofs, eh, section) - val = _lookup_string( - strings, tag.d_val) if tag.d_tag in STRING_TAGS else tag.d_val + val = ( + _lookup_string(strings, tag.d_val) + if tag.d_tag in STRING_TAGS + else tag.d_val + ) result.append((tag.d_tag, val)) return result @@ -351,7 +438,8 @@ strtab_data = self.sections[section.sh_link].contents() versym_data = versym[idx].contents() # associated symbol version table self.dyn_symbols += _parse_symbol_table( - section, strtab_data, self.hdr, versym_data, verneed) + section, strtab_data, self.hdr, versym_data, verneed + ) def _load_dyn_tags(self) -> None: self.dyn_tags = [] @@ -369,11 +457,11 @@ ph.sections.append(section) def query_dyn_tags(self, tag_in: int) -> List[Union[int, bytes]]: - '''Return the values of all dyn tags with the specified tag.''' + """Return the values of all dyn tags with the specified tag.""" return [val for (tag, val) in self.dyn_tags if tag == tag_in] def load(filename: str) -> ELFFile: - with open(filename, 'rb') as f: + with open(filename, "rb") as f: data = f.read() return ELFFile(data) diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -2,11 +2,11 @@ # Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' +""" Perform basic security checks on a series of executables. Exit status will be 0 if successful, and the program will be silent. Otherwise the exit status will be 1 and it will log which executables failed which checks. -''' +""" import sys from typing import List, Optional @@ -15,18 +15,18 @@ def check_ELF_PIE(executable) -> bool: - ''' + """ Check for position independent executable (PIE), allowing for address space randomization. - ''' + """ elf = pixie.load(executable) return elf.hdr.e_type == pixie.ET_DYN def check_ELF_NX(executable) -> bool: - ''' + """ Check that no sections are writable and executable (including the stack) - ''' + """ elf = pixie.load(executable) have_wx = False have_gnu_stack = False @@ -40,11 +40,11 @@ def check_ELF_RELRO(executable) -> bool: - ''' + """ Check for read-only relocations. GNU_RELRO program header must exist Dynamic section must have BIND_NOW flag - ''' + """ elf = pixie.load(executable) have_gnu_relro = False for ph in elf.program_headers: @@ -68,63 +68,63 @@ def check_ELF_Canary(executable) -> bool: - ''' + """ Check for use of stack canary - ''' + """ elf = pixie.load(executable) for symbol in elf.dyn_symbols: - if symbol.name == b'__stack_chk_fail': + if symbol.name == b"__stack_chk_fail": return True return False def check_ELF_separate_code(executable): - ''' + """ Check that sections are appropriately separated in virtual memory, based on their permissions. This checks for missing -Wl,-z,separate-code and potentially other problems. - ''' + """ elf = pixie.load(executable) R = pixie.PF_R W = pixie.PF_W E = pixie.PF_X EXPECTED_FLAGS = { # Read + execute - b'.init': R | E, - b'.plt': R | E, - b'.plt.got': R | E, - b'.plt.sec': R | E, - b'.text': R | E, - b'.fini': R | E, + b".init": R | E, + b".plt": R | E, + b".plt.got": R | E, + b".plt.sec": R | E, + b".text": R | E, + b".fini": R | E, # Read-only data - b'.interp': R, - b'.note.gnu.property': R, - b'.note.gnu.build-id': R, - b'.note.ABI-tag': R, - b'.gnu.hash': R, - b'.dynsym': R, - b'.dynstr': R, - b'.gnu.version': R, - b'.gnu.version_r': R, - b'.rela.dyn': R, - b'.rela.plt': R, - b'.rodata': R, - b'.eh_frame_hdr': R, - b'.eh_frame': R, - b'.qtmetadata': R, - b'.gcc_except_table': R, - b'.stapsdt.base': R, + b".interp": R, + b".note.gnu.property": R, + b".note.gnu.build-id": R, + b".note.ABI-tag": R, + b".gnu.hash": R, + b".dynsym": R, + b".dynstr": R, + b".gnu.version": R, + b".gnu.version_r": R, + b".rela.dyn": R, + b".rela.plt": R, + b".rodata": R, + b".eh_frame_hdr": R, + b".eh_frame": R, + b".qtmetadata": R, + b".gcc_except_table": R, + b".stapsdt.base": R, # Writable data - b'.init_array': R | W, - b'.fini_array': R | W, - b'.dynamic': R | W, - b'.got': R | W, - b'.data': R | W, - b'.bss': R | W, + b".init_array": R | W, + b".fini_array": R | W, + b".dynamic": R | W, + b".got": R | W, + b".data": R | W, + b".bss": R | W, } if elf.hdr.e_machine == pixie.EM_PPC64: # .plt is RW on ppc64 even with separate-code - EXPECTED_FLAGS[b'.plt'] = R | W + EXPECTED_FLAGS[b".plt"] = R | W # For all LOAD program headers get mapping to the list of sections, # and for each section, remember the flags of the associated program header. flags_per_section = {} @@ -135,7 +135,7 @@ flags_per_section[section.name] = ph.p_flags # Spot-check ELF LOAD program header flags per section # If these sections exist, check them against the expected R/W/E flags - for (section, flags) in flags_per_section.items(): + for section, flags in flags_per_section.items(): if section in EXPECTED_FLAGS: if EXPECTED_FLAGS[section] != flags: return False @@ -143,113 +143,120 @@ def check_PE_DYNAMIC_BASE(executable) -> bool: - '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' + """PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)""" binary = lief.parse(executable) - return lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists + return ( + lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE + in binary.optional_header.dll_characteristics_lists + ) + # Must support high-entropy 64-bit address space layout randomization # in addition to DYNAMIC_BASE to have secure ASLR. def check_PE_HIGH_ENTROPY_VA(executable) -> bool: - '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' + """PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR""" binary = lief.parse(executable) - return lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists + return ( + lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA + in binary.optional_header.dll_characteristics_lists + ) def check_PE_RELOC_SECTION(executable) -> bool: - '''Check for a reloc section. This is required for functional ASLR.''' + """Check for a reloc section. This is required for functional ASLR.""" binary = lief.parse(executable) return binary.has_relocations def check_MACHO_NOUNDEFS(executable) -> bool: - ''' + """ Check for no undefined references. - ''' + """ binary = lief.parse(executable) return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS) def check_MACHO_Canary(executable) -> bool: - ''' + """ Check for use of stack canary - ''' + """ binary = lief.parse(executable) - return binary.has_symbol('___stack_chk_fail') + return binary.has_symbol("___stack_chk_fail") def check_PIE(executable) -> bool: - ''' + """ Check for position independent executable (PIE), allowing for address space randomization. - ''' + """ binary = lief.parse(executable) return binary.is_pie def check_NX(executable) -> bool: - ''' + """ Check for no stack execution - ''' + """ binary = lief.parse(executable) return binary.has_nx CHECKS = { - 'ELF': [ - ('PIE', check_ELF_PIE), - ('NX', check_ELF_NX), - ('RELRO', check_ELF_RELRO), - ('Canary', check_ELF_Canary), - ('separate_code', check_ELF_separate_code), + "ELF": [ + ("PIE", check_ELF_PIE), + ("NX", check_ELF_NX), + ("RELRO", check_ELF_RELRO), + ("Canary", check_ELF_Canary), + ("separate_code", check_ELF_separate_code), + ], + "PE": [ + ("PIE", check_PIE), + ("DYNAMIC_BASE", check_PE_DYNAMIC_BASE), + ("HIGH_ENTROPY_VA", check_PE_HIGH_ENTROPY_VA), + ("NX", check_NX), + ("RELOC_SECTION", check_PE_RELOC_SECTION), ], - 'PE': [ - ('PIE', check_PIE), - ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), - ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), - ('NX', check_NX), - ('RELOC_SECTION', check_PE_RELOC_SECTION), + "MACHO": [ + ("PIE", check_PIE), + ("NOUNDEFS", check_MACHO_NOUNDEFS), + ("NX", check_NX), + ("Canary", check_MACHO_Canary), ], - 'MACHO': [ - ('PIE', check_PIE), - ('NOUNDEFS', check_MACHO_NOUNDEFS), - ('NX', check_NX), - ('Canary', check_MACHO_Canary), - ] } def identify_executable(executable) -> Optional[str]: - with open(filename, 'rb') as f: + with open(filename, "rb") as f: magic = f.read(4) - if magic.startswith(b'MZ'): - return 'PE' - elif magic.startswith(b'\x7fELF'): - return 'ELF' - elif magic.startswith(b'\xcf\xfa'): - return 'MACHO' + if magic.startswith(b"MZ"): + return "PE" + elif magic.startswith(b"\x7fELF"): + return "ELF" + elif magic.startswith(b"\xcf\xfa"): + return "MACHO" return None -if __name__ == '__main__': +if __name__ == "__main__": retval: int = 0 for filename in sys.argv[1:]: try: etype = identify_executable(filename) if etype is None: - print(f'{filename}: unknown format') + print(f"{filename}: unknown format") retval = 1 continue failed: List[str] = [] - for (name, func) in CHECKS[etype]: + for name, func in CHECKS[etype]: if not func(filename): failed.append(name) if failed: print(f"{filename}: failed {' '.join(failed)}") retval = 1 except IOError: - print(f'{filename}: cannot open') + print(f"{filename}: cannot open") retval = 1 sys.exit(retval) diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -2,14 +2,14 @@ # Copyright (c) 2014 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' +""" A script to check that the executables produced by gitian only contain certain symbols and are only linked against allowed libraries. Example usage: find contrib/gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py -''' +""" import subprocess import sys from typing import Optional @@ -34,46 +34,65 @@ # - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) # # See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. -MAX_VERSIONS = { - 'GCC': (8, 3, 0), - 'GLIBC': (2, 27), - 'LIBATOMIC': (1, 0) -} +MAX_VERSIONS = {"GCC": (8, 3, 0), "GLIBC": (2, 27), "LIBATOMIC": (1, 0)} # See here for a description of _IO_stdin_used: # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { - '_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', + "_edata", + "_end", + "__end__", + "_init", + "__bss_start", + "__bss_start__", + "_bss_end__", + "__bss_end__", + "_fini", + "_IO_stdin_used", + "stdin", + "stdout", + "stderr", # Jemalloc exported symbols - '__malloc_hook', 'malloc', 'calloc', 'malloc_usable_size', - '__free_hook', 'free', - '__realloc_hook', 'realloc', - '__memalign_hook', 'memalign', 'posix_memalign', 'aligned_alloc', 'valloc', + "__malloc_hook", + "malloc", + "calloc", + "malloc_usable_size", + "__free_hook", + "free", + "__realloc_hook", + "realloc", + "__memalign_hook", + "memalign", + "posix_memalign", + "aligned_alloc", + "valloc", # Figure out why we get these symbols exported on xenial. - '_ZNKSt5ctypeIcE8do_widenEc', 'in6addr_any', 'optarg', - '_ZNSt16_Sp_counted_baseILN9__gnu_cxx12_Lock_policyE2EE10_M_destroyEv' + "_ZNKSt5ctypeIcE8do_widenEc", + "in6addr_any", + "optarg", + "_ZNSt16_Sp_counted_baseILN9__gnu_cxx12_Lock_policyE2EE10_M_destroyEv", } # Allowed NEEDED libraries ELF_ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt - 'libgcc_s.so.1', # GCC base support - 'libc.so.6', # C library - 'libpthread.so.0', # threading - 'libanl.so.1', # DNS resolve - 'libm.so.6', # math library - 'librt.so.1', # real-time (clock) - 'libatomic.so.1', - 'ld-linux-x86-64.so.2', # 64-bit dynamic linker - 'ld-linux.so.2', # 32-bit dynamic linker - 'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker - 'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker + "libgcc_s.so.1", # GCC base support + "libc.so.6", # C library + "libpthread.so.0", # threading + "libanl.so.1", # DNS resolve + "libm.so.6", # math library + "librt.so.1", # real-time (clock) + "libatomic.so.1", + "ld-linux-x86-64.so.2", # 64-bit dynamic linker + "ld-linux.so.2", # 32-bit dynamic linker + "ld-linux-aarch64.so.1", # 64-bit ARM dynamic linker + "ld-linux-armhf.so.3", # 32-bit ARM dynamic linker # bitcoin-qt only - 'libxcb.so.1', # part of X11 - 'libfontconfig.so.1', # font support - 'libfreetype.so.6', # font parsing - 'libdl.so.2' # programming interface to dynamic linker + "libxcb.so.1", # part of X11 + "libfontconfig.so.1", # font support + "libfreetype.so.6", # font parsing + "libdl.so.2", # programming interface to dynamic linker } ARCH_MIN_GLIBC_VER = { pixie.EM_386: (2, 1), @@ -84,63 +103,64 @@ MACHO_ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt - 'libc++.1.dylib', # C++ Standard Library - 'libSystem.B.dylib', # libc, libm, libpthread, libinfo + "libc++.1.dylib", # C++ Standard Library + "libSystem.B.dylib", # libc, libm, libpthread, libinfo # bitcoin-qt only - 'AppKit', # user interface - 'ApplicationServices', # common application tasks. - 'Carbon', # deprecated c back-compat API - 'CFNetwork', # network services and changes in network configurations - 'CoreFoundation', # low level func, data types - 'CoreGraphics', # 2D rendering - 'CoreServices', # operating system services - 'CoreText', # interface for laying out text and handling fonts. - 'Foundation', # base layer functionality for apps/frameworks - 'ImageIO', # read and write image file formats. - 'IOKit', # user-space access to hardware devices and drivers. - 'libobjc.A.dylib', # Objective-C runtime library - 'Security', # access control and authentication - 'SystemConfiguration', # access network configuration settings + "AppKit", # user interface + "ApplicationServices", # common application tasks. + "Carbon", # deprecated c back-compat API + "CFNetwork", # network services and changes in network configurations + "CoreFoundation", # low level func, data types + "CoreGraphics", # 2D rendering + "CoreServices", # operating system services + "CoreText", # interface for laying out text and handling fonts. + "Foundation", # base layer functionality for apps/frameworks + "ImageIO", # read and write image file formats. + "IOKit", # user-space access to hardware devices and drivers. + "libobjc.A.dylib", # Objective-C runtime library + "Security", # access control and authentication + "SystemConfiguration", # access network configuration settings } PE_ALLOWED_LIBRARIES = { - 'ADVAPI32.dll', # security & registry - 'IPHLPAPI.DLL', # IP helper API - 'KERNEL32.dll', # win32 base APIs - 'msvcrt.dll', # C standard library for MSVC - 'SHELL32.dll', # shell API - 'USER32.dll', # user interface - 'WS2_32.dll', # sockets + "ADVAPI32.dll", # security & registry + "IPHLPAPI.DLL", # IP helper API + "KERNEL32.dll", # win32 base APIs + "msvcrt.dll", # C standard library for MSVC + "SHELL32.dll", # shell API + "USER32.dll", # user interface + "WS2_32.dll", # sockets # bitcoin-qt only - 'dwmapi.dll', # desktop window manager - 'CRYPT32.dll', # openssl - 'GDI32.dll', # graphics device interface - 'IMM32.dll', # input method editor - 'ole32.dll', # component object model - 'OLEAUT32.dll', # OLE Automation API - 'SHLWAPI.dll', # light weight shell API - 'UxTheme.dll', - 'VERSION.dll', # version checking - 'WINMM.dll', # WinMM audio API + "dwmapi.dll", # desktop window manager + "CRYPT32.dll", # openssl + "GDI32.dll", # graphics device interface + "IMM32.dll", # input method editor + "ole32.dll", # component object model + "OLEAUT32.dll", # OLE Automation API + "SHLWAPI.dll", # light weight shell API + "UxTheme.dll", + "VERSION.dll", # version checking + "WINMM.dll", # WinMM audio API } class CPPFilt(object): - ''' + """ Demangle C++ symbol names. Use a pipe to the 'c++filt' command. - ''' + """ def __init__(self): self.proc = subprocess.Popen( - determine_wellknown_cmd('CPPFILT', 'c++filt'), + determine_wellknown_cmd("CPPFILT", "c++filt"), stdin=subprocess.PIPE, stdout=subprocess.PIPE, - universal_newlines=True) + universal_newlines=True, + ) def __call__(self, mangled): - self.proc.stdin.write(mangled + '\n') + self.proc.stdin.write(mangled + "\n") self.proc.stdin.flush() return self.proc.stdout.readline().rstrip() @@ -151,15 +171,17 @@ def check_version(max_versions, version, arch) -> bool: - if '_' in version: - (lib, _, ver) = version.rpartition('_') + if "_" in version: + (lib, _, ver) = version.rpartition("_") else: lib = version - ver = '0' - ver = tuple([int(x) for x in ver.split('.')]) + ver = "0" + ver = tuple([int(x) for x in ver.split(".")]) if lib not in max_versions: return False - return ver <= max_versions[lib] or lib == 'GLIBC' and ver <= ARCH_MIN_GLIBC_VER[arch] + return ( + ver <= max_versions[lib] or lib == "GLIBC" and ver <= ARCH_MIN_GLIBC_VER[arch] + ) def check_imported_symbols(filename) -> bool: @@ -173,8 +195,9 @@ sym = symbol.name.decode() version = symbol.version.decode() if symbol.version is not None else None if version and not check_version(MAX_VERSIONS, version, elf.hdr.e_machine): - print(f'{filename}: symbol {cppfilt(sym)} from unsupported version ' - f'{version}') + print( + f"{filename}: symbol {cppfilt(sym)} from unsupported version {version}" + ) ok = False return ok @@ -189,7 +212,7 @@ sym = symbol.name.decode() if sym in IGNORE_EXPORTS: continue - print(f'{filename}: export of symbol {cppfilt(sym)} not allowed') + print(f"{filename}: export of symbol {cppfilt(sym)} not allowed") ok = False return ok @@ -200,7 +223,7 @@ for library_name in elf.query_dyn_tags(pixie.DT_NEEDED): assert isinstance(library_name, bytes) if library_name.decode() not in ELF_ALLOWED_LIBRARIES: - print(f'{filename}: NEEDED library {library_name.decode()} is not allowed') + print(f"{filename}: NEEDED library {library_name.decode()} is not allowed") ok = False return ok @@ -209,9 +232,9 @@ ok: bool = True binary = lief.parse(filename) for dylib in binary.libraries: - split = dylib.name.split('/') + split = dylib.name.split("/") if split[-1] not in MACHO_ALLOWED_LIBRARIES: - print(f'{split[-1]} is not in ALLOWED_LIBRARIES!') + print(f"{split[-1]} is not in ALLOWED_LIBRARIES!") ok = False return ok @@ -231,7 +254,7 @@ binary = lief.parse(filename) for dylib in binary.libraries: if dylib not in PE_ALLOWED_LIBRARIES: - print(f'{dylib} is not in ALLOWED_LIBRARIES!') + print(f"{dylib} is not in ALLOWED_LIBRARIES!") ok = False return ok @@ -244,53 +267,53 @@ CHECKS = { - 'ELF': [ - ('IMPORTED_SYMBOLS', check_imported_symbols), - ('EXPORTED_SYMBOLS', check_exported_symbols), - ('LIBRARY_DEPENDENCIES', check_ELF_libraries) + "ELF": [ + ("IMPORTED_SYMBOLS", check_imported_symbols), + ("EXPORTED_SYMBOLS", check_exported_symbols), + ("LIBRARY_DEPENDENCIES", check_ELF_libraries), + ], + "MACHO": [ + ("DYNAMIC_LIBRARIES", check_MACHO_libraries), + ("MIN_OS", check_MACHO_min_os), + ("SDK", check_MACHO_sdk), ], - 'MACHO': [ - ('DYNAMIC_LIBRARIES', check_MACHO_libraries), - ('MIN_OS', check_MACHO_min_os), - ('SDK', check_MACHO_sdk), + "PE": [ + ("DYNAMIC_LIBRARIES", check_PE_libraries), + ("SUBSYSTEM_VERSION", check_PE_subsystem_version), ], - 'PE': [ - ('DYNAMIC_LIBRARIES', check_PE_libraries), - ('SUBSYSTEM_VERSION', check_PE_subsystem_version), - ] } def identify_executable(filename) -> Optional[str]: - with open(filename, 'rb') as f: + with open(filename, "rb") as f: magic = f.read(4) - if magic.startswith(b'MZ'): - return 'PE' - elif magic.startswith(b'\x7fELF'): - return 'ELF' - elif magic.startswith(b'\xcf\xfa'): - return 'MACHO' + if magic.startswith(b"MZ"): + return "PE" + elif magic.startswith(b"\x7fELF"): + return "ELF" + elif magic.startswith(b"\xcf\xfa"): + return "MACHO" return None -if __name__ == '__main__': +if __name__ == "__main__": retval = 0 for filename in sys.argv[1:]: try: etype = identify_executable(filename) if etype is None: - print(f'{filename}: unknown format') + print(f"{filename}: unknown format") retval = 1 continue failed = [] - for (name, func) in CHECKS[etype]: + for name, func in CHECKS[etype]: if not func(filename): failed.append(name) if failed: print(f'{filename}: failed {" ".join(failed)}') retval = 1 except IOError: - print(f'{filename}: cannot open') + print(f"{filename}: cannot open") retval = 1 sys.exit(retval) diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -2,9 +2,9 @@ # Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' +""" Test script for security-check.py -''' +""" import os import subprocess import unittest @@ -13,15 +13,15 @@ def write_testcode(filename): - with open(filename, 'w', encoding="utf8") as f: - f.write(''' + with open(filename, "w", encoding="utf8") as f: + f.write(""" #include int main() { printf("the quick brown fox jumps over the lazy god\\n"); return 0; } - ''') + """) def clean_files(source, executable): @@ -30,73 +30,268 @@ def call_security_check(cc, source, executable, options): - subprocess.check_call([*cc, source, '-o', executable] + options) - p = subprocess.Popen(['./security-check.py', executable], stdout=subprocess.PIPE, - stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + subprocess.check_call([*cc, source, "-o", executable] + options) + p = subprocess.Popen( + ["./security-check.py", executable], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + universal_newlines=True, + ) (stdout, stderr) = p.communicate() return (p.returncode, stdout.rstrip()) class TestSecurityChecks(unittest.TestCase): def test_ELF(self): - source = 'test1.c' - executable = 'test1' - cc = determine_wellknown_cmd('CC', 'gcc') + source = "test1.c" + executable = "test1" + cc = determine_wellknown_cmd("CC", "gcc") write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack', '-fno-stack-protector', '-Wl,-znorelro', '-no-pie', '-fno-PIE', '-Wl,-z,separate-code']), - (1, executable + ': failed PIE NX RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack', '-fno-stack-protector', '-Wl,-znorelro', '-no-pie', '-fno-PIE', '-Wl,-z,separate-code']), - (1, executable + ': failed PIE RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack', '-fstack-protector-all', '-Wl,-znorelro', '-no-pie', '-fno-PIE', '-Wl,-z,separate-code']), - (1, executable + ': failed PIE RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack', '-fstack-protector-all', '-Wl,-znorelro', '-pie', '-fPIE', '-Wl,-z,separate-code']), - (1, executable + ': failed RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack', '-fstack-protector-all', '-Wl,-zrelro', '-Wl,-z,now', '-pie', '-fPIE', '-Wl,-z,noseparate-code']), - (1, executable + ': failed separate_code')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack', '-fstack-protector-all', '-Wl,-zrelro', '-Wl,-z,now', '-pie', '-fPIE', '-Wl,-z,separate-code']), - (0, '')) + self.assertEqual( + call_security_check( + cc, + source, + executable, + [ + "-Wl,-zexecstack", + "-fno-stack-protector", + "-Wl,-znorelro", + "-no-pie", + "-fno-PIE", + "-Wl,-z,separate-code", + ], + ), + (1, executable + ": failed PIE NX RELRO Canary"), + ) + self.assertEqual( + call_security_check( + cc, + source, + executable, + [ + "-Wl,-znoexecstack", + "-fno-stack-protector", + "-Wl,-znorelro", + "-no-pie", + "-fno-PIE", + "-Wl,-z,separate-code", + ], + ), + (1, executable + ": failed PIE RELRO Canary"), + ) + self.assertEqual( + call_security_check( + cc, + source, + executable, + [ + "-Wl,-znoexecstack", + "-fstack-protector-all", + "-Wl,-znorelro", + "-no-pie", + "-fno-PIE", + "-Wl,-z,separate-code", + ], + ), + (1, executable + ": failed PIE RELRO"), + ) + self.assertEqual( + call_security_check( + cc, + source, + executable, + [ + "-Wl,-znoexecstack", + "-fstack-protector-all", + "-Wl,-znorelro", + "-pie", + "-fPIE", + "-Wl,-z,separate-code", + ], + ), + (1, executable + ": failed RELRO"), + ) + self.assertEqual( + call_security_check( + cc, + source, + executable, + [ + "-Wl,-znoexecstack", + "-fstack-protector-all", + "-Wl,-zrelro", + "-Wl,-z,now", + "-pie", + "-fPIE", + "-Wl,-z,noseparate-code", + ], + ), + (1, executable + ": failed separate_code"), + ) + self.assertEqual( + call_security_check( + cc, + source, + executable, + [ + "-Wl,-znoexecstack", + "-fstack-protector-all", + "-Wl,-zrelro", + "-Wl,-z,now", + "-pie", + "-fPIE", + "-Wl,-z,separate-code", + ], + ), + (0, ""), + ) clean_files(source, executable) def test_PE(self): - source = 'test1.c' - executable = 'test1.exe' - cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') + source = "test1.c" + executable = "test1.exe" + cc = determine_wellknown_cmd("CC", "x86_64-w64-mingw32-gcc") write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat', '-Wl,--no-dynamicbase', '-Wl,--no-high-entropy-va', '-no-pie', '-fno-PIE']), - (1, executable + ': failed DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat', '-Wl,--no-dynamicbase', '-Wl,--no-high-entropy-va', '-no-pie', '-fno-PIE']), - (1, executable + ': failed DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat', '-Wl,--dynamicbase', '-Wl,--no-high-entropy-va', '-no-pie', '-fno-PIE']), - (1, executable + ': failed HIGH_ENTROPY_VA RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat', '-Wl,--dynamicbase', '-Wl,--high-entropy-va', '-no-pie', '-fno-PIE']), - (1, executable + ': failed RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat', '-Wl,--dynamicbase', '-Wl,--high-entropy-va', '-pie', '-fPIE']), - (0, '')) + self.assertEqual( + call_security_check( + cc, + source, + executable, + [ + "-Wl,--no-nxcompat", + "-Wl,--no-dynamicbase", + "-Wl,--no-high-entropy-va", + "-no-pie", + "-fno-PIE", + ], + ), + (1, executable + ": failed DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION"), + ) + self.assertEqual( + call_security_check( + cc, + source, + executable, + [ + "-Wl,--nxcompat", + "-Wl,--no-dynamicbase", + "-Wl,--no-high-entropy-va", + "-no-pie", + "-fno-PIE", + ], + ), + (1, executable + ": failed DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION"), + ) + self.assertEqual( + call_security_check( + cc, + source, + executable, + [ + "-Wl,--nxcompat", + "-Wl,--dynamicbase", + "-Wl,--no-high-entropy-va", + "-no-pie", + "-fno-PIE", + ], + ), + (1, executable + ": failed HIGH_ENTROPY_VA RELOC_SECTION"), + ) + self.assertEqual( + call_security_check( + cc, + source, + executable, + [ + "-Wl,--nxcompat", + "-Wl,--dynamicbase", + "-Wl,--high-entropy-va", + "-no-pie", + "-fno-PIE", + ], + ), + (1, executable + ": failed RELOC_SECTION"), + ) + self.assertEqual( + call_security_check( + cc, + source, + executable, + [ + "-Wl,--nxcompat", + "-Wl,--dynamicbase", + "-Wl,--high-entropy-va", + "-pie", + "-fPIE", + ], + ), + (0, ""), + ) clean_files(source, executable) def test_MACHO(self): - source = 'test1.c' - executable = 'test1' - cc = determine_wellknown_cmd('CC', 'clang') + source = "test1.c" + executable = "test1" + cc = determine_wellknown_cmd("CC", "clang") write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie', '-Wl,-flat_namespace', '-Wl,-allow_stack_execute', '-fno-stack-protector']), - (1, executable + ': failed PIE NOUNDEFS NX Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie', '-Wl,-flat_namespace', '-Wl,-allow_stack_execute', '-fstack-protector-all']), - (1, executable + ': failed PIE NOUNDEFS NX')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie', '-Wl,-flat_namespace', '-fstack-protector-all']), - (1, executable + ': failed PIE NOUNDEFS')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie', '-fstack-protector-all']), - (1, executable + ': failed PIE')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie', '-fstack-protector-all']), - (0, '')) + self.assertEqual( + call_security_check( + cc, + source, + executable, + [ + "-Wl,-no_pie", + "-Wl,-flat_namespace", + "-Wl,-allow_stack_execute", + "-fno-stack-protector", + ], + ), + (1, executable + ": failed PIE NOUNDEFS NX Canary"), + ) + self.assertEqual( + call_security_check( + cc, + source, + executable, + [ + "-Wl,-no_pie", + "-Wl,-flat_namespace", + "-Wl,-allow_stack_execute", + "-fstack-protector-all", + ], + ), + (1, executable + ": failed PIE NOUNDEFS NX"), + ) + self.assertEqual( + call_security_check( + cc, + source, + executable, + ["-Wl,-no_pie", "-Wl,-flat_namespace", "-fstack-protector-all"], + ), + (1, executable + ": failed PIE NOUNDEFS"), + ) + self.assertEqual( + call_security_check( + cc, source, executable, ["-Wl,-no_pie", "-fstack-protector-all"] + ), + (1, executable + ": failed PIE"), + ) + self.assertEqual( + call_security_check( + cc, source, executable, ["-Wl,-pie", "-fstack-protector-all"] + ), + (0, ""), + ) clean_files(source, executable) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -2,9 +2,9 @@ # Copyright (c) 2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' +""" Test script for symbol-check.py -''' +""" import os import subprocess import unittest @@ -14,9 +14,12 @@ def call_symbol_check(cc: List[str], source, executable, options): - subprocess.run([*cc, source, '-o', executable] + options, check=True) - p = subprocess.run(['./contrib/devtools/symbol-check.py', executable], - stdout=subprocess.PIPE, universal_newlines=True) + subprocess.run([*cc, source, "-o", executable] + options, check=True) + p = subprocess.run( + ["./contrib/devtools/symbol-check.py", executable], + stdout=subprocess.PIPE, + universal_newlines=True, + ) os.remove(source) os.remove(executable) return (p.returncode, p.stdout.rstrip()) @@ -24,14 +27,14 @@ class TestSymbolChecks(unittest.TestCase): def test_ELF(self): - source = 'test1.c' - executable = 'test1' - cc = determine_wellknown_cmd('CC', 'gcc') + source = "test1.c" + executable = "test1" + cc = determine_wellknown_cmd("CC", "gcc") # renameat2 was introduced in GLIBC 2.28, so is newer than the upper limit # of glibc for all platforms - with open(source, 'w', encoding="utf8") as f: - f.write(''' + with open(source, "w", encoding="utf8") as f: + f.write(""" #define _GNU_SOURCE #include #include @@ -44,19 +47,26 @@ renameat2(0, "test", 0, "test_", RENAME_EXCHANGE); return 0; } - ''') + """) - self.assertEqual(call_symbol_check(cc, source, executable, []), - (1, executable + ': symbol renameat2 from unsupported version GLIBC_2.28\n' + - executable + ': failed IMPORTED_SYMBOLS')) + self.assertEqual( + call_symbol_check(cc, source, executable, []), + ( + 1, + executable + + ": symbol renameat2 from unsupported version GLIBC_2.28\n" + + executable + + ": failed IMPORTED_SYMBOLS", + ), + ) # -lutil is part of the libc6 package so a safe bet that it's installed # it's also out of context enough that it's unlikely to ever become a real # dependency - source = 'test2.c' - executable = 'test2' - with open(source, 'w', encoding="utf8") as f: - f.write(''' + source = "test2.c" + executable = "test2" + with open(source, "w", encoding="utf8") as f: + f.write(""" #include int main() @@ -64,35 +74,41 @@ login(0); return 0; } - ''') + """) - self.assertEqual(call_symbol_check(cc, source, executable, ['-lutil']), - (1, executable + ': NEEDED library libutil.so.1 is not allowed\n' + - executable + ': failed LIBRARY_DEPENDENCIES')) + self.assertEqual( + call_symbol_check(cc, source, executable, ["-lutil"]), + ( + 1, + executable + + ": NEEDED library libutil.so.1 is not allowed\n" + + executable + + ": failed LIBRARY_DEPENDENCIES", + ), + ) # finally, check a conforming file that simply uses a math function - source = 'test3.c' - executable = 'test3' - with open(source, 'w', encoding="utf8") as f: - f.write(''' + source = "test3.c" + executable = "test3" + with open(source, "w", encoding="utf8") as f: + f.write(""" #include int main() { return (int)pow(2.0, 4.0); } - ''') + """) - self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), - (0, '')) + self.assertEqual(call_symbol_check(cc, source, executable, ["-lm"]), (0, "")) def test_MACHO(self): - source = 'test1.c' - executable = 'test1' - cc = determine_wellknown_cmd('CC', 'clang') + source = "test1.c" + executable = "test1" + cc = determine_wellknown_cmd("CC", "clang") - with open(source, 'w', encoding="utf8") as f: - f.write(''' + with open(source, "w", encoding="utf8") as f: + f.write(""" #include int main() @@ -101,19 +117,32 @@ return 0; } - ''') + """) self.assertEqual( - call_symbol_check(cc, source, executable, - ['-lexpat', '-Wl,-platform_version', '-Wl,macos', - '-Wl,11.4', '-Wl,11.4']), - (1, 'libexpat.1.dylib is not in ALLOWED_LIBRARIES!\n' + - f'{executable}: failed DYNAMIC_LIBRARIES MIN_OS SDK')) - - source = 'test2.c' - executable = 'test2' - with open(source, 'w', encoding="utf8") as f: - f.write(''' + call_symbol_check( + cc, + source, + executable, + [ + "-lexpat", + "-Wl,-platform_version", + "-Wl,macos", + "-Wl,11.4", + "-Wl,11.4", + ], + ), + ( + 1, + "libexpat.1.dylib is not in ALLOWED_LIBRARIES!\n" + + f"{executable}: failed DYNAMIC_LIBRARIES MIN_OS SDK", + ), + ) + + source = "test2.c" + executable = "test2" + with open(source, "w", encoding="utf8") as f: + f.write(""" #include int main() @@ -121,36 +150,52 @@ CGMainDisplayID(); return 0; } - ''') + """) self.assertEqual( - call_symbol_check(cc, source, executable, - ['-framework', 'CoreGraphics', '-Wl,-platform_version', - '-Wl,macos', '-Wl,11.4', '-Wl,11.4']), - (1, f'{executable}: failed MIN_OS SDK')) - - source = 'test3.c' - executable = 'test3' - with open(source, 'w', encoding="utf8") as f: - f.write(''' + call_symbol_check( + cc, + source, + executable, + [ + "-framework", + "CoreGraphics", + "-Wl,-platform_version", + "-Wl,macos", + "-Wl,11.4", + "-Wl,11.4", + ], + ), + (1, f"{executable}: failed MIN_OS SDK"), + ) + + source = "test3.c" + executable = "test3" + with open(source, "w", encoding="utf8") as f: + f.write(""" int main() { return 0; } - ''') + """) - self.assertEqual(call_symbol_check(cc, source, executable, - ['-Wl,-platform_version', '-Wl,macos', - '-Wl,10.15', '-Wl,11.4']), - (1, f'{executable}: failed SDK')) + self.assertEqual( + call_symbol_check( + cc, + source, + executable, + ["-Wl,-platform_version", "-Wl,macos", "-Wl,10.15", "-Wl,11.4"], + ), + (1, f"{executable}: failed SDK"), + ) def test_PE(self): - source = 'test1.c' - executable = 'test1.exe' - cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') + source = "test1.c" + executable = "test1.exe" + cc = determine_wellknown_cmd("CC", "x86_64-w64-mingw32-gcc") - with open(source, 'w', encoding="utf8") as f: - f.write(''' + with open(source, "w", encoding="utf8") as f: + f.write(""" #include int main() @@ -158,34 +203,59 @@ PdhConnectMachineA(NULL); return 0; } - ''') + """) self.assertEqual( call_symbol_check( - cc, source, executable, - ['-lpdh', '-Wl,--major-subsystem-version', '-Wl,6', - '-Wl,--minor-subsystem-version', '-Wl,1']), - (1, 'pdh.dll is not in ALLOWED_LIBRARIES!\n' + - executable + ': failed DYNAMIC_LIBRARIES')) - - source = 'test2.c' - executable = 'test2.exe' - - with open(source, 'w', encoding="utf8") as f: - f.write(''' + cc, + source, + executable, + [ + "-lpdh", + "-Wl,--major-subsystem-version", + "-Wl,6", + "-Wl,--minor-subsystem-version", + "-Wl,1", + ], + ), + ( + 1, + "pdh.dll is not in ALLOWED_LIBRARIES!\n" + + executable + + ": failed DYNAMIC_LIBRARIES", + ), + ) + + source = "test2.c" + executable = "test2.exe" + + with open(source, "w", encoding="utf8") as f: + f.write(""" int main() { return 0; } - ''') - - self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,--major-subsystem-version', '-Wl,9', '-Wl,--minor-subsystem-version', '-Wl,9']), - (1, executable + ': failed SUBSYSTEM_VERSION')) + """) - source = 'test3.c' - executable = 'test3.exe' - with open(source, 'w', encoding="utf8") as f: - f.write(''' + self.assertEqual( + call_symbol_check( + cc, + source, + executable, + [ + "-Wl,--major-subsystem-version", + "-Wl,9", + "-Wl,--minor-subsystem-version", + "-Wl,9", + ], + ), + (1, executable + ": failed SUBSYSTEM_VERSION"), + ) + + source = "test3.c" + executable = "test3.exe" + with open(source, "w", encoding="utf8") as f: + f.write(""" #include int main() @@ -193,14 +263,24 @@ CoFreeUnusedLibrariesEx(0,0); return 0; } - ''') + """) self.assertEqual( - call_symbol_check(cc, source, executable, - ['-lole32', '-Wl,--major-subsystem-version', '-Wl,6', - '-Wl,--minor-subsystem-version', '-Wl,1']), - (0, '')) - - -if __name__ == '__main__': + call_symbol_check( + cc, + source, + executable, + [ + "-lole32", + "-Wl,--major-subsystem-version", + "-Wl,6", + "-Wl,--minor-subsystem-version", + "-Wl,1", + ], + ), + (0, ""), + ) + + +if __name__ == "__main__": unittest.main() diff --git a/contrib/devtools/update-translations.py b/contrib/devtools/update-translations.py --- a/contrib/devtools/update-translations.py +++ b/contrib/devtools/update-translations.py @@ -2,7 +2,7 @@ # Copyright (c) 2014 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' +""" Run this script from the root of the repository to update all translations from transifex. It will do the following automatically: @@ -14,7 +14,7 @@ TODO: - auto-add new translations to the build system according to the translation process -''' +""" import io import os import re @@ -23,36 +23,34 @@ import xml.etree.ElementTree as ET # Name of transifex tool -TX = 'tx' +TX = "tx" # Name of source language file -SOURCE_LANG = 'bitcoin_en.ts' +SOURCE_LANG = "bitcoin_en.ts" # Directory with locale files -LOCALE_DIR = 'src/qt/locale' +LOCALE_DIR = "src/qt/locale" # Minimum number of messages for translation to be considered at all MIN_NUM_MESSAGES = 10 def check_at_repository_root(): - if not os.path.exists('.git'): - print('No .git directory found') - print( - 'Execute this script at the root of the repository', - file=sys.stderr) + if not os.path.exists(".git"): + print("No .git directory found") + print("Execute this script at the root of the repository", file=sys.stderr) sys.exit(1) def fetch_all_translations(): - if subprocess.call([TX, 'pull', '-f', '-a']): - print('Error while fetching translations', file=sys.stderr) + if subprocess.call([TX, "pull", "-f", "-a"]): + print("Error while fetching translations", file=sys.stderr) sys.exit(1) def find_format_specifiers(s): - '''Find all format specifiers in a string.''' + """Find all format specifiers in a string.""" pos = 0 specifiers = [] while True: - percent = s.find('%', pos) + percent = s.find("%", pos) if percent < 0: break specifiers.append(s[percent + 1]) @@ -61,11 +59,11 @@ def split_format_specifiers(specifiers): - '''Split format specifiers between numeric (Qt) and others (strprintf)''' + """Split format specifiers between numeric (Qt) and others (strprintf)""" numeric = [] other = [] for s in specifiers: - if s in {'1', '2', '3', '4', '5', '6', '7', '8', '9'}: + if s in {"1", "2", "3", "4", "5", "6", "7", "8", "9"}: numeric.append(s) else: other.append(s) @@ -84,8 +82,8 @@ def sanitize_string(s): - '''Sanitize string for printing''' - return s.replace('\n', ' ') + """Sanitize string for printing""" + return s.replace("\n", " ") def check_format_specifiers(source, translation, errors, numerus): @@ -94,43 +92,51 @@ # if this fails, go change the source as this is hacky and confusing! assert not (source_f[0] and source_f[1]) try: - translation_f = split_format_specifiers( - find_format_specifiers(translation)) + translation_f = split_format_specifiers(find_format_specifiers(translation)) except IndexError: - errors.append("Parse error in translation for '{}': '{}'".format( - sanitize_string(source), sanitize_string(translation))) + errors.append( + "Parse error in translation for '{}': '{}'".format( + sanitize_string(source), sanitize_string(translation) + ) + ) return False else: if source_f != translation_f: - if numerus and source_f == (set(), ['n']) and translation_f == ( - set(), []) and translation.find('%') == -1: + if ( + numerus + and source_f == (set(), ["n"]) + and translation_f == (set(), []) + and translation.find("%") == -1 + ): # Allow numerus translations to omit %n specifier (usually when # it only has one possible value) return True - errors.append("Mismatch between '{}' and '{}'".format( - sanitize_string(source), sanitize_string(translation))) + errors.append( + "Mismatch between '{}' and '{}'".format( + sanitize_string(source), sanitize_string(translation) + ) + ) return False return True -def all_ts_files(suffix=''): +def all_ts_files(suffix=""): for filename in os.listdir(LOCALE_DIR): # process only language files, and do not process source language - if not filename.endswith( - '.ts' + suffix) or filename == SOURCE_LANG + suffix: + if not filename.endswith(".ts" + suffix) or filename == SOURCE_LANG + suffix: continue if suffix: # remove provided suffix - filename = filename[0:-len(suffix)] + filename = filename[0 : -len(suffix)] filepath = os.path.join(LOCALE_DIR, filename) yield filename, filepath -FIX_RE = re.compile(b'[\x00-\x09\x0b\x0c\x0e-\x1f]') +FIX_RE = re.compile(b"[\x00-\x09\x0b\x0c\x0e-\x1f]") def remove_invalid_characters(s): - '''Remove invalid characters from translation string''' - return FIX_RE.sub(b'', s) + """Remove invalid characters from translation string""" + return FIX_RE.sub(b"", s) # Override cdata escape function to make our output match Qt's (optional, just for cleaner diffs for @@ -140,29 +146,29 @@ def escape_cdata(text): text = _orig_escape_cdata(text) - text = text.replace("'", ''') - text = text.replace('"', '"') + text = text.replace("'", "'") + text = text.replace('"', """) return text def postprocess_translations(reduce_diff_hacks=False): - print('Checking and postprocessing...') + print("Checking and postprocessing...") if reduce_diff_hacks: global _orig_escape_cdata _orig_escape_cdata = ET._escape_cdata ET._escape_cdata = escape_cdata - for (filename, filepath) in all_ts_files(): - os.rename(filepath, filepath + '.orig') + for filename, filepath in all_ts_files(): + os.rename(filepath, filepath + ".orig") have_errors = False - for (filename, filepath) in all_ts_files('.orig'): + for filename, filepath in all_ts_files(".orig"): # pre-fixups to cope with transifex output # need to override encoding because 'utf8' is not understood only # 'utf-8' - parser = ET.XMLParser(encoding='utf-8') - with open(filepath + '.orig', 'rb') as f: + parser = ET.XMLParser(encoding="utf-8") + with open(filepath + ".orig", "rb") as f: data = f.read() # remove control characters; this must be done over the entire file # otherwise the XML parser will fail @@ -171,15 +177,16 @@ # iterate over all messages in file root = tree.getroot() - for context in root.findall('context'): - for message in context.findall('message'): - numerus = message.get('numerus') == 'yes' - source = message.find('source').text - translation_node = message.find('translation') + for context in root.findall("context"): + for message in context.findall("message"): + numerus = message.get("numerus") == "yes" + source = message.find("source").text + translation_node = message.find("translation") # pick all numerusforms if numerus: translations = [ - i.text for i in translation_node.findall('numerusform')] + i.text for i in translation_node.findall("numerusform") + ] else: translations = [translation_node.text] @@ -188,31 +195,32 @@ continue errors = [] valid = check_format_specifiers( - source, translation, errors, numerus) + source, translation, errors, numerus + ) for error in errors: - print(f'{filename}: {error}') + print(f"{filename}: {error}") if not valid: # set type to unfinished and clear string if invalid translation_node.clear() - translation_node.set('type', 'unfinished') + translation_node.set("type", "unfinished") have_errors = True # Remove location tags - for location in message.findall('location'): + for location in message.findall("location"): message.remove(location) # Remove entire message if it is an unfinished translation - if translation_node.get('type') == 'unfinished': + if translation_node.get("type") == "unfinished": context.remove(message) # check if document is (virtually) empty, and remove it if so num_messages = 0 - for context in root.findall('context'): - for message in context.findall('message'): + for context in root.findall("context"): + for message in context.findall("message"): num_messages += 1 if num_messages < MIN_NUM_MESSAGES: - print(f'Removing {filepath}, as it contains only {num_messages} messages') + print(f"Removing {filepath}, as it contains only {num_messages} messages") continue # write fixed-up tree @@ -220,17 +228,17 @@ # formatting if reduce_diff_hacks: out = io.BytesIO() - tree.write(out, encoding='utf-8') + tree.write(out, encoding="utf-8") out = out.getvalue() - out = out.replace(b' />', b'/>') - with open(filepath, 'wb') as f: + out = out.replace(b" />", b"/>") + with open(filepath, "wb") as f: f.write(out) else: - tree.write(filepath, encoding='utf-8') + tree.write(filepath, encoding="utf-8") return have_errors -if __name__ == '__main__': +if __name__ == "__main__": check_at_repository_root() fetch_all_translations() postprocess_translations() diff --git a/contrib/devtools/utils.py b/contrib/devtools/utils.py --- a/contrib/devtools/utils.py +++ b/contrib/devtools/utils.py @@ -2,9 +2,9 @@ # Copyright (c) 2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' +""" Common utility functions -''' +""" import os import shutil import sys @@ -16,7 +16,7 @@ maybe_which = shutil.which(progname) if maybe_env: # Well-known vars are often meant to be word-split - return maybe_env.split(' ') + return maybe_env.split(" ") elif maybe_which: return [maybe_which] else: