diff --git a/.arclint b/.arclint --- a/.arclint +++ b/.arclint @@ -131,6 +131,10 @@ "check-rpc-mappings": { "type": "check-rpc-mappings", "include": "(^src/(rpc/|wallet/rpc).*\\.cpp$)" + }, + "lint-python-encoding": { + "type": "lint-python-encoding", + "include": "(\\.py$)" } } } diff --git a/arcanist/.phutil_module_cache b/arcanist/.phutil_module_cache --- a/arcanist/.phutil_module_cache +++ b/arcanist/.phutil_module_cache @@ -1 +1 @@ -{"__symbol_cache_version__":11,"ae5b7d3d6b8cf9598ce4abaf0cd56b21":{"have":{"class":{"ExtendedConfigurationDrivenLintEngine":19}},"need":{"function":{"newv":159,"pht":933},"class":{"ArcanistLintEngine":65,"ArcanistConfigurationDrivenLintEngine":171,"PhutilConsole":866},"class\/interface":{"ILintOnce":634}},"xmap":{"ExtendedConfigurationDrivenLintEngine":["ArcanistLintEngine"]}},"2809b09d2021203b43c57da33d1fe8bf":{"have":{"class":{"AssertWithSideEffectsLinter":210}},"need":{"function":{"pht":439},"class":{"ArcanistLinter":246,"ArcanistLintSeverity":926,"Filesystem":1170}},"xmap":{"AssertWithSideEffectsLinter":["ArcanistLinter"]}},"90a8b110dc475955f15bb81d37268cb5":{"have":{"class":{"AutoPEP8FormatLinter":75}},"need":{"function":{"pht":297,"execx":769,"id":1903},"class":{"ArcanistExternalLinter":104,"ArcanistLintMessage":1910,"Filesystem":1754,"ArcanistLinter":2017,"ArcanistLintSeverity":2095}},"xmap":{"AutoPEP8FormatLinter":["ArcanistExternalLinter"]}},"5ea58c19df0397ed8ee0f463d90d6c72":{"have":{"class":{"BoostDependenciesLinter":145}},"need":{"function":{"pht":330,"id":1440},"class":{"GlobalExternalLinter":177,"ArcanistLintMessage":1447,"Filesystem":609,"ArcanistLinter":1524,"ArcanistLintSeverity":1624}},"xmap":{"BoostDependenciesLinter":["GlobalExternalLinter"]}},"38f0c676bff5192a344464142caaa253":{"have":{"class":{"CHeaderLinter":99}},"need":{"function":{"pht":611},"class":{"ArcanistLinter":121,"ArcanistLintSeverity":1060,"Filesystem":1307}},"xmap":{"CHeaderLinter":["ArcanistLinter"]}},"9bb48ec0fe2e9ced8e27d42540d0571c":{"have":{"class":{"CheckDocLinter":106}},"need":{"function":{"pht":321,"id":1845},"class":{"GlobalExternalLinter":129,"ArcanistLintMessage":1852,"Filesystem":729,"ArcanistLinter":1900,"ArcanistLintSeverity":1986}},"xmap":{"CheckDocLinter":["GlobalExternalLinter"]}},"6af7410cfea496ff1d4dcc2624b6b8ea":{"have":{"class":{"ClangFormatLinter":79}},"need":{"function":{"pht":302,"execx":781,"id":1653},"class":{"ArcanistExternalLinter":105,"ArcanistLintMessage":1660,"Filesystem":1504,"ArcanistLinter":1767,"ArcanistLintSeverity":1845}},"xmap":{"ClangFormatLinter":["ArcanistExternalLinter"]}},"75579a609dd975aa0226add52700c622":{"have":{"class":{"FileNameLinter":103}},"need":{"function":{"pht":307},"class":{"ArcanistLinter":126,"ArcanistLintSeverity":662,"Filesystem":968}},"xmap":{"FileNameLinter":["ArcanistLinter"]}},"9285ad9415f8ebe564f7119e5a72c559":{"have":{"class":{"FormatStringLinter":146}},"need":{"function":{"pht":377,"csprintf":1492,"id":1872},"class":{"ArcanistExternalLinter":173,"ArcanistLintMessage":1879,"Filesystem":827,"ArcanistLinter":1956,"ArcanistLintSeverity":2044}},"xmap":{"FormatStringLinter":["ArcanistExternalLinter"]}},"2e11dd9ad67e594f863bc46ac59ea37e":{"have":{"class":{"GlobalExternalLinter":199}},"need":{"class":{"ArcanistExternalLinter":228},"interface":{"ILintOnce":262}},"xmap":{"GlobalExternalLinter":["ArcanistExternalLinter","ILintOnce"]}},"b2403124ec3e8be6cb4d10bf0f6c4134":{"have":{"interface":{"ILintOnce":69}},"need":[],"xmap":[]},"1f9bac7956f4f948a187828dcc6ba2d0":{"have":{"class":{"IncludeGuardLinter":98}},"need":{"function":{"pht":368},"class":{"ArcanistLinter":125,"ArcanistLintSeverity":721,"Filesystem":970}},"xmap":{"IncludeGuardLinter":["ArcanistLinter"]}},"2cbb6e7228d81557f777ad648704f343":{"have":{"class":{"IncludeQuotesLinter":100}},"need":{"function":{"pht":306},"class":{"ArcanistLinter":128,"ArcanistLintSeverity":663,"Filesystem":964}},"xmap":{"IncludeQuotesLinter":["ArcanistLinter"]}},"f151089cf79fdb8257b2272ed4782d88":{"have":{"class":{"IncludeSourceLinter":99}},"need":{"function":{"pht":391},"class":{"ArcanistLinter":127,"ArcanistLintSeverity":699,"Filesystem":938}},"xmap":{"IncludeSourceLinter":["ArcanistLinter"]}},"ce5dee893bedb5d93466655eede8ac47":{"have":{"class":{"LocaleDependenceLinter":160}},"need":{"function":{"pht":5429},"class":{"ArcanistLinter":191,"ArcanistLintSeverity":5932,"Filesystem":6178}},"xmap":{"LocaleDependenceLinter":["ArcanistLinter"]}},"6f2f22dd0f259fb2eaa284b4fab3bc29":{"have":{"class":{"PythonFormatLinter":123}},"need":{"function":{"pht":353,"id":1838},"class":{"ArcanistExternalLinter":150,"ArcanistLintMessage":1845,"Filesystem":776,"ArcanistLinter":1970,"ArcanistLintSeverity":2053}},"xmap":{"PythonFormatLinter":["ArcanistExternalLinter"]}},"09a933fbbf135320585be52750d93831":{"have":{"class":{"StdintLinter":90}},"need":{"function":{"pht":280},"class":{"ArcanistLinter":111,"ArcanistLintSeverity":589,"Filesystem":897}},"xmap":{"StdintLinter":["ArcanistLinter"]}},"25781df78f6eebfb223296b8265e9d19":{"have":{"class":{"TestsLinter":103}},"need":{"function":{"pht":318,"id":2629},"class":{"ArcanistExternalLinter":123,"ArcanistLintMessage":2636,"Filesystem":776,"ArcanistLinter":2684,"ArcanistLintSeverity":2792}},"xmap":{"TestsLinter":["ArcanistExternalLinter"]}},"63d19a8745cb2e1200cc26488dc7ad25":{"have":{"class":{"CheckRpcMappingsLinter":131}},"need":{"function":{"pht":310,"id":1386},"class":{"GlobalExternalLinter":162,"ArcanistLintMessage":1393,"Filesystem":573,"ArcanistLinter":1443,"ArcanistLintSeverity":1544}},"xmap":{"CheckRpcMappingsLinter":["GlobalExternalLinter"]}}} \ No newline at end of file +{"__symbol_cache_version__":11,"ae5b7d3d6b8cf9598ce4abaf0cd56b21":{"have":{"class":{"ExtendedConfigurationDrivenLintEngine":19}},"need":{"function":{"newv":159,"pht":933},"class":{"ArcanistLintEngine":65,"ArcanistConfigurationDrivenLintEngine":171,"PhutilConsole":866},"class\/interface":{"ILintOnce":634}},"xmap":{"ExtendedConfigurationDrivenLintEngine":["ArcanistLintEngine"]}},"2809b09d2021203b43c57da33d1fe8bf":{"have":{"class":{"AssertWithSideEffectsLinter":210}},"need":{"function":{"pht":439},"class":{"ArcanistLinter":246,"ArcanistLintSeverity":926,"Filesystem":1170}},"xmap":{"AssertWithSideEffectsLinter":["ArcanistLinter"]}},"90a8b110dc475955f15bb81d37268cb5":{"have":{"class":{"AutoPEP8FormatLinter":75}},"need":{"function":{"pht":297,"execx":769,"id":1903},"class":{"ArcanistExternalLinter":104,"ArcanistLintMessage":1910,"Filesystem":1754,"ArcanistLinter":2017,"ArcanistLintSeverity":2095}},"xmap":{"AutoPEP8FormatLinter":["ArcanistExternalLinter"]}},"5ea58c19df0397ed8ee0f463d90d6c72":{"have":{"class":{"BoostDependenciesLinter":145}},"need":{"function":{"pht":330,"id":1440},"class":{"GlobalExternalLinter":177,"ArcanistLintMessage":1447,"Filesystem":609,"ArcanistLinter":1524,"ArcanistLintSeverity":1624}},"xmap":{"BoostDependenciesLinter":["GlobalExternalLinter"]}},"38f0c676bff5192a344464142caaa253":{"have":{"class":{"CHeaderLinter":99}},"need":{"function":{"pht":611},"class":{"ArcanistLinter":121,"ArcanistLintSeverity":1060,"Filesystem":1307}},"xmap":{"CHeaderLinter":["ArcanistLinter"]}},"9bb48ec0fe2e9ced8e27d42540d0571c":{"have":{"class":{"CheckDocLinter":106}},"need":{"function":{"pht":321,"id":1845},"class":{"GlobalExternalLinter":129,"ArcanistLintMessage":1852,"Filesystem":729,"ArcanistLinter":1900,"ArcanistLintSeverity":1986}},"xmap":{"CheckDocLinter":["GlobalExternalLinter"]}},"63d19a8745cb2e1200cc26488dc7ad25":{"have":{"class":{"CheckRpcMappingsLinter":131}},"need":{"function":{"pht":310,"id":1386},"class":{"GlobalExternalLinter":162,"ArcanistLintMessage":1393,"Filesystem":573,"ArcanistLinter":1443,"ArcanistLintSeverity":1544}},"xmap":{"CheckRpcMappingsLinter":["GlobalExternalLinter"]}},"6af7410cfea496ff1d4dcc2624b6b8ea":{"have":{"class":{"ClangFormatLinter":79}},"need":{"function":{"pht":302,"execx":781,"id":1653},"class":{"ArcanistExternalLinter":105,"ArcanistLintMessage":1660,"Filesystem":1504,"ArcanistLinter":1767,"ArcanistLintSeverity":1845}},"xmap":{"ClangFormatLinter":["ArcanistExternalLinter"]}},"75579a609dd975aa0226add52700c622":{"have":{"class":{"FileNameLinter":103}},"need":{"function":{"pht":307},"class":{"ArcanistLinter":126,"ArcanistLintSeverity":662,"Filesystem":968}},"xmap":{"FileNameLinter":["ArcanistLinter"]}},"9285ad9415f8ebe564f7119e5a72c559":{"have":{"class":{"FormatStringLinter":146}},"need":{"function":{"pht":377,"csprintf":1492,"id":1872},"class":{"ArcanistExternalLinter":173,"ArcanistLintMessage":1879,"Filesystem":827,"ArcanistLinter":1956,"ArcanistLintSeverity":2044}},"xmap":{"FormatStringLinter":["ArcanistExternalLinter"]}},"2e11dd9ad67e594f863bc46ac59ea37e":{"have":{"class":{"GlobalExternalLinter":199}},"need":{"class":{"ArcanistExternalLinter":228},"interface":{"ILintOnce":262}},"xmap":{"GlobalExternalLinter":["ArcanistExternalLinter","ILintOnce"]}},"b2403124ec3e8be6cb4d10bf0f6c4134":{"have":{"interface":{"ILintOnce":69}},"need":[],"xmap":[]},"1f9bac7956f4f948a187828dcc6ba2d0":{"have":{"class":{"IncludeGuardLinter":98}},"need":{"function":{"pht":368},"class":{"ArcanistLinter":125,"ArcanistLintSeverity":721,"Filesystem":970}},"xmap":{"IncludeGuardLinter":["ArcanistLinter"]}},"2cbb6e7228d81557f777ad648704f343":{"have":{"class":{"IncludeQuotesLinter":100}},"need":{"function":{"pht":306},"class":{"ArcanistLinter":128,"ArcanistLintSeverity":663,"Filesystem":964}},"xmap":{"IncludeQuotesLinter":["ArcanistLinter"]}},"f151089cf79fdb8257b2272ed4782d88":{"have":{"class":{"IncludeSourceLinter":99}},"need":{"function":{"pht":391},"class":{"ArcanistLinter":127,"ArcanistLintSeverity":699,"Filesystem":938}},"xmap":{"IncludeSourceLinter":["ArcanistLinter"]}},"ce5dee893bedb5d93466655eede8ac47":{"have":{"class":{"LocaleDependenceLinter":160}},"need":{"function":{"pht":5429},"class":{"ArcanistLinter":191,"ArcanistLintSeverity":5932,"Filesystem":6178}},"xmap":{"LocaleDependenceLinter":["ArcanistLinter"]}},"6f2f22dd0f259fb2eaa284b4fab3bc29":{"have":{"class":{"PythonFormatLinter":123}},"need":{"function":{"pht":353,"id":1838},"class":{"ArcanistExternalLinter":150,"ArcanistLintMessage":1845,"Filesystem":776,"ArcanistLinter":1970,"ArcanistLintSeverity":2053}},"xmap":{"PythonFormatLinter":["ArcanistExternalLinter"]}},"09a933fbbf135320585be52750d93831":{"have":{"class":{"StdintLinter":90}},"need":{"function":{"pht":280},"class":{"ArcanistLinter":111,"ArcanistLintSeverity":589,"Filesystem":897}},"xmap":{"StdintLinter":["ArcanistLinter"]}},"25781df78f6eebfb223296b8265e9d19":{"have":{"class":{"TestsLinter":103}},"need":{"function":{"pht":318,"id":2629},"class":{"ArcanistExternalLinter":123,"ArcanistLintMessage":2636,"Filesystem":776,"ArcanistLinter":2684,"ArcanistLintSeverity":2792}},"xmap":{"TestsLinter":["ArcanistExternalLinter"]}},"03cf226fe177a9fce7dc046816cc466b":{"have":{"class":{"PythonOpenFileEncodingLinter":111}},"need":{"function":{"pht":325},"class":{"ArcanistLinter":148,"ArcanistLintSeverity":687,"Filesystem":991}},"xmap":{"PythonOpenFileEncodingLinter":["ArcanistLinter"]}}} \ No newline at end of file diff --git a/arcanist/__phutil_library_map__.php b/arcanist/__phutil_library_map__.php --- a/arcanist/__phutil_library_map__.php +++ b/arcanist/__phutil_library_map__.php @@ -26,6 +26,7 @@ 'IncludeSourceLinter' => 'linter/IncludeSourceLinter.php', 'LocaleDependenceLinter' => 'linter/LocaleDependenceLinter.php', 'PythonFormatLinter' => 'linter/PythonFormatLinter.php', + 'PythonOpenFileEncodingLinter' => 'linter/PythonOpenFileEncodingLinter.php', 'StdintLinter' => 'linter/StdintLinter.php', 'TestsLinter' => 'linter/TestsLinter.php', ), @@ -50,6 +51,7 @@ 'IncludeSourceLinter' => 'ArcanistLinter', 'LocaleDependenceLinter' => 'ArcanistLinter', 'PythonFormatLinter' => 'ArcanistExternalLinter', + 'PythonOpenFileEncodingLinter' => 'ArcanistLinter', 'StdintLinter' => 'ArcanistLinter', 'TestsLinter' => 'ArcanistExternalLinter', ), diff --git a/arcanist/linter/PythonOpenFileEncodingLinter.php b/arcanist/linter/PythonOpenFileEncodingLinter.php new file mode 100644 --- /dev/null +++ b/arcanist/linter/PythonOpenFileEncodingLinter.php @@ -0,0 +1,70 @@ + ArcanistLintSeverity::SEVERITY_ERROR, + ); + } + + public function getLintNameMap() { + return array( + self::ENCODING_NOT_FOUND => pht('Encoding should be specified when '. + 'opening a text file.'), + ); + } + + public function lintPath($path) { + $path = Filesystem::resolvePath($path, $this->getProjectRoot()); + $fileContent = Filesystem::readFile($path); + + $pattern = "/[(\s]open(\(((?>[^()]+)|(?1))*\))/"; + if (!preg_match_all($pattern, $fileContent, $matches, + PREG_OFFSET_CAPTURE)) { + return; + } + + foreach ($matches[0] as $match) { + list($open, $offset) = $match; + + $isBin = preg_match("/open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]/", $open); + $hasEncoding = preg_match("/encoding=.(ascii|utf8|utf-8)./", $open); + + if (!$isBin && !$hasEncoding) { + $this->raiseLintAtOffset( + $offset + 1, + self::ENCODING_NOT_FOUND, + pht("Python's open(...) seems to be used to open text files without ". + "explicitly specifying encoding, or with an invalid encoding. ". + "Encoding should be 'ascii', 'utf-8' or 'utf8' (e.g.: ". + "`open(f, 'r', encoding='utf-8')`)."), + substr($open, 1), + null); + } + } + } +} diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -39,7 +39,7 @@ # TODO: implement support for multiple include directories for arg in sorted(files.keys()): module = files[arg] - with open(arg, 'r') as f: + with open(arg, 'r', encoding="utf8") as f: for line in f: match = RE.match(line) if match: diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -160,7 +160,7 @@ def read_file(filename): - return open(os.path.abspath(filename), 'r').read() + return open(os.path.abspath(filename), 'r', encoding="utf8").read() def gather_file_info(filename): @@ -352,14 +352,14 @@ def read_file_lines(filename): - f = open(os.path.abspath(filename), 'r') + f = open(os.path.abspath(filename), 'r', encoding="utf8") file_lines = f.readlines() f.close() return file_lines def write_file_lines(filename, file_lines): - f = open(os.path.abspath(filename), 'w') + f = open(os.path.abspath(filename), 'w', encoding="utf8") f.write(''.join(file_lines)) f.close() diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -11,7 +11,7 @@ def write_testcode(filename): - with open(filename, 'w') as f: + with open(filename, 'w', encoding="utf8") as f: f.write(''' #include int main() diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py --- a/contrib/filter-lcov.py +++ b/contrib/filter-lcov.py @@ -16,8 +16,8 @@ outfile = args.outfile in_remove = False -with open(tracefile, 'r') as f: - with open(outfile, 'w') as wf: +with open(tracefile, 'r', encoding="utf8") as f: + with open(outfile, 'w', encoding="utf8") as wf: for line in f: for p in pattern: if line.startswith("SF:") and p in line: diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -85,7 +85,7 @@ def get_block_hashes(settings): blkindex = [] - f = open(settings['hashlist'], "r") + f = open(settings['hashlist'], "r", encoding="utf8") for line in f: line = line.rstrip() if settings['rev_hash_bytes'] == 'true': @@ -279,7 +279,7 @@ print("Usage: linearize-data.py CONFIG-FILE") sys.exit(1) - f = open(sys.argv[1]) + f = open(sys.argv[1], encoding="utf8") for line in f: # skip comment lines m = re.search(r'^\s*#', line) diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -102,7 +102,7 @@ def get_rpc_cookie(): # Open the cookie file - with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r') as f: + with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r', encoding="ascii") as f: combined = f.readline() combined_split = combined.split(":") settings['rpcuser'] = combined_split[0] @@ -114,7 +114,7 @@ print("Usage: linearize-hashes.py CONFIG-FILE") sys.exit(1) - f = open(sys.argv[1]) + f = open(sys.argv[1], encoding="utf8") for line in f: # skip comment lines m = re.search(r'^\s*#', line) diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -133,10 +133,10 @@ g.write( ' * IPv4 as well as onion addresses are wrapped inside an IPv6 address accordingly.\n') g.write(' */\n') - with open(os.path.join(indir, 'nodes_main.txt'), 'r') as f: + with open(os.path.join(indir, 'nodes_main.txt'), 'r', encoding="utf8") as f: process_nodes(g, f, 'pnSeed6_main', 8333) g.write('\n') - with open(os.path.join(indir, 'nodes_test.txt'), 'r') as f: + with open(os.path.join(indir, 'nodes_test.txt'), 'r', encoding="utf8") as f: process_nodes(g, f, 'pnSeed6_test', 18333) g.write('#endif // BITCOIN_CHAINPARAMSSEEDS_H\n') diff --git a/share/qt/extract_strings_qt.py b/share/qt/extract_strings_qt.py --- a/share/qt/extract_strings_qt.py +++ b/share/qt/extract_strings_qt.py @@ -67,7 +67,7 @@ messages = parse_po(out.decode('utf-8')) -f = open(OUT_CPP, 'w') +f = open(OUT_CPP, 'w', encoding="utf8") f.write(""" #include diff --git a/test/functional/combine_logs.py b/test/functional/combine_logs.py --- a/test/functional/combine_logs.py +++ b/test/functional/combine_logs.py @@ -69,7 +69,7 @@ Log events may be split over multiple lines. We use the timestamp regex match as the marker for a new log event.""" try: - with open(logfile, 'r') as infile: + with open(logfile, 'r', encoding='utf-8') as infile: event = '' timestamp = '' for line in infile: diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -43,7 +43,7 @@ self.block_filename).st_size >= (block_count * 65), timeout=10) # file content should equal the generated blocks hashes - with open(self.block_filename, 'r') as f: + with open(self.block_filename, 'r', encoding="utf-8") as f: assert_equal(sorted(blocks), sorted(f.read().splitlines())) self.log.info("test -walletnotify") @@ -54,7 +54,7 @@ # file content should equal the generated transaction hashes txids_rpc = list( map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count))) - with open(self.tx_filename, 'r') as f: + with open(self.tx_filename, 'r', encoding="ascii") as f: assert_equal(sorted(txids_rpc), sorted(f.read().splitlines())) os.remove(self.tx_filename) @@ -69,7 +69,7 @@ # file content should equal the generated transaction hashes txids_rpc = list( map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count))) - with open(self.tx_filename, 'r') as f: + with open(self.tx_filename, 'r', encoding="ascii") as f: assert_equal(sorted(txids_rpc), sorted(f.read().splitlines())) # Create an invalid chain and ensure the node warns. diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -48,7 +48,7 @@ # Check that bitcoin has been built with ZMQ enabled. config = configparser.ConfigParser() - config.read_file(open(self.options.configfile)) + config.read_file(open(self.options.configfile, encoding='utf-8')) if not config["components"].getboolean("ENABLE_ZMQ"): raise SkipTest("bitcoind has not been built with zmq enabled.") diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -96,12 +96,12 @@ 'mocktime': int(mocktime), 'stats': self.expected_stats, } - with open(filename, 'w') as f: + with open(filename, 'w', encoding="utf8") as f: json.dump(to_dump, f, sort_keys=True, indent=2, default=EncodeDecimal) def load_test_data(self, filename): - with open(filename, 'r') as f: + with open(filename, 'r', encoding="utf8") as f: d = json.load(f, parse_float=decimal.Decimal) blocks = d['blocks'] mocktime = d['mocktime'] diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -35,7 +35,7 @@ self.user = ''.join(SystemRandom().choice( string.ascii_letters + string.digits) for _ in range(10)) config = configparser.ConfigParser() - config.read_file(open(self.options.configfile)) + config.read_file(open(self.options.configfile, encoding='utf-8')) gen_rpcauth = config['environment']['RPCAUTH'] p = subprocess.Popen([gen_rpcauth, self.user], stdout=subprocess.PIPE, universal_newlines=True) diff --git a/test/functional/test_framework/cdefs.py b/test/functional/test_framework/cdefs.py --- a/test/functional/test_framework/cdefs.py +++ b/test/functional/test_framework/cdefs.py @@ -48,7 +48,7 @@ # Slurp in consensus.h contents _consensus_h_fh = open(os.path.join(get_srcdir(), 'src', 'consensus', - 'consensus.h'), 'rt') + 'consensus.h'), 'rt', encoding='utf-8') _consensus_h_contents = _consensus_h_fh.read() _consensus_h_fh.close() diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -121,7 +121,7 @@ self.options.cachedir = os.path.abspath(self.options.cachedir) config = configparser.ConfigParser() - config.read_file(open(self.options.configfile)) + config.read_file(open(self.options.configfile, encoding='utf-8')) self.options.bitcoind = os.getenv( "BITCOIND", default=config["environment"]["BUILDDIR"] + '/src/bitcoind' + config["environment"]["EXEEXT"]) self.options.bitcoincli = os.getenv( @@ -345,7 +345,8 @@ self.log = logging.getLogger('TestFramework') self.log.setLevel(logging.DEBUG) # Create file handler to log all messages - fh = logging.FileHandler(self.options.tmpdir + '/test_framework.log') + fh = logging.FileHandler( + self.options.tmpdir + '/test_framework.log', encoding='utf-8') fh.setLevel(logging.DEBUG) # Create console handler to log messages to stderr. By default this # logs only error messages, but can be configured with --loglevel. diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -371,7 +371,7 @@ assert password is None # Ensure that there is only one rpcpassword line password = line.split("=")[1].strip("\n") if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")): - with open(os.path.join(datadir, "regtest", ".cookie"), 'r') as f: + with open(os.path.join(datadir, "regtest", ".cookie"), 'r', encoding="ascii") as f: userpass = f.read() split_userpass = userpass.split(':') user = split_userpass[0] diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -139,7 +139,7 @@ config = configparser.ConfigParser() configfile = os.path.join(os.path.abspath( os.path.dirname(__file__)), "..", "config.ini") - config.read_file(open(configfile)) + config.read_file(open(configfile, encoding="utf8")) src_dir = config["environment"]["SRCDIR"] build_dir = config["environment"]["BUILDDIR"] @@ -647,7 +647,7 @@ if not os.path.isfile(coverage_ref_filename): raise RuntimeError("No coverage reference found") - with open(coverage_ref_filename, 'r') as f: + with open(coverage_ref_filename, 'r', encoding="utf8") as f: all_cmds.update([i.strip() for i in f.readlines()]) for root, dirs, files in os.walk(self.dir): @@ -656,7 +656,7 @@ coverage_filenames.add(os.path.join(root, filename)) for filename in coverage_filenames: - with open(filename, 'r') as f: + with open(filename, 'r', encoding="utf8") as f: covered_cmds.update([i.strip() for i in f.readlines()]) return all_cmds - covered_cmds @@ -710,7 +710,7 @@ def load_timings(self): if os.path.isfile(self.timing_file): - with open(self.timing_file) as f: + with open(self.timing_file, encoding="utf8") as f: return json.load(f) else: return [] @@ -742,7 +742,7 @@ passed_results)) merged_timings = self.get_merged_timings(new_timings) - with open(self.timing_file, 'w') as f: + with open(self.timing_file, 'w', encoding="utf8") as f: json.dump(merged_timings, f, indent=True) diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -99,7 +99,7 @@ ['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist') # should not initialize if the specified walletdir is not a directory not_a_dir = wallet_dir('notadir') - open(not_a_dir, 'a').close() + open(not_a_dir, 'a', encoding="utf8").close() self.nodes[0].assert_start_raises_init_error( ['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + re.escape(not_a_dir) + '" is not a directory') diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -48,7 +48,7 @@ args_used = set() args_docd = set() for file in files: - with open(file, 'r') as f: + with open(file, 'r', encoding='utf-8') as f: content = f.read() args_used |= set(re.findall(re.compile(REGEX_ARG), content)) args_docd |= set(re.findall(re.compile(REGEX_DOC), content)) diff --git a/test/lint/check-rpc-mappings.py b/test/lint/check-rpc-mappings.py --- a/test/lint/check-rpc-mappings.py +++ b/test/lint/check-rpc-mappings.py @@ -51,7 +51,7 @@ """Find and parse dispatch table in implementation file `fname`.""" cmds = [] in_rpcs = False - with open(fname, "r") as f: + with open(fname, "r", encoding="utf8") as f: for line in f: line = line.rstrip() if not in_rpcs: @@ -80,7 +80,7 @@ """Find and parse conversion table in implementation file `fname`.""" cmds = [] in_rpcs = False - with open(fname, "r") as f: + with open(fname, "r", encoding="utf8") as f: for line in f: line = line.rstrip() if not in_rpcs: diff --git a/test/lint/lint-python-format.py b/test/lint/lint-python-format.py --- a/test/lint/lint-python-format.py +++ b/test/lint/lint-python-format.py @@ -163,7 +163,7 @@ def find_snippets(file): """Find code snippets in the source file that contains the percent ('%') character""" - with open(file, 'r') as f: + with open(file, 'r', encoding='utf-8') as f: snippet_line = "" snippets = {} diff --git a/test/util/bitcoin-util-test.py b/test/util/bitcoin-util-test.py --- a/test/util/bitcoin-util-test.py +++ b/test/util/bitcoin-util-test.py @@ -23,7 +23,8 @@ def main(): config = configparser.ConfigParser() - config.read_file(open(os.path.dirname(__file__) + "/../config.ini")) + config.read_file(open(os.path.join(os.path.dirname( + __file__), "../config.ini"), encoding="utf8")) parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-v', '--verbose', action='store_true') @@ -45,7 +46,7 @@ def bctester(testDir, input_basename, buildenv): """ Loads and parses the input file, runs all tests and reports results""" input_filename = os.path.join(testDir, input_basename) - raw_data = open(input_filename).read() + raw_data = open(input_filename, encoding="utf8").read() input_data = json.loads(raw_data) failed_testcases = [] @@ -84,7 +85,7 @@ inputData = None if "input" in testObj: filename = os.path.join(testDir, testObj["input"]) - inputData = open(filename).read() + inputData = open(filename, encoding="utf8").read() stdinCfg = subprocess.PIPE # Read the expected output data (if there is any) @@ -96,7 +97,8 @@ # output type from file extension (determines how to compare) outputType = os.path.splitext(outputFn)[1][1:] try: - outputData = open(os.path.join(testDir, outputFn)).read() + outputData = open(os.path.join(testDir, outputFn), + encoding="utf8").read() except: logging.error("Output file " + outputFn + " can not be opened") raise diff --git a/test/util/rpcauth-test.py b/test/util/rpcauth-test.py --- a/test/util/rpcauth-test.py +++ b/test/util/rpcauth-test.py @@ -19,7 +19,7 @@ config_path = os.path.abspath( os.path.join(os.sep, os.path.abspath(os.path.dirname(__file__)), "../config.ini")) - with open(config_path) as config_file: + with open(config_path, encoding="utf8") as config_file: config.read_file(config_file) sys.path.insert(0, os.path.dirname(config['environment']['RPCAUTH'])) self.rpcauth = importlib.import_module('rpcauth')