diff --git a/.arclint b/.arclint index 03d34d8eb5..8354387a5f 100644 --- a/.arclint +++ b/.arclint @@ -1,140 +1,147 @@ { "linters": { "generated": { "type": "generated" }, "clang-format": { "type": "clang-format", "version": ">=7.0", "bin": ["clang-format-7", "clang-format"], "include": "(^src/.*\\.(h|c|cpp|mm)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" ] }, "autopep8": { "type": "autopep8", "version": ">=1.3.4", "include": "(\\.py$)" }, "flake8": { "type": "flake8", "include": "(\\.py$)", "flags": [ "--select=E112,E113,E115,E116,E125,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E714,E721,E741,E742,E743,E901,E902,F401,F402,F403,F404,F405,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W292,W293,W601,W602,W603,W604,W605,W606" ] }, "lint-format-strings": { "type": "lint-format-strings", "include": "(^src/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" ] }, "check-doc": { "type": "check-doc", "include": "(^src/.*\\.(h|c|cpp)$)" }, "lint-tests": { "type": "lint-tests", "include": "(^src/(rpc/|wallet/)?test/.*\\.(cpp)$)" }, "lint-python-format": { "type": "lint-python-format", "include": "(\\.py$)", "exclude": [ "(^test/lint/lint-python-format\\.py$)" ] }, "phpcs": { "type": "phpcs", "include": "(\\.php$)", "exclude": [ "(^arcanist/__phutil_library_.+\\.php$)" ], "phpcs.standard": "arcanist/phpcs.xml" }, "lint-locale-dependence": { "type": "lint-locale-dependence", "include": "(^src/.*\\.(h|cpp)$)", "exclude": [ "(^src/(crypto/ctaes/|leveldb/|secp256k1/|seeder/|tinyformat.h|univalue/))" ] }, "lint-cheader": { "type": "lint-cheader", "include": "(^src/.*\\.(h|cpp)$)", "exclude": [ "(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)" ] }, "spelling": { "type": "spelling", "exclude": [ "(^build-aux/m4/)", "(^depends/)", "(^doc/release-notes/)", "(^src/(qt/locale|secp256k1|univalue|leveldb)/)", "(^test/lint/dictionary/)" ], "spelling.dictionaries": [ "test/lint/dictionary/english.json" ] }, "lint-assert-with-side-effects": { "type": "lint-assert-with-side-effects", "include": "(^src/.*\\.(h|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" ] }, "lint-include-quotes": { "type": "lint-include-quotes", "include": "(^src/.*\\.(h|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" ] }, "lint-include-guard": { "type": "lint-include-guard", "include": "(^src/.*\\.h$)", "exclude": [ "(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)", "(^src/tinyformat.h$)" ] }, "lint-include-source": { "type": "lint-include-source", "include": "(^src/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" ] }, "lint-stdint": { "type": "lint-stdint", "include": "(^src/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" ] }, "lint-source-filename": { "type": "lint-source-filename", "include": "(^src/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)" ] }, "lint-boost-dependencies": { "type": "lint-boost-dependencies", "include": "(^src/.*\\.(h|cpp)$)" }, "check-rpc-mappings": { "type": "check-rpc-mappings", "include": "(^src/(rpc/|wallet/rpc).*\\.cpp$)" }, "lint-python-encoding": { "type": "lint-python-encoding", "include": "(\\.py$)" + }, + "lint-python-shebang": { + "type": "lint-python-shebang", + "include": "(\\.py$)", + "exclude": [ + "(__init__\\.py$)" + ] } } } diff --git a/arcanist/.phutil_module_cache b/arcanist/.phutil_module_cache index 1874b79755..1d737f5ddc 100644 --- a/arcanist/.phutil_module_cache +++ b/arcanist/.phutil_module_cache @@ -1 +1 @@ -{"__symbol_cache_version__":11,"ae5b7d3d6b8cf9598ce4abaf0cd56b21":{"have":{"class":{"ExtendedConfigurationDrivenLintEngine":19}},"need":{"function":{"newv":159,"pht":933},"class":{"ArcanistLintEngine":65,"ArcanistConfigurationDrivenLintEngine":171,"PhutilConsole":866},"class\/interface":{"ILintOnce":634}},"xmap":{"ExtendedConfigurationDrivenLintEngine":["ArcanistLintEngine"]}},"2809b09d2021203b43c57da33d1fe8bf":{"have":{"class":{"AssertWithSideEffectsLinter":210}},"need":{"function":{"pht":439},"class":{"ArcanistLinter":246,"ArcanistLintSeverity":926,"Filesystem":1170}},"xmap":{"AssertWithSideEffectsLinter":["ArcanistLinter"]}},"90a8b110dc475955f15bb81d37268cb5":{"have":{"class":{"AutoPEP8FormatLinter":75}},"need":{"function":{"pht":297,"execx":769,"id":1903},"class":{"ArcanistExternalLinter":104,"ArcanistLintMessage":1910,"Filesystem":1754,"ArcanistLinter":2017,"ArcanistLintSeverity":2095}},"xmap":{"AutoPEP8FormatLinter":["ArcanistExternalLinter"]}},"5ea58c19df0397ed8ee0f463d90d6c72":{"have":{"class":{"BoostDependenciesLinter":145}},"need":{"function":{"pht":330,"id":1440},"class":{"GlobalExternalLinter":177,"ArcanistLintMessage":1447,"Filesystem":609,"ArcanistLinter":1524,"ArcanistLintSeverity":1624}},"xmap":{"BoostDependenciesLinter":["GlobalExternalLinter"]}},"38f0c676bff5192a344464142caaa253":{"have":{"class":{"CHeaderLinter":99}},"need":{"function":{"pht":611},"class":{"ArcanistLinter":121,"ArcanistLintSeverity":1060,"Filesystem":1307}},"xmap":{"CHeaderLinter":["ArcanistLinter"]}},"9bb48ec0fe2e9ced8e27d42540d0571c":{"have":{"class":{"CheckDocLinter":106}},"need":{"function":{"pht":321,"id":1845},"class":{"GlobalExternalLinter":129,"ArcanistLintMessage":1852,"Filesystem":729,"ArcanistLinter":1900,"ArcanistLintSeverity":1986}},"xmap":{"CheckDocLinter":["GlobalExternalLinter"]}},"63d19a8745cb2e1200cc26488dc7ad25":{"have":{"class":{"CheckRpcMappingsLinter":131}},"need":{"function":{"pht":310,"id":1386},"class":{"GlobalExternalLinter":162,"ArcanistLintMessage":1393,"Filesystem":573,"ArcanistLinter":1443,"ArcanistLintSeverity":1544}},"xmap":{"CheckRpcMappingsLinter":["GlobalExternalLinter"]}},"6af7410cfea496ff1d4dcc2624b6b8ea":{"have":{"class":{"ClangFormatLinter":79}},"need":{"function":{"pht":302,"execx":781,"id":1653},"class":{"ArcanistExternalLinter":105,"ArcanistLintMessage":1660,"Filesystem":1504,"ArcanistLinter":1767,"ArcanistLintSeverity":1845}},"xmap":{"ClangFormatLinter":["ArcanistExternalLinter"]}},"75579a609dd975aa0226add52700c622":{"have":{"class":{"FileNameLinter":103}},"need":{"function":{"pht":307},"class":{"ArcanistLinter":126,"ArcanistLintSeverity":662,"Filesystem":968}},"xmap":{"FileNameLinter":["ArcanistLinter"]}},"9285ad9415f8ebe564f7119e5a72c559":{"have":{"class":{"FormatStringLinter":146}},"need":{"function":{"pht":377,"csprintf":1492,"id":1872},"class":{"ArcanistExternalLinter":173,"ArcanistLintMessage":1879,"Filesystem":827,"ArcanistLinter":1956,"ArcanistLintSeverity":2044}},"xmap":{"FormatStringLinter":["ArcanistExternalLinter"]}},"2e11dd9ad67e594f863bc46ac59ea37e":{"have":{"class":{"GlobalExternalLinter":199}},"need":{"class":{"ArcanistExternalLinter":228},"interface":{"ILintOnce":262}},"xmap":{"GlobalExternalLinter":["ArcanistExternalLinter","ILintOnce"]}},"b2403124ec3e8be6cb4d10bf0f6c4134":{"have":{"interface":{"ILintOnce":69}},"need":[],"xmap":[]},"1f9bac7956f4f948a187828dcc6ba2d0":{"have":{"class":{"IncludeGuardLinter":98}},"need":{"function":{"pht":368},"class":{"ArcanistLinter":125,"ArcanistLintSeverity":721,"Filesystem":970}},"xmap":{"IncludeGuardLinter":["ArcanistLinter"]}},"2cbb6e7228d81557f777ad648704f343":{"have":{"class":{"IncludeQuotesLinter":100}},"need":{"function":{"pht":306},"class":{"ArcanistLinter":128,"ArcanistLintSeverity":663,"Filesystem":964}},"xmap":{"IncludeQuotesLinter":["ArcanistLinter"]}},"f151089cf79fdb8257b2272ed4782d88":{"have":{"class":{"IncludeSourceLinter":99}},"need":{"function":{"pht":391},"class":{"ArcanistLinter":127,"ArcanistLintSeverity":699,"Filesystem":938}},"xmap":{"IncludeSourceLinter":["ArcanistLinter"]}},"ce5dee893bedb5d93466655eede8ac47":{"have":{"class":{"LocaleDependenceLinter":160}},"need":{"function":{"pht":5429},"class":{"ArcanistLinter":191,"ArcanistLintSeverity":5932,"Filesystem":6178}},"xmap":{"LocaleDependenceLinter":["ArcanistLinter"]}},"6f2f22dd0f259fb2eaa284b4fab3bc29":{"have":{"class":{"PythonFormatLinter":123}},"need":{"function":{"pht":353,"id":1838},"class":{"ArcanistExternalLinter":150,"ArcanistLintMessage":1845,"Filesystem":776,"ArcanistLinter":1970,"ArcanistLintSeverity":2053}},"xmap":{"PythonFormatLinter":["ArcanistExternalLinter"]}},"09a933fbbf135320585be52750d93831":{"have":{"class":{"StdintLinter":90}},"need":{"function":{"pht":280},"class":{"ArcanistLinter":111,"ArcanistLintSeverity":589,"Filesystem":897}},"xmap":{"StdintLinter":["ArcanistLinter"]}},"25781df78f6eebfb223296b8265e9d19":{"have":{"class":{"TestsLinter":103}},"need":{"function":{"pht":318,"id":2629},"class":{"ArcanistExternalLinter":123,"ArcanistLintMessage":2636,"Filesystem":776,"ArcanistLinter":2684,"ArcanistLintSeverity":2792}},"xmap":{"TestsLinter":["ArcanistExternalLinter"]}},"03cf226fe177a9fce7dc046816cc466b":{"have":{"class":{"PythonOpenFileEncodingLinter":111}},"need":{"function":{"pht":325},"class":{"ArcanistLinter":148,"ArcanistLintSeverity":687,"Filesystem":991}},"xmap":{"PythonOpenFileEncodingLinter":["ArcanistLinter"]}}} \ No newline at end of file +{"__symbol_cache_version__":11,"ae5b7d3d6b8cf9598ce4abaf0cd56b21":{"have":{"class":{"ExtendedConfigurationDrivenLintEngine":19}},"need":{"function":{"newv":159,"pht":933},"class":{"ArcanistLintEngine":65,"ArcanistConfigurationDrivenLintEngine":171,"PhutilConsole":866},"class\/interface":{"ILintOnce":634}},"xmap":{"ExtendedConfigurationDrivenLintEngine":["ArcanistLintEngine"]}},"2809b09d2021203b43c57da33d1fe8bf":{"have":{"class":{"AssertWithSideEffectsLinter":210}},"need":{"function":{"pht":439},"class":{"ArcanistLinter":246,"ArcanistLintSeverity":926,"Filesystem":1170}},"xmap":{"AssertWithSideEffectsLinter":["ArcanistLinter"]}},"90a8b110dc475955f15bb81d37268cb5":{"have":{"class":{"AutoPEP8FormatLinter":75}},"need":{"function":{"pht":297,"execx":769,"id":1903},"class":{"ArcanistExternalLinter":104,"ArcanistLintMessage":1910,"Filesystem":1754,"ArcanistLinter":2017,"ArcanistLintSeverity":2095}},"xmap":{"AutoPEP8FormatLinter":["ArcanistExternalLinter"]}},"5ea58c19df0397ed8ee0f463d90d6c72":{"have":{"class":{"BoostDependenciesLinter":145}},"need":{"function":{"pht":330,"id":1440},"class":{"GlobalExternalLinter":177,"ArcanistLintMessage":1447,"Filesystem":609,"ArcanistLinter":1524,"ArcanistLintSeverity":1624}},"xmap":{"BoostDependenciesLinter":["GlobalExternalLinter"]}},"38f0c676bff5192a344464142caaa253":{"have":{"class":{"CHeaderLinter":99}},"need":{"function":{"pht":611},"class":{"ArcanistLinter":121,"ArcanistLintSeverity":1060,"Filesystem":1307}},"xmap":{"CHeaderLinter":["ArcanistLinter"]}},"9bb48ec0fe2e9ced8e27d42540d0571c":{"have":{"class":{"CheckDocLinter":106}},"need":{"function":{"pht":321,"id":1845},"class":{"GlobalExternalLinter":129,"ArcanistLintMessage":1852,"Filesystem":729,"ArcanistLinter":1900,"ArcanistLintSeverity":1986}},"xmap":{"CheckDocLinter":["GlobalExternalLinter"]}},"63d19a8745cb2e1200cc26488dc7ad25":{"have":{"class":{"CheckRpcMappingsLinter":131}},"need":{"function":{"pht":310,"id":1386},"class":{"GlobalExternalLinter":162,"ArcanistLintMessage":1393,"Filesystem":573,"ArcanistLinter":1443,"ArcanistLintSeverity":1544}},"xmap":{"CheckRpcMappingsLinter":["GlobalExternalLinter"]}},"75579a609dd975aa0226add52700c622":{"have":{"class":{"FileNameLinter":103}},"need":{"function":{"pht":307},"class":{"ArcanistLinter":126,"ArcanistLintSeverity":662,"Filesystem":968}},"xmap":{"FileNameLinter":["ArcanistLinter"]}},"9285ad9415f8ebe564f7119e5a72c559":{"have":{"class":{"FormatStringLinter":146}},"need":{"function":{"pht":377,"csprintf":1492,"id":1872},"class":{"ArcanistExternalLinter":173,"ArcanistLintMessage":1879,"Filesystem":827,"ArcanistLinter":1956,"ArcanistLintSeverity":2044}},"xmap":{"FormatStringLinter":["ArcanistExternalLinter"]}},"2e11dd9ad67e594f863bc46ac59ea37e":{"have":{"class":{"GlobalExternalLinter":199}},"need":{"class":{"ArcanistExternalLinter":228},"interface":{"ILintOnce":262}},"xmap":{"GlobalExternalLinter":["ArcanistExternalLinter","ILintOnce"]}},"b2403124ec3e8be6cb4d10bf0f6c4134":{"have":{"interface":{"ILintOnce":69}},"need":[],"xmap":[]},"1f9bac7956f4f948a187828dcc6ba2d0":{"have":{"class":{"IncludeGuardLinter":98}},"need":{"function":{"pht":368},"class":{"ArcanistLinter":125,"ArcanistLintSeverity":721,"Filesystem":970}},"xmap":{"IncludeGuardLinter":["ArcanistLinter"]}},"2cbb6e7228d81557f777ad648704f343":{"have":{"class":{"IncludeQuotesLinter":100}},"need":{"function":{"pht":306},"class":{"ArcanistLinter":128,"ArcanistLintSeverity":663,"Filesystem":964}},"xmap":{"IncludeQuotesLinter":["ArcanistLinter"]}},"f151089cf79fdb8257b2272ed4782d88":{"have":{"class":{"IncludeSourceLinter":99}},"need":{"function":{"pht":391},"class":{"ArcanistLinter":127,"ArcanistLintSeverity":699,"Filesystem":938}},"xmap":{"IncludeSourceLinter":["ArcanistLinter"]}},"ce5dee893bedb5d93466655eede8ac47":{"have":{"class":{"LocaleDependenceLinter":160}},"need":{"function":{"pht":5429},"class":{"ArcanistLinter":191,"ArcanistLintSeverity":5932,"Filesystem":6178}},"xmap":{"LocaleDependenceLinter":["ArcanistLinter"]}},"6f2f22dd0f259fb2eaa284b4fab3bc29":{"have":{"class":{"PythonFormatLinter":123}},"need":{"function":{"pht":353,"id":1838},"class":{"ArcanistExternalLinter":150,"ArcanistLintMessage":1845,"Filesystem":776,"ArcanistLinter":1970,"ArcanistLintSeverity":2053}},"xmap":{"PythonFormatLinter":["ArcanistExternalLinter"]}},"03cf226fe177a9fce7dc046816cc466b":{"have":{"class":{"PythonOpenFileEncodingLinter":111}},"need":{"function":{"pht":325},"class":{"ArcanistLinter":148,"ArcanistLintSeverity":687,"Filesystem":991}},"xmap":{"PythonOpenFileEncodingLinter":["ArcanistLinter"]}},"09a933fbbf135320585be52750d93831":{"have":{"class":{"StdintLinter":90}},"need":{"function":{"pht":280},"class":{"ArcanistLinter":111,"ArcanistLintSeverity":589,"Filesystem":897}},"xmap":{"StdintLinter":["ArcanistLinter"]}},"25781df78f6eebfb223296b8265e9d19":{"have":{"class":{"TestsLinter":103}},"need":{"function":{"pht":318,"id":2629},"class":{"ArcanistExternalLinter":123,"ArcanistLintMessage":2636,"Filesystem":776,"ArcanistLinter":2684,"ArcanistLintSeverity":2792}},"xmap":{"TestsLinter":["ArcanistExternalLinter"]}},"454160ed72e88797129d84ee589de304":{"have":{"class":{"PythonShebangLinter":87}},"need":{"function":{"pht":290},"class":{"ArcanistLinter":115,"ArcanistLintSeverity":614,"Filesystem":854}},"xmap":{"PythonShebangLinter":["ArcanistLinter"]}},"1c03a4631b02b8e1ebcc3a078dc1e4af":{"have":{"class":{"ClangFormatLinter":79}},"need":{"function":{"pht":302,"execx":781,"id":2235},"class":{"ArcanistExternalLinter":105,"ArcanistLintMessage":2242,"Filesystem":2086,"ArcanistLinter":2349,"ArcanistLintSeverity":2427}},"xmap":{"ClangFormatLinter":["ArcanistExternalLinter"]}}} \ No newline at end of file diff --git a/arcanist/__phutil_library_map__.php b/arcanist/__phutil_library_map__.php index 4452ff9f93..dd2d9dc029 100644 --- a/arcanist/__phutil_library_map__.php +++ b/arcanist/__phutil_library_map__.php @@ -1,58 +1,60 @@ 2, 'class' => array( 'AssertWithSideEffectsLinter' => 'linter/AssertWithSideEffectsLinter.php', 'AutoPEP8FormatLinter' => 'linter/AutoPEP8Linter.php', 'BoostDependenciesLinter' => 'linter/BoostDependenciesLinter.php', 'CHeaderLinter' => 'linter/CHeaderLinter.php', 'CheckDocLinter' => 'linter/CheckDocLinter.php', 'CheckRpcMappingsLinter' => 'linter/CheckRpcMappingsLinter.php', 'ClangFormatLinter' => 'linter/ClangFormatLinter.php', 'ExtendedConfigurationDrivenLintEngine' => 'engine/ExtendedConfigurationDrivenLintEngine.php', 'FileNameLinter' => 'linter/FileNameLinter.php', 'FormatStringLinter' => 'linter/FormatStringLinter.php', 'GlobalExternalLinter' => 'linter/GlobalExternalLinter.php', 'ILintOnce' => 'linter/ILintOnce.php', 'IncludeGuardLinter' => 'linter/IncludeGuardLinter.php', 'IncludeQuotesLinter' => 'linter/IncludeQuotesLinter.php', 'IncludeSourceLinter' => 'linter/IncludeSourceLinter.php', 'LocaleDependenceLinter' => 'linter/LocaleDependenceLinter.php', 'PythonFormatLinter' => 'linter/PythonFormatLinter.php', 'PythonOpenFileEncodingLinter' => 'linter/PythonOpenFileEncodingLinter.php', + 'PythonShebangLinter' => 'linter/PythonShebangLinter.php', 'StdintLinter' => 'linter/StdintLinter.php', 'TestsLinter' => 'linter/TestsLinter.php', ), 'function' => array(), 'xmap' => array( 'AssertWithSideEffectsLinter' => 'ArcanistLinter', 'AutoPEP8FormatLinter' => 'ArcanistExternalLinter', 'BoostDependenciesLinter' => 'GlobalExternalLinter', 'CHeaderLinter' => 'ArcanistLinter', 'CheckDocLinter' => 'GlobalExternalLinter', 'CheckRpcMappingsLinter' => 'GlobalExternalLinter', 'ClangFormatLinter' => 'ArcanistExternalLinter', 'ExtendedConfigurationDrivenLintEngine' => 'ArcanistLintEngine', 'FileNameLinter' => 'ArcanistLinter', 'FormatStringLinter' => 'ArcanistExternalLinter', 'GlobalExternalLinter' => array( 'ArcanistExternalLinter', 'ILintOnce', ), 'IncludeGuardLinter' => 'ArcanistLinter', 'IncludeQuotesLinter' => 'ArcanistLinter', 'IncludeSourceLinter' => 'ArcanistLinter', 'LocaleDependenceLinter' => 'ArcanistLinter', 'PythonFormatLinter' => 'ArcanistExternalLinter', 'PythonOpenFileEncodingLinter' => 'ArcanistLinter', + 'PythonShebangLinter' => 'ArcanistLinter', 'StdintLinter' => 'ArcanistLinter', 'TestsLinter' => 'ArcanistExternalLinter', ), )); diff --git a/arcanist/linter/PythonShebangLinter.php b/arcanist/linter/PythonShebangLinter.php new file mode 100644 index 0000000000..e86f42f8db --- /dev/null +++ b/arcanist/linter/PythonShebangLinter.php @@ -0,0 +1,48 @@ + ArcanistLintSeverity::SEVERITY_ERROR, + ); + } + + public function getLintNameMap() { + return array( + self::BAD_SHEBANG_FOUND => pht('Missing or unexpected shebang.'), + ); + } + + public function lintPath($path) { + $absPath = Filesystem::resolvePath($path, $this->getProjectRoot()); + $fileContent = Filesystem::readFile($absPath); + + if (!preg_match_all("%^#!/usr/bin/env python3%", $fileContent)) { + return $this->raiseLintAtPath( + self::BAD_SHEBANG_FOUND, + pht("Shebang should be #!/usr/bin/env python3")); + } + } +} diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 62b431a4e6..f1639cb1ff 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,66 +1,65 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015-2016 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Test script for security-check.py ''' -from __future__ import division, print_function import subprocess import unittest def write_testcode(filename): with open(filename, 'w', encoding="utf8") as f: f.write(''' #include int main() { printf("the quick brown fox jumps over the lazy god\\n"); return 0; } ''') def call_security_check(cc, source, executable, options): subprocess.check_call([cc, source, '-o', executable] + options) - p = subprocess.Popen(['./security-check.py', executable], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + p = subprocess.Popen(['./security-check.py', executable], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() return (p.returncode, stdout.rstrip()) class TestSecurityChecks(unittest.TestCase): def test_ELF(self): source = 'test1.c' executable = 'test1' cc = 'gcc' write_testcode(source) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack', '-fno-stack-protector', '-Wl,-znorelro']), (1, executable + ': failed PIE NX RELRO Canary')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack', '-fno-stack-protector', '-Wl,-znorelro']), (1, executable + ': failed PIE RELRO Canary')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack', '-fstack-protector-all', '-Wl,-znorelro']), (1, executable + ': failed PIE RELRO')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack', '-fstack-protector-all', '-Wl,-znorelro', '-pie', '-fPIE']), (1, executable + ': failed RELRO')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack', '-fstack-protector-all', '-Wl,-zrelro', '-Wl,-z,now', '-pie', '-fPIE']), (0, '')) def test_PE(self): source = 'test1.c' executable = 'test1.exe' cc = 'i686-w64-mingw32-gcc' write_testcode(source) self.assertEqual(call_security_check(cc, source, executable, []), (1, executable + ': failed PIE NX')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat']), (1, executable + ': failed PIE')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat', '-Wl,--dynamicbase']), (0, '')) if __name__ == '__main__': unittest.main() diff --git a/contrib/testgen/base58.py b/contrib/testgen/base58.py index 66077ba76a..c098cbe180 100644 --- a/contrib/testgen/base58.py +++ b/contrib/testgen/base58.py @@ -1,126 +1,127 @@ +#!/usr/bin/env python3 # Copyright (c) 2012-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Bitcoin base58 encoding and decoding. Based on https://bitcointalk.org/index.php?topic=1026.0 (public domain) ''' import hashlib # for compatibility with following code... class SHA256: new = hashlib.sha256 if str != bytes: # Python 3.x def ord(c): return c def chr(n): return bytes((n,)) __b58chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' __b58base = len(__b58chars) b58chars = __b58chars def b58encode(v): """ encode v, which is a string of bytes, to base58. """ long_value = 0 for (i, c) in enumerate(v[::-1]): if isinstance(c, str): c = ord(c) long_value += (256**i) * c result = '' while long_value >= __b58base: div, mod = divmod(long_value, __b58base) result = __b58chars[mod] + result long_value = div result = __b58chars[long_value] + result # Bitcoin does a little leading-zero-compression: # leading 0-bytes in the input become leading-1s nPad = 0 for c in v: if c == 0: nPad += 1 else: break return (__b58chars[0] * nPad) + result def b58decode(v, length=None): """ decode v into a string of len bytes """ long_value = 0 for i, c in enumerate(v[::-1]): pos = __b58chars.find(c) assert pos != -1 long_value += pos * (__b58base**i) result = bytes() while long_value >= 256: div, mod = divmod(long_value, 256) result = chr(mod) + result long_value = div result = chr(long_value) + result nPad = 0 for c in v: if c == __b58chars[0]: nPad += 1 continue break result = bytes(nPad) + result if length is not None and len(result) != length: return None return result def checksum(v): """Return 32-bit checksum based on SHA256""" return SHA256.new(SHA256.new(v).digest()).digest()[0:4] def b58encode_chk(v): """b58encode a string, with 32-bit checksum""" return b58encode(v + checksum(v)) def b58decode_chk(v): """decode a base58 string, check and remove checksum""" result = b58decode(v) if result is None: return None if result[-4:] == checksum(result[:-4]): return result[:-4] else: return None def get_bcaddress_version(strAddress): """ Returns None if strAddress is invalid. Otherwise returns integer version of address. """ addr = b58decode_chk(strAddress) if addr is None or len(addr) != 21: return None version = addr[0] return ord(version) if __name__ == '__main__': # Test case (from http://gitorious.org/bitcoin/python-base58.git) assert get_bcaddress_version('15VjRaDX9zpbA8LVnbrCAFzrVzN7ixHNsC') is 0 _ohai = 'o hai'.encode('ascii') _tmp = b58encode(_ohai) assert _tmp == 'DYB3oMS' assert b58decode(_tmp, 5) == _ohai print("Tests passed") diff --git a/src/bench/data/convert-raw-to-header.py b/src/bench/data/convert-raw-to-header.py index 0291835f6e..3063d9aeb5 100755 --- a/src/bench/data/convert-raw-to-header.py +++ b/src/bench/data/convert-raw-to-header.py @@ -1,21 +1,21 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2019 The Bitcoin developers import sys def main(test_name, input_file): with open(input_file, "rb") as f: contents = f.read() print("static unsigned const char {}[] = {{".format(test_name)) print(", ".join(map(lambda x: "0x{:02x}".format(ord(x)), contents))) print("};") if __name__ == "__main__": if len(sys.argv) != 3: print("We need additional pylons!") sys.exit(1) main(sys.argv[1], sys.argv[2]) diff --git a/src/test/data/generate_header.py b/src/test/data/generate_header.py index c21e38b341..544cb9cd0f 100755 --- a/src/test/data/generate_header.py +++ b/src/test/data/generate_header.py @@ -1,24 +1,24 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2018 The Bitcoin developers import sys def main(test_name, input_file): with open(input_file, "rb") as f: contents = f.read() print("#include \n") print("namespace json_tests {") print("static const uint8_t {}[] = {{".format(test_name)) print(", ".join(map(lambda x: "0x{:02x}".format(ord(x)), contents))) print("};") print("};") if __name__ == "__main__": if len(sys.argv) != 3: print("We need additional pylons!") sys.exit(1) main(sys.argv[1], sys.argv[2]) diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 977a7aead4..e08031f31e 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -1,193 +1,194 @@ +#!/usr/bin/env python3 # Copyright (c) 2011 Jeff Garzik # # Previous copyright, from python-jsonrpc/jsonrpc/proxy.py: # # Copyright (c) 2007 Jan-Klaas Kollhof # # This file is part of jsonrpc. # # jsonrpc is free software; you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation; either version 2.1 of the License, or # (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this software; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """HTTP proxy for opening RPC connection to bitcoind. AuthServiceProxy has the following improvements over python-jsonrpc's ServiceProxy class: - HTTP connections persist for the life of the AuthServiceProxy object (if server supports HTTP/1.1) - sends protocol 'version', per JSON-RPC 1.1 - sends proper, incrementing 'id' - sends Basic HTTP authentication headers - parses all JSON numbers that look like floats as Decimal - uses standard Python json lib """ import base64 import decimal import http.client import json import logging import socket import time import urllib.parse HTTP_TIMEOUT = 30 USER_AGENT = "AuthServiceProxy/0.1" log = logging.getLogger("BitcoinRPC") class JSONRPCException(Exception): def __init__(self, rpc_error): try: errmsg = '{} ({})'.format(rpc_error['message'], rpc_error['code']) except (KeyError, TypeError): errmsg = '' super().__init__(errmsg) self.error = rpc_error def EncodeDecimal(o): if isinstance(o, decimal.Decimal): return str(o) raise TypeError(repr(o) + " is not JSON serializable") class AuthServiceProxy(): __id_count = 0 # ensure_ascii: escape unicode as \uXXXX, passed to json.dumps def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connection=None, ensure_ascii=True): self.__service_url = service_url self._service_name = service_name self.ensure_ascii = ensure_ascii # can be toggled on the fly by tests self.__url = urllib.parse.urlparse(service_url) port = 80 if self.__url.port is None else self.__url.port user = None if self.__url.username is None else self.__url.username.encode( 'utf8') passwd = None if self.__url.password is None else self.__url.password.encode( 'utf8') authpair = user + b':' + passwd self.__auth_header = b'Basic ' + base64.b64encode(authpair) if connection: # Callables re-use the connection of the original proxy self.__conn = connection elif self.__url.scheme == 'https': self.__conn = http.client.HTTPSConnection( self.__url.hostname, port, timeout=timeout) else: self.__conn = http.client.HTTPConnection( self.__url.hostname, port, timeout=timeout) def __getattr__(self, name): if name.startswith('__') and name.endswith('__'): # Python internal stuff raise AttributeError if self._service_name is not None: name = "{}.{}".format(self._service_name, name) return AuthServiceProxy(self.__service_url, name, connection=self.__conn) def _request(self, method, path, postdata): ''' Do a HTTP request, with retry if we get disconnected (e.g. due to a timeout). This is a workaround for https://bugs.python.org/issue3566 which is fixed in Python 3.5. ''' headers = {'Host': self.__url.hostname, 'User-Agent': USER_AGENT, 'Authorization': self.__auth_header, 'Content-type': 'application/json'} try: self.__conn.request(method, path, postdata, headers) return self._get_response() except http.client.BadStatusLine as e: if e.line == "''": # if connection was closed, try again self.__conn.close() self.__conn.request(method, path, postdata, headers) return self._get_response() else: raise except (BrokenPipeError, ConnectionResetError): # Python 3.5+ raises BrokenPipeError instead of BadStatusLine when the connection was reset # ConnectionResetError happens on FreeBSD with Python 3.4 self.__conn.close() self.__conn.request(method, path, postdata, headers) return self._get_response() def get_request(self, *args, **argsn): AuthServiceProxy.__id_count += 1 log.debug("-{}-> {} {}".format( AuthServiceProxy.__id_count, self._service_name, json.dumps( args, default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) if args and argsn: raise ValueError( 'Cannot handle both named and positional arguments') return {'version': '1.1', 'method': self._service_name, 'params': args or argsn, 'id': AuthServiceProxy.__id_count} def __call__(self, *args, **argsn): postdata = json.dumps(self.get_request( *args, **argsn), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) response = self._request( 'POST', self.__url.path, postdata.encode('utf-8')) if response['error'] is not None: raise JSONRPCException(response['error']) elif 'result' not in response: raise JSONRPCException({ 'code': -343, 'message': 'missing JSON-RPC result'}) else: return response['result'] def batch(self, rpc_call_list): postdata = json.dumps( list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) log.debug("--> " + postdata) return self._request('POST', self.__url.path, postdata.encode('utf-8')) def _get_response(self): req_start_time = time.time() try: http_response = self.__conn.getresponse() except socket.timeout: raise JSONRPCException({ 'code': -344, 'message': '{!r} RPC took longer than {} seconds. Consider ' 'using larger timeout for calls that take ' 'longer to return.'.format((self._service_name, self.__conn.timeout))}) if http_response is None: raise JSONRPCException({ 'code': -342, 'message': 'missing HTTP response from server'}) content_type = http_response.getheader('Content-Type') if content_type != 'application/json': raise JSONRPCException({ 'code': -342, 'message': 'non-JSON HTTP response with \'{} {}\' from server'.format( http_response.status, http_response.reason)}) responsedata = http_response.read().decode('utf8') response = json.loads(responsedata, parse_float=decimal.Decimal) elapsed = time.time() - req_start_time if "error" in response and response["error"] is None: log.debug("<-{}- [{:.6f}] {}".format(response["id"], elapsed, json.dumps( response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) else: log.debug("<-- [{:.6f}] {}".format(elapsed, responsedata)) return response def __truediv__(self, relative_uri): return AuthServiceProxy("{}/{}".format(self.__service_url, relative_uri), self._service_name, connection=self.__conn) diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index 625269bf4a..c99da0db5c 100755 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -1,242 +1,243 @@ +#!/usr/bin/env python3 # Copyright (c) 2011 Sam Rushing """ECC secp256k1 OpenSSL wrapper. WARNING: This module does not mlock() secrets; your private keys may end up on disk in swap! Use with caution! This file is modified from python-bitcoinlib. """ import ctypes import ctypes.util import hashlib ssl = ctypes.cdll.LoadLibrary(ctypes.util.find_library('ssl') or 'libeay32') ssl.BN_new.restype = ctypes.c_void_p ssl.BN_new.argtypes = [] ssl.BN_free.restype = None ssl.BN_free.argtypes = [ctypes.c_void_p] ssl.BN_bin2bn.restype = ctypes.c_void_p ssl.BN_bin2bn.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.c_void_p] ssl.BN_CTX_free.restype = None ssl.BN_CTX_free.argtypes = [ctypes.c_void_p] ssl.BN_CTX_new.restype = ctypes.c_void_p ssl.BN_CTX_new.argtypes = [] ssl.ECDH_compute_key.restype = ctypes.c_int ssl.ECDH_compute_key.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p] ssl.ECDSA_sign.restype = ctypes.c_int ssl.ECDSA_sign.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] ssl.ECDSA_verify.restype = ctypes.c_int ssl.ECDSA_verify.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p] ssl.EC_KEY_free.restype = None ssl.EC_KEY_free.argtypes = [ctypes.c_void_p] ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p ssl.EC_KEY_new_by_curve_name.argtypes = [ctypes.c_int] ssl.EC_KEY_get0_group.restype = ctypes.c_void_p ssl.EC_KEY_get0_group.argtypes = [ctypes.c_void_p] ssl.EC_KEY_get0_public_key.restype = ctypes.c_void_p ssl.EC_KEY_get0_public_key.argtypes = [ctypes.c_void_p] ssl.EC_KEY_set_private_key.restype = ctypes.c_int ssl.EC_KEY_set_private_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p] ssl.EC_KEY_set_conv_form.restype = None ssl.EC_KEY_set_conv_form.argtypes = [ctypes.c_void_p, ctypes.c_int] ssl.EC_KEY_set_public_key.restype = ctypes.c_int ssl.EC_KEY_set_public_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p] ssl.i2o_ECPublicKey.restype = ctypes.c_void_p ssl.i2o_ECPublicKey.argtypes = [ctypes.c_void_p, ctypes.c_void_p] ssl.EC_POINT_new.restype = ctypes.c_void_p ssl.EC_POINT_new.argtypes = [ctypes.c_void_p] ssl.EC_POINT_free.restype = None ssl.EC_POINT_free.argtypes = [ctypes.c_void_p] ssl.EC_POINT_mul.restype = ctypes.c_int ssl.EC_POINT_mul.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] # this specifies the curve used with ECDSA. NID_secp256k1 = 714 # from openssl/obj_mac.h SECP256K1_ORDER = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141 SECP256K1_ORDER_HALF = SECP256K1_ORDER // 2 # Thx to Sam Devlin for the ctypes magic 64-bit fix. def _check_result(val, func, args): if val == 0: raise ValueError else: return ctypes.c_void_p(val) ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p ssl.EC_KEY_new_by_curve_name.errcheck = _check_result class CECKey(): """Wrapper around OpenSSL's EC_KEY""" POINT_CONVERSION_COMPRESSED = 2 POINT_CONVERSION_UNCOMPRESSED = 4 def __init__(self): self.k = ssl.EC_KEY_new_by_curve_name(NID_secp256k1) def __del__(self): if ssl: ssl.EC_KEY_free(self.k) self.k = None def set_secretbytes(self, secret): priv_key = ssl.BN_bin2bn(secret, 32, ssl.BN_new()) group = ssl.EC_KEY_get0_group(self.k) pub_key = ssl.EC_POINT_new(group) ctx = ssl.BN_CTX_new() if not ssl.EC_POINT_mul(group, pub_key, priv_key, None, None, ctx): raise ValueError( "Could not derive public key from the supplied secret.") ssl.EC_POINT_mul(group, pub_key, priv_key, None, None, ctx) ssl.EC_KEY_set_private_key(self.k, priv_key) ssl.EC_KEY_set_public_key(self.k, pub_key) ssl.BN_free(priv_key) ssl.EC_POINT_free(pub_key) ssl.BN_CTX_free(ctx) return self.k def set_privkey(self, key): self.mb = ctypes.create_string_buffer(key) return ssl.d2i_ECPrivateKey(ctypes.byref(self.k), ctypes.byref(ctypes.pointer(self.mb)), len(key)) def set_pubkey(self, key): self.mb = ctypes.create_string_buffer(key) return ssl.o2i_ECPublicKey(ctypes.byref(self.k), ctypes.byref(ctypes.pointer(self.mb)), len(key)) def get_privkey(self): size = ssl.i2d_ECPrivateKey(self.k, 0) mb_pri = ctypes.create_string_buffer(size) ssl.i2d_ECPrivateKey(self.k, ctypes.byref(ctypes.pointer(mb_pri))) return mb_pri.raw def get_pubkey(self): size = ssl.i2o_ECPublicKey(self.k, 0) mb = ctypes.create_string_buffer(size) ssl.i2o_ECPublicKey(self.k, ctypes.byref(ctypes.pointer(mb))) return mb.raw def get_raw_ecdh_key(self, other_pubkey): ecdh_keybuffer = ctypes.create_string_buffer(32) r = ssl.ECDH_compute_key(ctypes.pointer(ecdh_keybuffer), 32, ssl.EC_KEY_get0_public_key(other_pubkey.k), self.k, 0) if r != 32: raise Exception('CKey.get_ecdh_key(): ECDH_compute_key() failed') return ecdh_keybuffer.raw def get_ecdh_key(self, other_pubkey, kdf=lambda k: hashlib.sha256(k).digest()): # FIXME: be warned it's not clear what the kdf should be as a default r = self.get_raw_ecdh_key(other_pubkey) return kdf(r) def sign(self, hash, low_s=True): # FIXME: need unit tests for below cases if not isinstance(hash, bytes): raise TypeError( 'Hash must be bytes instance; got {!r}'.format(hash.__class__)) if len(hash) != 32: raise ValueError('Hash must be exactly 32 bytes long') sig_size0 = ctypes.c_uint32() sig_size0.value = ssl.ECDSA_size(self.k) mb_sig = ctypes.create_string_buffer(sig_size0.value) result = ssl.ECDSA_sign(0, hash, len( hash), mb_sig, ctypes.byref(sig_size0), self.k) assert 1 == result assert mb_sig.raw[0] == 0x30 assert mb_sig.raw[1] == sig_size0.value - 2 total_size = mb_sig.raw[1] assert mb_sig.raw[2] == 2 r_size = mb_sig.raw[3] assert mb_sig.raw[4 + r_size] == 2 s_size = mb_sig.raw[5 + r_size] s_value = int.from_bytes( mb_sig.raw[6 + r_size:6 + r_size + s_size], byteorder='big') if (not low_s) or s_value <= SECP256K1_ORDER_HALF: return mb_sig.raw[:sig_size0.value] else: low_s_value = SECP256K1_ORDER - s_value low_s_bytes = (low_s_value).to_bytes(33, byteorder='big') while len(low_s_bytes) > 1 and low_s_bytes[0] == 0 and low_s_bytes[1] < 0x80: low_s_bytes = low_s_bytes[1:] new_s_size = len(low_s_bytes) new_total_size_byte = ( total_size + new_s_size - s_size).to_bytes(1, byteorder='big') new_s_size_byte = (new_s_size).to_bytes(1, byteorder='big') return b'\x30' + new_total_size_byte + mb_sig.raw[2:5 + r_size] + new_s_size_byte + low_s_bytes def verify(self, hash, sig): """Verify a DER signature""" return ssl.ECDSA_verify(0, hash, len(hash), sig, len(sig), self.k) == 1 def set_compressed(self, compressed): if compressed: form = self.POINT_CONVERSION_COMPRESSED else: form = self.POINT_CONVERSION_UNCOMPRESSED ssl.EC_KEY_set_conv_form(self.k, form) class CPubKey(bytes): """An encapsulated public key Attributes: is_valid - Corresponds to CPubKey.IsValid() is_fullyvalid - Corresponds to CPubKey.IsFullyValid() is_compressed - Corresponds to CPubKey.IsCompressed() """ def __new__(cls, buf, _cec_key=None): self = super(CPubKey, cls).__new__(cls, buf) if _cec_key is None: _cec_key = CECKey() self._cec_key = _cec_key self.is_fullyvalid = _cec_key.set_pubkey(self) != 0 return self @property def is_valid(self): return len(self) > 0 @property def is_compressed(self): return len(self) == 33 def verify(self, hash, sig): return self._cec_key.verify(hash, sig) def __str__(self): return repr(self) def __repr__(self): return '{}({})'.format(self.__class__.__name__, super(CPubKey, self).__repr__()) diff --git a/test/functional/test_framework/txtools.py b/test/functional/test_framework/txtools.py index a163468860..27157ef0a0 100644 --- a/test/functional/test_framework/txtools.py +++ b/test/functional/test_framework/txtools.py @@ -1,70 +1,71 @@ +#!/usr/bin/env python3 import random from .cdefs import MAX_TXOUT_PUBKEY_SCRIPT, MIN_TX_SIZE from .messages import CTransaction, CTxOut, FromHex, ToHex from .script import CScript, OP_RETURN def pad_tx(tx, pad_to_size=MIN_TX_SIZE): """ Pad a transaction with op_return junk data until it is at least pad_to_size, or leave it alone if it's already bigger than that. """ curr_size = len(tx.serialize()) if curr_size >= pad_to_size: # Bail early txn is already big enough return # This code attempts to pad a transaction with opreturn vouts such that # it will be exactly pad_to_size. In order to do this we have to create # vouts of size x (maximum OP_RETURN size - vout overhead), plus the final # one subsumes any runoff which would be less than vout overhead. # # There are two cases where this is not possible: # 1. The transaction size is between pad_to_size and pad_to_size - extrabytes # 2. The transaction is already greater than pad_to_size # # Visually: # | .. x .. | .. x .. | .. x .. | .. x + desired_size % x | # VOUT_1 VOUT_2 VOUT_3 VOUT_4 # txout.value + txout.pk_script bytes + op_return extra_bytes = 8 + 1 + 1 required_padding = pad_to_size - curr_size while required_padding > 0: # We need at least extra_bytes left over each time, or we can't # subsume the final (and possibly undersized) iteration of the loop padding_len = min(required_padding, MAX_TXOUT_PUBKEY_SCRIPT - extra_bytes) assert padding_len >= 0, "Can't pad less than 0 bytes, trying {}".format( padding_len) # We will end up with less than 1 UTXO of bytes after this, add # them to this txn next_iteration_padding = required_padding - padding_len - extra_bytes if next_iteration_padding > 0 and next_iteration_padding < extra_bytes: padding_len += next_iteration_padding # If we're at exactly, or below, extra_bytes we don't want a 1 extra byte padding if padding_len <= extra_bytes: tx.vout.append(CTxOut(0, CScript([OP_RETURN]))) else: # Subtract the overhead for the TxOut padding_len -= extra_bytes padding = random.randrange( 1 << 8 * padding_len - 2, 1 << 8 * padding_len - 1) tx.vout.append( CTxOut(0, CScript([OP_RETURN, padding]))) curr_size = len(tx.serialize()) required_padding = pad_to_size - curr_size assert curr_size >= pad_to_size, "{} !>= {}".format(curr_size, pad_to_size) tx.rehash() def pad_raw_tx(rawtx_hex, min_size=MIN_TX_SIZE): """ Pad a raw transaction with OP_RETURN data until it reaches at least min_size """ tx = CTransaction() FromHex(tx, rawtx_hex) pad_tx(tx, min_size) return ToHex(tx)